Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jikos/trivial

[deliverable/linux.git] / include / linux / percpu-refcount.h

diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 5d8920e23073d8caff4ebe834eaab219e53dba0f..3dfbf237cd8f32fc684e2934e7a8ace0da043b14 100644 (file)
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -57,11 +57,9 @@ struct percpu_ref {
        atomic_t                count;
        /*
         * The low bit of the pointer indicates whether the ref is in percpu
-        * mode; if set, then get/put will manipulate the atomic_t (this is a
-        * hack because we need to keep the pointer around for
-        * percpu_ref_kill_rcu())
+        * mode; if set, then get/put will manipulate the atomic_t.
         */
-       unsigned __percpu       *pcpu_count;
+       unsigned long           pcpu_count_ptr;
        percpu_ref_func_t       *release;
        percpu_ref_func_t       *confirm_kill;
        struct rcu_head         rcu;
@@ -69,7 +67,8 @@ struct percpu_ref {
 
 int __must_check percpu_ref_init(struct percpu_ref *ref,
                                 percpu_ref_func_t *release);
-void percpu_ref_cancel_init(struct percpu_ref *ref);
+void percpu_ref_reinit(struct percpu_ref *ref);
+void percpu_ref_exit(struct percpu_ref *ref);
 void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
                                 percpu_ref_func_t *confirm_kill);
 
@@ -88,12 +87,28 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
        return percpu_ref_kill_and_confirm(ref, NULL);
 }
 
-#define PCPU_STATUS_BITS       2
-#define PCPU_STATUS_MASK       ((1 << PCPU_STATUS_BITS) - 1)
-#define PCPU_REF_PTR           0
 #define PCPU_REF_DEAD          1
 
-#define REF_STATUS(count)      (((unsigned long) count) & PCPU_STATUS_MASK)
+/*
+ * Internal helper.  Don't use outside percpu-refcount proper.  The
+ * function doesn't return the pointer and let the caller test it for NULL
+ * because doing so forces the compiler to generate two conditional
+ * branches as it can't assume that @ref->pcpu_count is not NULL.
+ */
+static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
+                                   unsigned __percpu **pcpu_countp)
+{
+       unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
+
+       /* paired with smp_store_release() in percpu_ref_reinit() */
+       smp_read_barrier_depends();
+
+       if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
+               return false;
+
+       *pcpu_countp = (unsigned __percpu *)pcpu_ptr;
+       return true;
+}
 
 /**
  * percpu_ref_get - increment a percpu refcount
@@ -107,9 +122,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
 
        rcu_read_lock_sched();
 
-       pcpu_count = ACCESS_ONCE(ref->pcpu_count);
-
-       if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
+       if (__pcpu_ref_alive(ref, &pcpu_count))
                this_cpu_inc(*pcpu_count);
        else
                atomic_inc(&ref->count);
@@ -133,9 +146,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
 
        rcu_read_lock_sched();
 
-       pcpu_count = ACCESS_ONCE(ref->pcpu_count);
-
-       if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
+       if (__pcpu_ref_alive(ref, &pcpu_count)) {
                this_cpu_inc(*pcpu_count);
                ret = true;
        } else {
@@ -168,9 +179,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
 
        rcu_read_lock_sched();
 
-       pcpu_count = ACCESS_ONCE(ref->pcpu_count);
-
-       if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
+       if (__pcpu_ref_alive(ref, &pcpu_count)) {
                this_cpu_inc(*pcpu_count);
                ret = true;
        }
@@ -193,9 +202,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
 
        rcu_read_lock_sched();
 
-       pcpu_count = ACCESS_ONCE(ref->pcpu_count);
-
-       if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
+       if (__pcpu_ref_alive(ref, &pcpu_count))
                this_cpu_dec(*pcpu_count);
        else if (unlikely(atomic_dec_and_test(&ref->count)))
                ref->release(ref);
@@ -203,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
        rcu_read_unlock_sched();
 }
 
+/**
+ * percpu_ref_is_zero - test whether a percpu refcount reached zero
+ * @ref: percpu_ref to test
+ *
+ * Returns %true if @ref reached zero.
+ */
+static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
+{
+       unsigned __percpu *pcpu_count;
+
+       if (__pcpu_ref_alive(ref, &pcpu_count))
+               return false;
+       return !atomic_read(&ref->count);
+}
+
 #endif