mempool: Detect poison corruption on alloc

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I1d4181f089aa08d1c14466751661bba8a3727cf5

diff --git a/include/rseq/mempool.h b/include/rseq/mempool.h
index 81884d4593b85536810a94e57f21ebdafdb63b19..3d8aac82b1efab4f019c9e1c8aeb455de2d39a69 100644 (file)
--- a/include/rseq/mempool.h
+++ b/include/rseq/mempool.h
@@ -430,9 +430,11 @@ int rseq_mempool_attr_set_init(struct rseq_mempool_attr *attr,
  *   - Check for double-free of pointers.
  *
  *   - Detect memory leaks on pool destruction.
-  *
+ *
  *   - Detect free-list corruption on pool destruction.
  *
+ *   - Detect poison value corruption on allocation and pool destruction.
+ *
  * There is a marginal runtime overhead on malloc/free operations.
  *
  * The memory overhead is (pool->percpu_len / pool->item_len) / CHAR_BIT

diff --git a/src/rseq-mempool.c b/src/rseq-mempool.c
index a68d1f887cb3c41e494eaee31c831d4b02ab21b6..583ba936d4f642a9d629bd98bb0288ad9f14b6ef 100644 (file)
--- a/src/rseq-mempool.c
+++ b/src/rseq-mempool.c
@@ -132,6 +132,12 @@ struct rseq_mempool_set {
        struct rseq_mempool *entries[POOL_SET_NR_ENTRIES];
 };
 
+static
+const char *get_pool_name(const struct rseq_mempool *pool)
+{
+       return pool->name ? : "<anonymous>";
+}
+
 static
 void *__rseq_pool_range_percpu_ptr(struct rseq_mempool_range *range, int cpu,
                uintptr_t item_offset, size_t stride)
@@ -165,7 +171,38 @@ void rseq_percpu_poison_item(struct rseq_mempool *pool,
                size_t offset;
 
                for (offset = 0; offset < pool->item_len; offset += sizeof(uintptr_t))
-                       *((uintptr_t *) p) = poison;
+                       *((uintptr_t *) (p + offset)) = poison;
+       }
+}
+
+/* Always inline for __builtin_return_address(0). */
+static inline __attribute__((always_inline))
+void rseq_percpu_check_poison_item(struct rseq_mempool *pool,
+               struct rseq_mempool_range *range, uintptr_t item_offset)
+{
+       uintptr_t poison = pool->attr.poison;
+       int i;
+
+       if (!pool->attr.robust_set || !pool->attr.poison_set)
+               return;
+       for (i = 0; i < pool->attr.max_nr_cpus; i++) {
+               char *p = __rseq_pool_range_percpu_ptr(range, i,
+                               item_offset, pool->attr.stride);
+               size_t offset;
+
+               for (offset = 0; offset < pool->item_len; offset += sizeof(uintptr_t)) {
+                       uintptr_t v;
+
+                       /* Skip poison check for free-list pointer. */
+                       if (i == 0 && offset == 0)
+                               continue;
+                       v = *((uintptr_t *) (p + offset));
+                       if (v != poison) {
+                               fprintf(stderr, "%s: Poison corruption detected (0x%lx) for pool: \"%s\" (%p), item offset: %zu, caller: %p.\n",
+                                       __func__, (unsigned long) v, get_pool_name(pool), pool, item_offset, (void *) __builtin_return_address(0));
+                               abort();
+                       }
+               }
        }
 }
 
@@ -270,12 +307,6 @@ int create_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range *ra
        return 0;
 }
 
-static
-const char *get_pool_name(const struct rseq_mempool *pool)
-{
-       return pool->name ? : "<anonymous>";
-}
-
 static
 bool addr_in_pool(const struct rseq_mempool *pool, void *addr)
 {
@@ -664,6 +695,7 @@ void __rseq_percpu *__rseq_percpu_malloc(struct rseq_mempool *pool, bool zeroed)
                /* Remove node from free list (update head). */
                pool->free_list_head = node->next;
                item_offset = (uintptr_t) ((void *) node - range_base);
+               rseq_percpu_check_poison_item(pool, range, item_offset);
                addr = (void __rseq_percpu *) node;
                goto end;
        }