* - rseq_percpu_pool_set_malloc(),
* - rseq_percpu_pool_set_zmalloc().
*
+ * The @stride argument to __rseq_percpu_free() is a configurable
+ * stride, which must match the stride received by pool creation.
+ * rseq_percpu_free() uses the default RSEQ_PERCPU_STRIDE stride.
+ *
* This API is MT-safe.
*/
void __rseq_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride);
#define rseq_percpu_free(ptr) __rseq_percpu_free(ptr, RSEQ_PERCPU_STRIDE)
/*
- * rseq_percpu_ptr: Decode a per-cpu pointer.
+ * rseq_percpu_ptr: Offset a per-cpu pointer for a given CPU.
*
- * Decode a per-cpu pointer @ptr to get the associated pointer for the
- * given @cpu. The @ptr argument is a __rseq_percpu encoded pointer
- * returned by either:
+ * Offset a per-cpu pointer @ptr to get the associated pointer for the
+ * given @cpu. The @ptr argument is a __rseq_percpu pointer returned by
+ * either:
*
* - rseq_percpu_malloc(),
* - rseq_percpu_zmalloc(),
* - rseq_percpu_pool_set_malloc(),
* - rseq_percpu_pool_set_zmalloc().
*
- * The __rseq_percpu pointer can be decoded with rseq_percpu_ptr() even
- * after it has been freed, as long as its associated pool has not been
- * destroyed. However, memory pointed to by the decoded pointer should
- * not be accessed after the __rseq_percpu pointer has been freed.
+ * The macros rseq_percpu_ptr() and __rseq_percpu_ptr() preserve the
+ * type of the @ptr parameter for the returned pointer, but removes the
+ * __rseq_percpu annotation.
*
- * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
- * for the returned pointer, but removes the __rseq_percpu annotation.
+ * The macro __rseq_percpu_ptr() takes a configurable @stride argument,
+ * whereas rseq_percpu_ptr() uses the RSEQ_PERCPU_STRIDE default stride.
+ * This must match the stride used for pool creation.
*
* This API is MT-safe.
*/
-void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu, size_t percpu_stride);
-#define rseq_percpu_ptr(ptr, cpu) \
- ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu, RSEQ_PERCPU_STRIDE))
+#define __rseq_percpu_ptr(ptr, cpu, stride) \
+ ((__typeof__(*(ptr)) *) ((uintptr_t) (ptr) + ((unsigned int) (cpu) * (uintptr_t) (stride))))
+
+#define rseq_percpu_ptr(ptr, cpu) __rseq_percpu_ptr(ptr, cpu, RSEQ_PERCPU_STRIDE)
/*
* rseq_percpu_pool_set_create: Create a pool set.
#define MOVE_PAGES_BATCH_SIZE 4096
+#define RANGE_HEADER_OFFSET sizeof(struct rseq_percpu_pool_range)
+
struct free_list_node;
struct free_list_node {
struct rseq_percpu_pool_range {
struct rseq_percpu_pool_range *next;
struct rseq_percpu_pool *pool; /* Backward ref. to container pool. */
+ void *header;
void *base;
size_t next_unused;
/* Track alloc/free. */
return pool->ranges->base + (stride * cpu) + item_offset;
}
-void *__rseq_percpu_ptr(void __rseq_percpu *_ptr, int cpu, size_t stride)
-{
- uintptr_t ptr = (uintptr_t) _ptr;
- uintptr_t item_offset = ptr & MAX_POOL_LEN_MASK;
- uintptr_t pool_index = ptr >> POOL_INDEX_SHIFT;
- struct rseq_percpu_pool *pool = &rseq_percpu_pool[pool_index];
-
- assert(cpu >= 0);
- return __rseq_pool_percpu_ptr(pool, cpu, item_offset, stride);
-}
-
static
void rseq_percpu_zero_item(struct rseq_percpu_pool *pool, uintptr_t item_offset)
{
{
destroy_alloc_bitmap(pool, range);
/* range is a header located one page before the aligned mapping. */
- return pool->attr.munmap_func(pool->attr.mmap_priv, range,
+ return pool->attr.munmap_func(pool->attr.mmap_priv, range->header,
(pool->percpu_stride * pool->max_nr_cpus) + rseq_get_page_len());
}
{
struct rseq_percpu_pool_range *range;
unsigned long page_size;
+ void *header;
void *base;
page_size = rseq_get_page_len();
base = aligned_mmap_anonymous(pool, page_size,
pool->percpu_stride * pool->max_nr_cpus,
pool->percpu_stride,
- (void **) &range, page_size);
+ &header, page_size);
if (!base)
return NULL;
+ range = (struct rseq_percpu_pool_range *) (base - RANGE_HEADER_OFFSET);
range->pool = pool;
range->base = base;
+ range->header = header;
if (pool->attr.robust_set) {
if (create_alloc_bitmap(pool, range))
goto error_alloc;
/* Remove node from free list (update head). */
pool->free_list_head = node->next;
item_offset = (uintptr_t) ((void *) node - pool->ranges->base);
- addr = (void *) (((uintptr_t) pool->index << POOL_INDEX_SHIFT) | item_offset);
+ addr = (void __rseq_percpu *) (pool->ranges->base + item_offset);
goto end;
}
if (pool->ranges->next_unused + pool->item_len > pool->percpu_stride) {
goto end;
}
item_offset = pool->ranges->next_unused;
- addr = (void *) (((uintptr_t) pool->index << POOL_INDEX_SHIFT) | item_offset);
+ addr = (void __rseq_percpu *) (pool->ranges->base + item_offset);
pool->ranges->next_unused += pool->item_len;
end:
if (addr)
void __rseq_percpu_free(void __rseq_percpu *_ptr, size_t percpu_stride)
{
uintptr_t ptr = (uintptr_t) _ptr;
- uintptr_t item_offset = ptr & MAX_POOL_LEN_MASK;
- uintptr_t pool_index = ptr >> POOL_INDEX_SHIFT;
- struct rseq_percpu_pool *pool = &rseq_percpu_pool[pool_index];
+ void *range_base = (void *) (ptr & (~(percpu_stride - 1)));
+ struct rseq_percpu_pool_range *range = (struct rseq_percpu_pool_range *) (range_base - RANGE_HEADER_OFFSET);
+ struct rseq_percpu_pool *pool = range->pool;
+ uintptr_t item_offset = ptr & (percpu_stride - 1);
struct free_list_node *head, *item;
pthread_mutex_lock(&pool->lock);
/* Add ptr to head of free list */
head = pool->free_list_head;
/* Free-list is in CPU 0 range. */
- item = (struct free_list_node *)__rseq_pool_percpu_ptr(pool, 0, item_offset, percpu_stride);
+ item = (struct free_list_node *) ptr;
item->next = head;
pool->free_list_head = item;
pthread_mutex_unlock(&pool->lock);
projects / librseq.git / commitdiff
