#ifndef _RSEQ_MEMPOOL_H
#define _RSEQ_MEMPOOL_H
+#include <rseq/compiler.h>
#include <stddef.h>
#include <sys/types.h>
#include <sys/mman.h>
extern "C" {
#endif
+#if RSEQ_BITS_PER_LONG == 64
+# define RSEQ_PERCPU_STRIDE (1U << 24) /* 64-bit stride: 16MB */
+#else
+# define RSEQ_PERCPU_STRIDE (1U << 16) /* 32-bit stride: 64kB */
+#endif
+
/*
* Tag pointers returned by:
* - rseq_percpu_malloc(),
* This API is MT-safe.
*/
struct rseq_percpu_pool *rseq_percpu_pool_create(const char *pool_name,
- size_t item_len, size_t percpu_len, int max_nr_cpus,
+ size_t item_len, size_t percpu_stride, int max_nr_cpus,
const struct rseq_pool_attr *attr);
/*
*
* This API is MT-safe.
*/
-void rseq_percpu_free(void __rseq_percpu *ptr);
+void __rseq_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride);
+
+#define rseq_percpu_free(ptr) __rseq_percpu_free(ptr, RSEQ_PERCPU_STRIDE)
/*
* rseq_percpu_ptr: Decode a per-cpu pointer.
*
* This API is MT-safe.
*/
-void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu);
-#define rseq_percpu_ptr(ptr, cpu) ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu))
+void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu, size_t percpu_stride);
+#define rseq_percpu_ptr(ptr, cpu) \
+ ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu, RSEQ_PERCPU_STRIDE))
/*
* rseq_percpu_pool_set_create: Create a pool set.
unsigned int index;
size_t item_len;
- size_t percpu_len;
+ size_t percpu_stride;
int item_order;
int max_nr_cpus;
};
static
-void *__rseq_pool_percpu_ptr(struct rseq_percpu_pool *pool, int cpu, uintptr_t item_offset)
+void *__rseq_pool_percpu_ptr(struct rseq_percpu_pool *pool, int cpu,
+ uintptr_t item_offset, size_t stride)
{
/* TODO: Implement multi-ranges support. */
- return pool->ranges->base + (pool->percpu_len * cpu) + item_offset;
+ return pool->ranges->base + (stride * cpu) + item_offset;
}
-void *__rseq_percpu_ptr(void __rseq_percpu *_ptr, int cpu)
+void *__rseq_percpu_ptr(void __rseq_percpu *_ptr, int cpu, size_t stride)
{
uintptr_t ptr = (uintptr_t) _ptr;
uintptr_t item_offset = ptr & MAX_POOL_LEN_MASK;
struct rseq_percpu_pool *pool = &rseq_percpu_pool[pool_index];
assert(cpu >= 0);
- return __rseq_pool_percpu_ptr(pool, cpu, item_offset);
+ return __rseq_pool_percpu_ptr(pool, cpu, item_offset, stride);
}
static
int i;
for (i = 0; i < pool->max_nr_cpus; i++) {
- char *p = __rseq_pool_percpu_ptr(pool, i, item_offset);
+ char *p = __rseq_pool_percpu_ptr(pool, i,
+ item_offset, pool->percpu_stride);
memset(p, 0, pool->item_len);
}
}
static
int rseq_percpu_pool_range_init_numa(struct rseq_percpu_pool *pool, struct rseq_percpu_pool_range *range, int numa_flags)
{
- unsigned long nr_pages;
- long ret, page_len;
+ unsigned long nr_pages, page_len;
+ long ret;
int cpu;
if (!numa_flags)
return 0;
page_len = rseq_get_page_len();
- nr_pages = pool->percpu_len >> rseq_get_count_order_ulong(page_len);
+ nr_pages = pool->percpu_stride >> rseq_get_count_order_ulong(page_len);
for (cpu = 0; cpu < pool->max_nr_cpus; cpu++) {
int status[MOVE_PAGES_BATCH_SIZE];
{
size_t count;
- count = ((pool->percpu_len >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
+ count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
/*
* Not being able to create the validation bitmap is an error
return;
for (range = pool->ranges; range; range = range->next) {
- total_item += pool->percpu_len >> pool->item_order;
- total_never_allocated += (pool->percpu_len - range->next_unused) >> pool->item_order;
+ total_item += pool->percpu_stride >> pool->item_order;
+ total_never_allocated += (pool->percpu_stride - range->next_unused) >> pool->item_order;
}
max_list_traversal = total_item - total_never_allocated;
if (!bitmap)
return;
- count = ((pool->percpu_len >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
+ count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
/* Assert that all items in the pool were freed. */
for (size_t k = 0; k < count; ++k)
{
destroy_alloc_bitmap(pool, range);
return pool->attr.munmap_func(pool->attr.mmap_priv, range->base,
- pool->percpu_len * pool->max_nr_cpus);
+ pool->percpu_stride * pool->max_nr_cpus);
}
static
return NULL;
range->pool = pool;
- base = pool->attr.mmap_func(pool->attr.mmap_priv, pool->percpu_len * pool->max_nr_cpus);
+ base = pool->attr.mmap_func(pool->attr.mmap_priv, pool->percpu_stride * pool->max_nr_cpus);
if (!base)
goto error_alloc;
range->base = base;
}
struct rseq_percpu_pool *rseq_percpu_pool_create(const char *pool_name,
- size_t item_len, size_t percpu_len, int max_nr_cpus,
+ size_t item_len, size_t percpu_stride, int max_nr_cpus,
const struct rseq_pool_attr *_attr)
{
struct rseq_percpu_pool *pool;
}
item_len = 1UL << order;
- /* Align percpu_len on page size. */
- percpu_len = rseq_align(percpu_len, rseq_get_page_len());
+ if (!percpu_stride)
+ percpu_stride = RSEQ_PERCPU_STRIDE; /* Use default */
- if (max_nr_cpus < 0 || item_len > percpu_len ||
- percpu_len > (UINTPTR_MAX >> POOL_INDEX_BITS)) {
+ if (max_nr_cpus < 0 || item_len > percpu_stride ||
+ percpu_stride > (UINTPTR_MAX >> POOL_INDEX_BITS) ||
+ percpu_stride < (size_t) rseq_get_page_len() ||
+ !is_pow2(percpu_stride)) {
errno = EINVAL;
return NULL;
}
found_empty:
memcpy(&pool->attr, &attr, sizeof(attr));
pthread_mutex_init(&pool->lock, NULL);
- pool->percpu_len = percpu_len;
+ pool->percpu_stride = percpu_stride;
pool->max_nr_cpus = max_nr_cpus;
pool->index = i;
pool->item_len = item_len;
addr = (void *) (((uintptr_t) pool->index << POOL_INDEX_SHIFT) | item_offset);
goto end;
}
- if (pool->ranges->next_unused + pool->item_len > pool->percpu_len) {
+ if (pool->ranges->next_unused + pool->item_len > pool->percpu_stride) {
errno = ENOMEM;
addr = NULL;
goto end;
bitmap[k] &= ~mask;
}
-void rseq_percpu_free(void __rseq_percpu *_ptr)
+void __rseq_percpu_free(void __rseq_percpu *_ptr, size_t percpu_stride)
{
uintptr_t ptr = (uintptr_t) _ptr;
uintptr_t item_offset = ptr & MAX_POOL_LEN_MASK;
/* Add ptr to head of free list */
head = pool->free_list_head;
/* Free-list is in CPU 0 range. */
- item = (struct free_list_node *)__rseq_pool_percpu_ptr(pool, 0, item_offset);
+ item = (struct free_list_node *)__rseq_pool_percpu_ptr(pool, 0, item_offset, percpu_stride);
item->next = head;
pool->free_list_head = item;
pthread_mutex_unlock(&pool->lock);
#define RSEQ_DEFAULT_PAGE_SIZE 4096
static inline
-long rseq_get_page_len(void)
+unsigned long rseq_get_page_len(void)
{
long page_len = sysconf(_SC_PAGE_SIZE);
if (page_len < 0)
page_len = RSEQ_DEFAULT_PAGE_SIZE;
- return page_len;
+ return (unsigned long) page_len;
}
static inline
return __builtin_popcountl(v);
}
+static inline
+bool is_pow2(uint64_t x)
+{
+ return !(x & (x - 1));
+}
+
+/*
+ * Calculate offset needed to align p on alignment towards higher
+ * addresses. Alignment must be a power of 2
+ */
+static inline
+off_t offset_align(uintptr_t p, size_t alignment)
+{
+ return (alignment - p) & (alignment - 1);
+}
+
#endif /* _RSEQ_UTILS_H */
struct list_head node;
};
-static void test_mempool_fill(size_t len)
+static void test_mempool_fill(size_t stride)
{
struct test_data __rseq_percpu *ptr;
struct test_data *iter, *tmp;
mempool = rseq_percpu_pool_create("test_data",
sizeof(struct test_data),
- len, CPU_SETSIZE, attr);
- ok(mempool, "Create mempool of size %zu", len);
+ stride, CPU_SETSIZE, attr);
+ ok(mempool, "Create mempool of size %zu", stride);
rseq_pool_attr_destroy(attr);
for (;;) {
if (!ptr)
break;
/* Link items in cpu 0. */
- cpuptr = rseq_percpu_ptr(ptr, 0);
+ cpuptr = (struct test_data *) __rseq_percpu_ptr(ptr, 0, stride);
cpuptr->backref = ptr;
/* Randomize items in list. */
if (count & 1)
count++;
}
- ok(count * sizeof(struct test_data) == len, "Allocated %" PRIu64 " objects in pool", count);
+ ok(count * sizeof(struct test_data) == stride, "Allocated %" PRIu64 " objects in pool", count);
list_for_each_entry(iter, &list, node) {
ptr = iter->backref;
for (i = 0; i < CPU_SETSIZE; i++) {
- struct test_data *cpuptr = rseq_percpu_ptr(ptr, i);
+ struct test_data *cpuptr = (struct test_data *) __rseq_percpu_ptr(ptr, i, stride);
if (cpuptr->value != 0)
abort();
list_for_each_entry_safe(iter, tmp, &list, node) {
ptr = iter->backref;
- rseq_percpu_free(ptr);
+ __rseq_percpu_free(ptr, stride);
}
ret = rseq_percpu_pool_destroy(mempool);
ok(ret == 0, "Destroy mempool");
#include <stdbool.h>
#include <rseq/mempool.h>
-#define PERCPU_POOL_LEN (1024*1024) /* 1MB */
-
#if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
enum {
MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ = (1 << 7),
mempool = rseq_percpu_pool_create("spinlock_test_data",
sizeof(struct spinlock_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
mempool = rseq_percpu_pool_create("inc_test_data",
sizeof(struct inc_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create("percpu_list", sizeof(struct percpu_list),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create("percpu_buffer", sizeof(struct percpu_buffer),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
mempool = rseq_percpu_pool_create("percpu_memcpy_buffer",
sizeof(struct percpu_memcpy_buffer),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
long long total_count = 0;
mempool = rseq_percpu_pool_create("percpu_list", sizeof(struct percpu_list),
- PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
+ 0, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
projects / librseq.git / commitdiff
