*/
#define __rseq_percpu
+struct rseq_mmap_attr;
struct rseq_percpu_pool;
/*
* next power of two). The reserved allocation size is @percpu_len, and
* the maximum CPU value expected is (@max_nr_cpus - 1).
*
- * Arguments @mmap_prot, @mmap_flags, @mmap_fd, @mmap_offset are passed
- * as arguments to mmap(2) when allocating the memory area holding the
- * percpu pool.
- *
- * Argument @numa_flags are passed to move_pages(2). The expected flags
- * are:
- * 0: do not move pages to specific numa nodes
- * (use for e.g. mm_cid indexing).
- * MPOL_MF_MOVE: move process-private pages to cpu-specific numa nodes.
- * MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
- * (requires CAP_SYS_NICE).
+ * The @mmap_attr pointer used to specify the memory allocator callbacks
+ * to use to manage the memory for the pool. If NULL, use a default
+ * internal implementation. The @mmap_attr can be destroyed immediately
+ * after rseq_percpu_pool_create() returns. The caller keeps ownership
+ * of @mmap_attr.
*
* Returns a pointer to the created percpu pool. Return NULL on error,
* with errno set accordingly:
* ENOMEM: Not enough resources (memory or pool indexes) available to
* allocate pool.
*
- * In addition, if mmap(2) fails, NULL is returned and errno is
- * propagated from mmap(2).
+ * In addition, if the mmap_attr mmap callback fails, NULL is returned
+ * and errno is propagated from the callback. The default callback can
+ * return errno=ENOMEM.
*
* This API is MT-safe.
*/
struct rseq_percpu_pool *rseq_percpu_pool_create(size_t item_len,
size_t percpu_len, int max_nr_cpus,
- int mmap_prot, int mmap_flags, int mmap_fd, off_t mmap_offset,
- int numa_flags);
+ const struct rseq_mmap_attr *mmap_attr);
/*
* rseq_percpu_pool_destroy: Destroy a per-cpu memory pool.
* Return values: 0 on success, -1 on error, with errno set accordingly:
* ENOENT: Trying to free a pool which was not allocated.
*
- * If munmap(2) fails, -1 is returned and errno is propagated from
- * munmap(2).
+ * If the munmap_func callback fails, -1 is returned and errno is
+ * propagated from the callback. The default callback can return
+ * errno=EINVAL.
*
* This API is MT-safe.
*/
*/
void __rseq_percpu *rseq_percpu_pool_set_zmalloc(struct rseq_percpu_pool_set *pool_set, size_t len);
+/*
+ * rseq_percpu_pool_init_numa: Move pages to the NUMA node associated to their CPU topology.
+ *
+ * For pages allocated within @pool, invoke move_pages(2) with the given
+ * @numa_flags to move the pages to the NUMA node associated to their
+ * CPU topology.
+ *
+ * Argument @numa_flags are passed to move_pages(2). The expected flags are:
+ * MPOL_MF_MOVE: move process-private pages to cpu-specific numa nodes.
+ * MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
+ * (requires CAP_SYS_NICE).
+ *
+ * Returns 0 on success, else return -1 with errno set by move_pages(2).
+ */
+int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags);
+
+/*
+ * rseq_mmap_attr_create: Create a mmap attribute structure.
+ *
+ * The @mmap_func callback used to map the memory for the pool.
+ *
+ * The @munmap_func callback used to unmap the memory when the pool
+ * is destroyed.
+ *
+ * The @mmap_priv argument is a private data pointer passed to both
+ * @mmap_func and @munmap_func callbacks.
+ */
+struct rseq_mmap_attr *rseq_mmap_attr_create(void *(*mmap_func)(void *priv, size_t len),
+ int (*munmap_func)(void *priv, void *ptr, size_t len),
+ void *mmap_priv);
+
+/*
+ * rseq_mmap_attr_destroy: Destroy a mmap attribute structure.
+ */
+void rseq_mmap_attr_destroy(struct rseq_mmap_attr *attr);
+
#ifdef __cplusplus
}
#endif
/* This lock protects pool create/destroy. */
static pthread_mutex_t pool_lock = PTHREAD_MUTEX_INITIALIZER;
+struct rseq_mmap_attr {
+ void *(*mmap_func)(void *priv, size_t len);
+ int (*munmap_func)(void *priv, void *ptr, size_t len);
+ void *mmap_priv;
+};
+
struct rseq_percpu_pool {
void *base;
unsigned int index;
size_t next_unused;
/* This lock protects allocation/free within the pool. */
pthread_mutex_t lock;
+
+ struct rseq_mmap_attr mmap_attr;
};
//TODO: the array of pools should grow dynamically on create.
}
#ifdef HAVE_LIBNUMA
-static
-void rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags)
+int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags)
{
unsigned long nr_pages, page;
long ret, page_len;
int cpu;
if (!numa_flags)
- return;
+ return 0;
page_len = rseq_get_page_len();
nr_pages = pool->percpu_len >> rseq_get_count_order_ulong(page_len);
for (cpu = 0; cpu < pool->max_nr_cpus; cpu++) {
int status = -EPERM;
ret = move_pages(0, 1, &pageptr, &node, &status, numa_flags);
- if (ret) {
- perror("move_pages");
- abort();
- }
+ if (ret)
+ return ret;
}
}
+ return 0;
}
#else
-static
void rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool __attribute__((unused)),
int numa_flags __attribute__((unused)))
{
+ return 0;
}
#endif
-/*
- * Expected numa_flags:
- * 0: do not move pages to specific numa nodes (use for e.g. mm_cid indexing).
- * MPOL_MF_MOVE: move process-private pages to cpu-specific numa nodes.
- * MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes (requires CAP_SYS_NICE).
- */
+static
+void *default_mmap_func(void *priv __attribute__((unused)), size_t len)
+{
+ void *base;
+
+ base = mmap(NULL, len, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ if (base == MAP_FAILED)
+ return NULL;
+ return base;
+}
+
+static
+int default_munmap_func(void *priv __attribute__((unused)), void *ptr, size_t len)
+{
+ return munmap(ptr, len);
+}
+
struct rseq_percpu_pool *rseq_percpu_pool_create(size_t item_len,
size_t percpu_len, int max_nr_cpus,
- int mmap_prot, int mmap_flags, int mmap_fd,
- off_t mmap_offset, int numa_flags)
+ const struct rseq_mmap_attr *mmap_attr)
{
+ void *(*mmap_func)(void *priv, size_t len);
+ int (*munmap_func)(void *priv, void *ptr, size_t len);
+ void *mmap_priv;
struct rseq_percpu_pool *pool;
void *base;
unsigned int i;
return NULL;
}
+ if (mmap_attr) {
+ mmap_func = mmap_attr->mmap_func;
+ munmap_func = mmap_attr->munmap_func;
+ mmap_priv = mmap_attr->mmap_priv;
+ } else {
+ mmap_func = default_mmap_func;
+ munmap_func = default_munmap_func;
+ mmap_priv = NULL;
+ }
pthread_mutex_lock(&pool_lock);
/* Linear scan in array of pools to find empty spot. */
for (i = FIRST_POOL; i < MAX_NR_POOLS; i++) {
goto end;
found_empty:
- base = mmap(NULL, percpu_len * max_nr_cpus, mmap_prot,
- mmap_flags, mmap_fd, mmap_offset);
- if (base == MAP_FAILED) {
+ base = mmap_func(mmap_priv, percpu_len * max_nr_cpus);
+ if (!base) {
pool = NULL;
goto end;
}
- rseq_percpu_pool_init_numa(pool, numa_flags);
pthread_mutex_init(&pool->lock, NULL);
pool->base = base;
pool->percpu_len = percpu_len;
pool->index = i;
pool->item_len = item_len;
pool->item_order = order;
+ pool->mmap_attr.mmap_func = mmap_func;
+ pool->mmap_attr.munmap_func = munmap_func;
+ pool->mmap_attr.mmap_priv = mmap_priv;
end:
pthread_mutex_unlock(&pool_lock);
return pool;
ret = -1;
goto end;
}
- ret = munmap(pool->base, pool->percpu_len * pool->max_nr_cpus);
+ ret = pool->mmap_attr.munmap_func(pool->mmap_attr.mmap_priv, pool->base,
+ pool->percpu_len * pool->max_nr_cpus);
if (ret)
goto end;
pthread_mutex_destroy(&pool->lock);
{
return __rseq_percpu_pool_set_malloc(pool_set, len, true);
}
+
+struct rseq_mmap_attr *rseq_mmap_attr_create(void *(*mmap_func)(void *priv, size_t len),
+ int (*munmap_func)(void *priv, void *ptr, size_t len),
+ void *mmap_priv)
+{
+ struct rseq_mmap_attr *attr = calloc(1, sizeof(struct rseq_mmap_attr));
+
+ if (!attr)
+ return NULL;
+ attr->mmap_func = mmap_func;
+ attr->munmap_func = munmap_func;
+ attr->mmap_priv = mmap_priv;
+ return attr;
+}
+
+void rseq_mmap_attr_destroy(struct rseq_mmap_attr *attr)
+{
+ free(attr);
+}
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create(sizeof(struct spinlock_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create(sizeof(struct inc_test_data),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create(sizeof(struct percpu_list),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create(sizeof(struct percpu_buffer),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
struct rseq_percpu_pool *mempool;
mempool = rseq_percpu_pool_create(sizeof(struct percpu_memcpy_buffer),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
int ret;
mempool = rseq_percpu_pool_create(sizeof(struct percpu_list),
- PERCPU_POOL_LEN, CPU_SETSIZE, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0, 0);
+ PERCPU_POOL_LEN, CPU_SETSIZE, NULL);
if (!mempool) {
perror("rseq_percpu_pool_create");
abort();
projects / librseq.git / commitdiff
