return (void *) rseq_mempool_set_percpu_malloc_init(pool_set, init_ptr, len);
}
-
/*
* rseq_mempool_init_numa: Move pages to the NUMA node associated to their CPU topology.
*
* There is a marginal runtime overhead on malloc/free operations.
*
* The memory overhead is (pool->percpu_len / pool->item_len) / CHAR_BIT
- * bytes, over the lifetime of the pool.
+ * bytes, plus one additional stride range for a separate free list,
+ * over the lifetime of the pool.
*
* Returns 0 on success, -1 with errno=EINVAL if arguments are invalid.
*/
enum rseq_mempool_populate_policy {
/*
- * RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE (default):
- * Do not populate pages for any of the CPUs when creating the
- * mempool. Rely on copy-on-write (COW) of per-cpu pages to
- * populate per-cpu pages from the initial values pages on
- * first write. This mempool is only meant for single-process
- * use (private mapping). Note that this type of pool cannot
- * be accessed from children processes across fork. It is
- * however valid to destroy a pool from a child process after
- * a fork to free its remaining resources.
+ * RSEQ_MEMPOOL_POPULATE_COW_INIT (default):
+ * Rely on copy-on-write (COW) of per-cpu pages to populate
+ * per-cpu pages from the initial values pages on first write.
+ * Note that this type of pool cannot be accessed from
+ * children processes across fork. It is however valid to
+ * destroy a pool from a child process after a fork to free
+ * its remaining resources.
*/
- RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE = 0,
+ RSEQ_MEMPOOL_POPULATE_COW_INIT = 0,
+
/*
- * RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL:
- * Populate pages for all CPUs from 0 to (max_nr_cpus - 1)
- * when creating the mempool. This mempool is only meant for
- * single-process use (private mapping).
+ * RSEQ_MEMPOOL_POPULATE_COW_ZERO:
+ * Rely on copy-on-write (COW) of per-cpu pages to populate
+ * per-cpu pages from the zero page on first write. As long
+ * as the user only uses malloc, zmalloc, or malloc_init with
+ * zeroed content to allocate items, it does not trigger COW
+ * of per-cpu pages, leaving in place the zero page until an
+ * active CPU writes to its per-cpu item. The recommended (and
+ * default) poison value for this pool policy is 0 to prevent
+ * useless COW page allocation.
*/
- RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL = 1,
+ RSEQ_MEMPOOL_POPULATE_COW_ZERO = 1,
};
/*
# define DEFAULT_COW_INIT_POISON_VALUE 0x55555555UL
#endif
+/*
+ * Define the default COW_ZERO poison value as zero to prevent useless
+ * COW page allocation when writing poison values when freeing items.
+ */
#define DEFAULT_COW_ZERO_POISON_VALUE 0x0
struct free_list_node;
/*
* Memory layout of a mempool range:
- * - Canary header page (for destroy-after-fork detection),
+ * - Canary header page (for detection of destroy-after-fork of
+ * COW_INIT pool),
* - Header page (contains struct rseq_mempool_range at the
* very end),
* - Base of the per-cpu data, starting with CPU 0.
- * Aliases with free-list for non-robust populate all pool.
+ * Aliases with free-list for non-robust COW_ZERO pool.
* - CPU 1,
* ...
* - CPU max_nr_cpus - 1
- * - init values (unpopulated for RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL).
- * Aliases with free-list for non-robust populate none pool.
+ * - init values (only allocated for COW_INIT pool).
+ * Aliases with free-list for non-robust COW_INIT pool.
* - free list (for robust pool).
*
* The free list aliases the CPU 0 memory area for non-robust
- * populate all pools. It aliases with init values for
- * non-robust populate none pools. It is located immediately
- * after the init values for robust pools.
+ * COW_ZERO pools. It aliases with init values for non-robust
+ * COW_INIT pools. It is located immediately after the init
+ * values for robust pools.
*/
void *header;
void *base;
/*
* The init values contains malloc_init/zmalloc values.
- * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL.
+ * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_COW_ZERO.
*/
void *init;
size_t next_unused;
int item_order;
/*
- * The free list chains freed items on the CPU 0 address range.
- * We should rethink this decision if false sharing between
- * malloc/free from other CPUs and data accesses from CPU 0
- * becomes an issue. This is a NULL-terminated singly-linked
- * list.
+ * COW_INIT non-robust pools:
+ * The free list chains freed items on the init
+ * values address range.
+ *
+ * COW_ZERO non-robust pools:
+ * The free list chains freed items on the CPU 0
+ * address range. We should rethink this
+ * decision if false sharing between malloc/free
+ * from other CPUs and data accesses from CPU 0
+ * becomes an issue.
+ *
+ * Robust pools: The free list chains freed items in the
+ * address range dedicated for the free list.
+ *
+ * This is a NULL-terminated singly-linked list.
*/
struct free_list_node *free_list_head;
/* Skip cpus. */
p -= pool->attr.max_nr_cpus * pool->attr.stride;
/* Skip init values */
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
p -= pool->attr.stride;
} else {
- /* Populate none free list is in init values */
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ /* COW_INIT free list is in init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
p -= pool->attr.max_nr_cpus * pool->attr.stride;
}
return p;
/* Skip cpus. */
p += pool->attr.max_nr_cpus * pool->attr.stride;
/* Skip init values */
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
p += pool->attr.stride;
} else {
- /* Populate none free list is in init values */
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ /* COW_INIT free list is in init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
p += pool->attr.max_nr_cpus * pool->attr.stride;
}
return (struct free_list_node *) p;
* write to the page. This eliminates useless COW over
* the zero page just for overwriting it with zeroes.
*
- * This means zmalloc() in populate all policy pool do
+ * This means zmalloc() in COW_ZERO policy pool do
* not trigger COW for CPUs which are not actively
* writing to the pool. This is however not the case for
* malloc_init() in populate-all pools if it populates
* COW of the page.
*
* It is recommended to use zero as poison value for
- * populate-all pools to eliminate COW due to writing
- * poison to unused CPU memory.
+ * COW_ZERO pools to eliminate COW due to writing
+ * poison to CPU memory still backed by the zero page.
*/
if (rseq_cmp_item(p, pool->item_len, poison, NULL) == 0)
continue;
header_len = POOL_HEADER_NR_PAGES * page_size;
range_len = pool->attr.stride * pool->attr.max_nr_cpus;
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
range_len += pool->attr.stride; /* init values */
if (pool->attr.robust_set)
- range_len += pool->attr.stride; /* free list */
+ range_len += pool->attr.stride; /* dedicated free list */
base = aligned_mmap_anonymous(page_size, range_len,
pool->attr.stride, &header, header_len);
if (!base)
range->mmap_addr = header;
range->mmap_len = header_len + range_len;
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL) {
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT) {
range->init = base + (pool->attr.stride * pool->attr.max_nr_cpus);
/* Populate init values pages from memfd */
memfd = rseq_memfd_create_init(pool->name, pool->attr.stride);
size_t page_size;
char *addr;
- if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_COW_INIT)
return true;
range = pool->range_list;
if (!range)
/*
* Validate that the pool mappings are accessible before doing
* free list/poison validation and unmapping ranges. This allows
- * calling pool destroy in child process after a fork for
- * populate-none pools to free pool resources.
+ * calling pool destroy in child process after a fork for COW_INIT
+ * pools to free pool resources.
*/
mapping_accessible = pool_mappings_accessible(pool);
if (_attr)
memcpy(&attr, _attr, sizeof(attr));
+ /*
+ * Validate that the pool populate policy requested is known.
+ */
+ switch (attr.populate_policy) {
+ case RSEQ_MEMPOOL_POPULATE_COW_INIT:
+ break;
+ case RSEQ_MEMPOOL_POPULATE_COW_ZERO:
+ break;
+ default:
+ errno = EINVAL;
+ return NULL;
+ }
+
switch (attr.type) {
case MEMPOOL_TYPE_PERCPU:
if (attr.max_nr_cpus < 0) {
break;
case MEMPOOL_TYPE_GLOBAL:
/* Override populate policy for global type. */
- if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE)
- attr.populate_policy = RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL;
+ if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ attr.populate_policy = RSEQ_MEMPOOL_POPULATE_COW_ZERO;
/* Use a 1-cpu pool for global mempool type. */
attr.max_nr_cpus = 1;
break;
attr.stride = RSEQ_MEMPOOL_STRIDE; /* Use default */
if (attr.robust_set && !attr.poison_set) {
attr.poison_set = true;
- if (attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
attr.poison = DEFAULT_COW_INIT_POISON_VALUE;
else
attr.poison = DEFAULT_COW_ZERO_POISON_VALUE;
item = __rseq_percpu_to_free_list_ptr(pool, _ptr);
/*
* Setting the next pointer will overwrite the first uintptr_t
- * poison for either CPU 0 (populate all) or init data (populate
- * none).
+ * poison for either CPU 0 (COW_ZERO, non-robust), or init data
+ * (COW_INIT, non-robust).
*/
item->next = head;
pool->free_list_head = item;
attr->init_set = true;
attr->init_func = init_func;
attr->init_priv = init_priv;
+ attr->populate_policy = RSEQ_MEMPOOL_POPULATE_COW_INIT;
return 0;
}
ok(ret == 0, "Setting mempool poison");
ret = rseq_mempool_attr_set_populate_policy(attr, policy);
ok(ret == 0, "Setting mempool populate policy to %s",
- policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE ? "NONE" : "ALL");
+ policy == RSEQ_MEMPOOL_POPULATE_COW_INIT ? "COW_INIT" : "COW_ZERO");
mempool = rseq_mempool_create("test_data",
sizeof(struct test_data), attr);
ok(mempool, "Create mempool of size %zu", stride);
ptr = (struct test_data __rseq_percpu *) rseq_mempool_percpu_malloc(pool);
/*
* Corrupt free list: For robust pools, the free list is located
- * after the last cpu memory range for populate all, and after
- * the init values memory range for populate none.
+ * after the last cpu memory range for COW_ZERO, and after the init
+ * values memory range for COW_INIT.
*/
- if (policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (policy == RSEQ_MEMPOOL_POPULATE_COW_ZERO)
cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool));
else
cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool) + 1);
ptr = (struct test_data __rseq_percpu *) rseq_mempool_percpu_malloc(pool);
/*
* Corrupt free list: For robust pools, the free list is located
- * after the last cpu memory range for populate all, and after
- * the init values memory range for populate none.
+ * after the last cpu memory range for COW_ZERO, and after the init
+ * values memory range for COW_INIT.
*/
- if (policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
+ if (policy == RSEQ_MEMPOOL_POPULATE_COW_ZERO)
cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool));
else
cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool) + 1);
for (nr_ranges = 1; nr_ranges < 32; nr_ranges <<= 1) {
/* From page size to 64kB */
for (len = rseq_get_page_len(); len < 65536; len <<= 1) {
- test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL, nr_ranges, len);
- test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE, nr_ranges, len);
+ test_mempool_fill(RSEQ_MEMPOOL_POPULATE_COW_ZERO, nr_ranges, len);
+ test_mempool_fill(RSEQ_MEMPOOL_POPULATE_COW_INIT, nr_ranges, len);
}
}
len = 65536;
/* From min(page size, 64kB) to 4MB */
for (; len < 4096 * 1024; len <<= 1) {
- test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL, 1, len);
- test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE, 1, len);
+ test_mempool_fill(RSEQ_MEMPOOL_POPULATE_COW_ZERO, 1, len);
+ test_mempool_fill(RSEQ_MEMPOOL_POPULATE_COW_INIT, 1, len);
}
- run_robust_tests(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL);
- run_robust_tests(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE);
- ok(run_fork_destroy_pool_test(fork_child, RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL),
- "fork destroy pool test populate private all");
- ok(run_fork_destroy_pool_test(fork_child, RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE),
- "fork destroy pool test populate private none");
+ run_robust_tests(RSEQ_MEMPOOL_POPULATE_COW_ZERO);
+ run_robust_tests(RSEQ_MEMPOOL_POPULATE_COW_INIT);
+ ok(run_fork_destroy_pool_test(fork_child, RSEQ_MEMPOOL_POPULATE_COW_ZERO),
+ "fork destroy pool test populate COW_ZERO");
+ ok(run_fork_destroy_pool_test(fork_child, RSEQ_MEMPOOL_POPULATE_COW_INIT),
+ "fork destroy pool test populate COW_INIT");
exit(exit_status());
}
projects / librseq.git / commitdiff
