mempool: Introduce "private" populate policy

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I3a4907ee65a58e7dbb3d03bf5096175620892915

diff --git a/include/rseq/mempool.h b/include/rseq/mempool.h
index 9c10066dfb6c2b72476071c6ce3d0c3fb86778ae..345d39153960d01c3ed70970e28f29a22c59f881 100644 (file)
--- a/include/rseq/mempool.h
+++ b/include/rseq/mempool.h
@@ -539,19 +539,21 @@ int rseq_mempool_attr_set_poison(struct rseq_mempool_attr *attr,
 
 enum rseq_mempool_populate_policy {
        /*
-        * RSEQ_MEMPOOL_POPULATE_NONE (default):
+        * RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE (default):
         *   Do not populate pages for any of the CPUs when creating the
         *   mempool. Rely on copy-on-write (COW) of per-cpu pages to
         *   populate per-cpu pages from the initial values pages on
-        *   first write.
+        *   first write. This mempool is only meant for single-process
+        *   use (private mapping).
         */
-       RSEQ_MEMPOOL_POPULATE_NONE = 0,
+       RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE = 0,
        /*
-        * RSEQ_MEMPOOL_POPULATE_ALL:
+        * RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL:
         *   Populate pages for all CPUs from 0 to (max_nr_cpus - 1)
-        *   when creating the mempool.
+        *   when creating the mempool. This mempool is only meant for
+        *   single-process use (private mapping).
         */
-       RSEQ_MEMPOOL_POPULATE_ALL = 1,
+       RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL = 1,
 };
 
 /*

diff --git a/src/rseq-mempool.c b/src/rseq-mempool.c
index 9e0b7371de1570d9f2ab2567ff69e5c31687af08..e0f7e49f2fcee8702a6bf85ad84562d03d1aed4b 100644 (file)
--- a/src/rseq-mempool.c
+++ b/src/rseq-mempool.c
@@ -103,7 +103,7 @@ struct rseq_mempool_range {
         * - CPU 1,
         * ...
         * - CPU max_nr_cpus - 1
-        * - init values (unpopulated for RSEQ_MEMPOOL_POPULATE_ALL).
+        * - init values (unpopulated for RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL).
         *   Aliases with free-list for non-robust populate none pool.
         * - free list (for robust pool).
         *
@@ -116,7 +116,7 @@ struct rseq_mempool_range {
        void *base;
        /*
         * The init values contains malloc_init/zmalloc values.
-        * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_ALL.
+        * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL.
         */
        void *init;
        size_t next_unused;
@@ -216,12 +216,12 @@ void __rseq_percpu *__rseq_free_list_to_percpu_ptr(const struct rseq_mempool *po
                /* Skip cpus. */
                p -= pool->attr.max_nr_cpus * pool->attr.stride;
                /* Skip init values */
-               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                        p -= pool->attr.stride;
 
        } else {
                /* Populate none free list is in init values */
-               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                        p -= pool->attr.max_nr_cpus * pool->attr.stride;
        }
        return p;
@@ -235,12 +235,12 @@ struct free_list_node *__rseq_percpu_to_free_list_ptr(const struct rseq_mempool
                /* Skip cpus. */
                p += pool->attr.max_nr_cpus * pool->attr.stride;
                /* Skip init values */
-               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                        p += pool->attr.stride;
 
        } else {
                /* Populate none free list is in init values */
-               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+               if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                        p += pool->attr.max_nr_cpus * pool->attr.stride;
        }
        return (struct free_list_node *) p;
@@ -746,7 +746,7 @@ struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool)
        page_size = rseq_get_page_len();
 
        range_len = pool->attr.stride * pool->attr.max_nr_cpus;
-       if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+       if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                range_len += pool->attr.stride; /* init values */
        if (pool->attr.robust_set)
                range_len += pool->attr.stride; /* free list */
@@ -761,7 +761,7 @@ struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool)
        range->mmap_addr = header;
        range->mmap_len = page_size + range_len;
 
-       if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL) {
+       if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL) {
                range->init = base + (pool->attr.stride * pool->attr.max_nr_cpus);
                /* Populate init values pages from memfd */
                if (rseq_memfd_reserve_init(range->init, pool->attr.stride))
@@ -831,7 +831,7 @@ int rseq_mempool_memfd_ref(struct rseq_mempool *pool)
 {
        int ret = 0;
 
-       if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_ALL)
+       if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                return 0;
 
        pthread_mutex_lock(&memfd.lock);
@@ -852,7 +852,7 @@ unlock:
 static
 void rseq_mempool_memfd_unref(struct rseq_mempool *pool)
 {
-       if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_ALL)
+       if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                return;
 
        pthread_mutex_lock(&memfd.lock);
@@ -931,7 +931,8 @@ struct rseq_mempool *rseq_mempool_create(const char *pool_name,
                break;
        case MEMPOOL_TYPE_GLOBAL:
                /* Override populate policy for global type. */
-               attr.populate_policy = RSEQ_MEMPOOL_POPULATE_ALL;
+               if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE)
+                       attr.populate_policy = RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL;
                /* Use a 1-cpu pool for global mempool type. */
                attr.max_nr_cpus = 1;
                break;

diff --git a/tests/mempool_cow_race_test.c b/tests/mempool_cow_race_test.c
index 2f2bbb27679429ed88e74b147521aa27fdb664b9..c086c4d7985a629ab04ac1522168c8d7dbe4a367 100644 (file)
--- a/tests/mempool_cow_race_test.c
+++ b/tests/mempool_cow_race_test.c
@@ -79,7 +79,7 @@ static void *test_init_thread(void *arg)
                ret = rseq_mempool_attr_set_max_nr_ranges(attr, 1);
                if (ret)
                        abort();
-               ret = rseq_mempool_attr_set_populate_policy(attr, RSEQ_MEMPOOL_POPULATE_NONE);
+               ret = rseq_mempool_attr_set_populate_policy(attr, RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE);
                if (ret)
                        abort();
                mempool = rseq_mempool_create("test_data", sizeof(struct test_data), attr);

diff --git a/tests/mempool_test.c b/tests/mempool_test.c
index 0462499ff473dba7c91835a5695742b767aba9b7..e869f4a2a0a2e4c58a309fc3cc1131e9fd79d2f8 100644 (file)
--- a/tests/mempool_test.c
+++ b/tests/mempool_test.c
@@ -67,7 +67,7 @@ static void test_mempool_fill(enum rseq_mempool_populate_policy policy,
        ok(ret == 0, "Setting mempool poison");
        ret = rseq_mempool_attr_set_populate_policy(attr, policy);
        ok(ret == 0, "Setting mempool populate policy to %s",
-               policy == RSEQ_MEMPOOL_POPULATE_NONE ? "NONE" : "ALL");
+               policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE ? "NONE" : "ALL");
        mempool = rseq_mempool_create("test_data",
                        sizeof(struct test_data), attr);
        ok(mempool, "Create mempool of size %zu", stride);
@@ -159,7 +159,7 @@ static void test_robust_corrupt_after_free(struct rseq_mempool *pool,
         * after the last cpu memory range for populate all, and after
         * the init values memory range for populate none.
         */
-       if (policy == RSEQ_MEMPOOL_POPULATE_ALL)
+       if (policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool));
        else
                cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool) + 1);
@@ -190,7 +190,7 @@ static void test_robust_free_list_corruption(struct rseq_mempool *pool,
         * after the last cpu memory range for populate all, and after
         * the init values memory range for populate none.
         */
-       if (policy == RSEQ_MEMPOOL_POPULATE_ALL)
+       if (policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL)
                cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool));
        else
                cpuptr = (struct test_data *) rseq_percpu_ptr(ptr, rseq_mempool_get_max_nr_cpus(pool) + 1);
@@ -277,7 +277,7 @@ static void run_robust_tests(enum rseq_mempool_populate_policy policy)
 
        ret = rseq_mempool_attr_set_populate_policy(attr, policy);
        ok(ret == 0, "Setting mempool populate policy to %s",
-               policy == RSEQ_MEMPOOL_POPULATE_NONE ? "NONE" : "ALL");
+               policy == RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE ? "PRIVATE_NONE" : "PRIVATE_ALL");
 
        pool = rseq_mempool_create("mempool-robust",
                                sizeof(struct test_data), attr);
@@ -315,8 +315,8 @@ int main(void)
        for (nr_ranges = 1; nr_ranges < 32; nr_ranges <<= 1) {
                /* From page size to 64kB */
                for (len = rseq_get_page_len(); len < 65536; len <<= 1) {
-                       test_mempool_fill(RSEQ_MEMPOOL_POPULATE_ALL, nr_ranges, len);
-                       test_mempool_fill(RSEQ_MEMPOOL_POPULATE_NONE, nr_ranges, len);
+                       test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL, nr_ranges, len);
+                       test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE, nr_ranges, len);
                }
        }
 
@@ -325,12 +325,12 @@ int main(void)
                len = 65536;
        /* From min(page size, 64kB) to 4MB */
        for (; len < 4096 * 1024; len <<= 1) {
-               test_mempool_fill(RSEQ_MEMPOOL_POPULATE_ALL, 1, len);
-               test_mempool_fill(RSEQ_MEMPOOL_POPULATE_NONE, 1, len);
+               test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL, 1, len);
+               test_mempool_fill(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE, 1, len);
        }
 
-       run_robust_tests(RSEQ_MEMPOOL_POPULATE_ALL);
-       run_robust_tests(RSEQ_MEMPOOL_POPULATE_NONE);
+       run_robust_tests(RSEQ_MEMPOOL_POPULATE_PRIVATE_ALL);
+       run_robust_tests(RSEQ_MEMPOOL_POPULATE_PRIVATE_NONE);
 
        exit(exit_status());
 }