From 57d8b5869d5cbf370b4e3f9a651de7c6e7dbf9db Mon Sep 17 00:00:00 2001 From: Olivier Dion Date: Thu, 7 Mar 2024 16:14:36 -0500 Subject: [PATCH] percpu-alloc: Add move_pages batching Change-Id: I5e9f40d73f45f2abeded1d1f62056a478fd8455a Signed-off-by: Olivier Dion Signed-off-by: Mathieu Desnoyers --- src/rseq-alloc-utils.h | 2 ++ src/rseq-percpu-alloc.c | 46 ++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/rseq-alloc-utils.h b/src/rseq-alloc-utils.h index 6a57c3d..387d128 100644 --- a/src/rseq-alloc-utils.h +++ b/src/rseq-alloc-utils.h @@ -4,6 +4,8 @@ #ifndef _RSEQ_ALLOC_UTILS_H #define _RSEQ_ALLOC_UTILS_H +#define RSEQ_ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) + #define __rseq_align_mask(v, mask) (((v) + (mask)) & ~(mask)) #define rseq_align(v, align) __rseq_align_mask(v, (__typeof__(v)) (align) - 1) diff --git a/src/rseq-percpu-alloc.c b/src/rseq-percpu-alloc.c index 0a509c6..39794a9 100644 --- a/src/rseq-percpu-alloc.c +++ b/src/rseq-percpu-alloc.c @@ -68,6 +68,8 @@ #define BIT_PER_ULONG (8 * sizeof(unsigned long)) +#define MOVE_PAGES_BATCH_SIZE 4096 + struct free_list_node; struct free_list_node { @@ -158,7 +160,7 @@ void rseq_percpu_zero_item(struct rseq_percpu_pool *pool, uintptr_t item_offset) #ifdef HAVE_LIBNUMA int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags) { - unsigned long nr_pages, page; + unsigned long nr_pages; long ret, page_len; int cpu; @@ -167,16 +169,44 @@ int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags) page_len = rseq_get_page_len(); nr_pages = pool->percpu_len >> rseq_get_count_order_ulong(page_len); for (cpu = 0; cpu < pool->max_nr_cpus; cpu++) { - int node = numa_node_of_cpu(cpu); - /* TODO: batch move_pages() call with an array of pages. */ - for (page = 0; page < nr_pages; page++) { - void *pageptr = __rseq_pool_percpu_ptr(pool, cpu, page * page_len); - int status = -EPERM; + int status[MOVE_PAGES_BATCH_SIZE]; + int nodes[MOVE_PAGES_BATCH_SIZE]; + void *pages[MOVE_PAGES_BATCH_SIZE]; + + nodes[0] = numa_node_of_cpu(cpu); + for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) { + nodes[k] = nodes[0]; + } + + for (unsigned long page = 0; page < nr_pages;) { + + size_t max_k = RSEQ_ARRAY_SIZE(pages); + size_t left = nr_pages - page; - ret = move_pages(0, 1, &pageptr, &node, &status, numa_flags); - if (ret) + if (left < max_k) { + max_k = left; + } + + for (size_t k = 0; k < max_k; ++k, ++page) { + pages[k] = __rseq_pool_percpu_ptr(pool, cpu, page * page_len); + status[k] = -EPERM; + } + + ret = move_pages(0, max_k, pages, nodes, status, numa_flags); + + if (ret < 0) return ret; + + if (ret > 0) { + fprintf(stderr, "%lu pages were not migrated\n", ret); + for (size_t k = 0; k < max_k; ++k) { + if (status[k] < 0) + fprintf(stderr, + "Error while moving page %p to numa node %d: %u\n", + pages[k], nodes[k], -status[k]); + } + } } } return 0; -- 2.34.1