Commit | Line | Data |
---|---|---|
b54c5158 MD |
1 | #define _GNU_SOURCE |
2 | #include <assert.h> | |
3 | #include <pthread.h> | |
4 | #include <sched.h> | |
5 | #include <stdint.h> | |
6 | #include <stdio.h> | |
7 | #include <stdlib.h> | |
8 | #include <string.h> | |
9 | #include <syscall.h> | |
10 | #include <unistd.h> | |
11 | #include <poll.h> | |
12 | #include <sys/types.h> | |
13 | #include <signal.h> | |
14 | #include <errno.h> | |
15 | ||
16 | static inline pid_t gettid(void) | |
17 | { | |
18 | return syscall(__NR_gettid); | |
19 | } | |
20 | ||
21 | #define NR_INJECT 9 | |
22 | static int loop_cnt[NR_INJECT + 1]; | |
23 | ||
24 | static int opt_modulo; | |
25 | ||
26 | static int opt_yield, opt_signal, opt_sleep, opt_fallback_cnt = 3, | |
27 | opt_disable_rseq, opt_threads = 200, | |
28 | opt_reps = 5000, opt_disable_mod = 0, opt_test = 's'; | |
29 | ||
30 | static __thread unsigned int signals_delivered; | |
31 | ||
32 | static struct rseq_lock rseq_lock; | |
33 | ||
34 | #ifndef BENCHMARK | |
35 | ||
36 | static __thread unsigned int yield_mod_cnt, nr_retry; | |
37 | ||
38 | #define printf_nobench(fmt, ...) printf(fmt, ## __VA_ARGS__) | |
39 | ||
40 | #define RSEQ_INJECT_INPUT \ | |
41 | , [loop_cnt_1]"m"(loop_cnt[1]) \ | |
42 | , [loop_cnt_2]"m"(loop_cnt[2]) \ | |
43 | , [loop_cnt_3]"m"(loop_cnt[3]) \ | |
44 | , [loop_cnt_4]"m"(loop_cnt[4]) \ | |
45 | , [loop_cnt_5]"m"(loop_cnt[5]) | |
46 | ||
47 | #if defined(__x86_64__) || defined(__i386__) | |
48 | ||
49 | #define INJECT_ASM_REG "eax" | |
50 | ||
51 | #define RSEQ_INJECT_CLOBBER \ | |
52 | , INJECT_ASM_REG | |
53 | ||
54 | #define RSEQ_INJECT_ASM(n) \ | |
55 | "mov %[loop_cnt_" #n "], %%" INJECT_ASM_REG "\n\t" \ | |
56 | "test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \ | |
57 | "jz 333f\n\t" \ | |
58 | "222:\n\t" \ | |
59 | "dec %%" INJECT_ASM_REG "\n\t" \ | |
60 | "jnz 222b\n\t" \ | |
61 | "333:\n\t" | |
62 | ||
63 | #elif defined(__ARMEL__) | |
64 | ||
65 | #define INJECT_ASM_REG "r4" | |
66 | ||
67 | #define RSEQ_INJECT_CLOBBER \ | |
68 | , INJECT_ASM_REG | |
69 | ||
70 | #define RSEQ_INJECT_ASM(n) \ | |
71 | "ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ | |
72 | "cmp " INJECT_ASM_REG ", #0\n\t" \ | |
73 | "beq 333f\n\t" \ | |
74 | "222:\n\t" \ | |
75 | "subs " INJECT_ASM_REG ", #1\n\t" \ | |
76 | "bne 222b\n\t" \ | |
77 | "333:\n\t" | |
78 | ||
79 | #elif __PPC__ | |
80 | #define INJECT_ASM_REG "r18" | |
81 | ||
82 | #define RSEQ_INJECT_CLOBBER \ | |
83 | , INJECT_ASM_REG | |
84 | ||
85 | #define RSEQ_INJECT_ASM(n) \ | |
86 | "lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ | |
87 | "cmpwi %%" INJECT_ASM_REG ", 0\n\t" \ | |
88 | "beq 333f\n\t" \ | |
89 | "222:\n\t" \ | |
90 | "subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \ | |
91 | "bne 222b\n\t" \ | |
92 | "333:\n\t" | |
93 | #else | |
94 | #error unsupported target | |
95 | #endif | |
96 | ||
97 | #define RSEQ_INJECT_FAILED \ | |
98 | nr_retry++; | |
99 | ||
100 | #define RSEQ_INJECT_C(n) \ | |
101 | { \ | |
102 | int loc_i, loc_nr_loops = loop_cnt[n]; \ | |
103 | \ | |
104 | for (loc_i = 0; loc_i < loc_nr_loops; loc_i++) { \ | |
105 | barrier(); \ | |
106 | } \ | |
107 | if (loc_nr_loops == -1 && opt_modulo) { \ | |
108 | if (yield_mod_cnt == opt_modulo - 1) { \ | |
109 | if (opt_sleep > 0) \ | |
110 | poll(NULL, 0, opt_sleep); \ | |
111 | if (opt_yield) \ | |
112 | sched_yield(); \ | |
113 | if (opt_signal) \ | |
114 | raise(SIGUSR1); \ | |
115 | yield_mod_cnt = 0; \ | |
116 | } else { \ | |
117 | yield_mod_cnt++; \ | |
118 | } \ | |
119 | } \ | |
120 | } | |
121 | ||
122 | #define RSEQ_FALLBACK_CNT \ | |
123 | opt_fallback_cnt | |
124 | ||
125 | #else | |
126 | ||
127 | #define printf_nobench(fmt, ...) | |
128 | ||
129 | #endif /* BENCHMARK */ | |
130 | ||
131 | #include <rseq.h> | |
132 | ||
133 | struct percpu_lock_entry { | |
134 | intptr_t v; | |
135 | } __attribute__((aligned(128))); | |
136 | ||
137 | struct percpu_lock { | |
138 | struct percpu_lock_entry c[CPU_SETSIZE]; | |
139 | }; | |
140 | ||
141 | struct test_data_entry { | |
142 | intptr_t count; | |
143 | } __attribute__((aligned(128))); | |
144 | ||
145 | struct spinlock_test_data { | |
146 | struct percpu_lock lock; | |
147 | struct test_data_entry c[CPU_SETSIZE]; | |
148 | }; | |
149 | ||
150 | struct spinlock_thread_test_data { | |
151 | struct spinlock_test_data *data; | |
152 | int reps; | |
153 | int reg; | |
154 | }; | |
155 | ||
156 | struct inc_test_data { | |
157 | struct test_data_entry c[CPU_SETSIZE]; | |
158 | }; | |
159 | ||
160 | struct inc_thread_test_data { | |
161 | struct inc_test_data *data; | |
162 | int reps; | |
163 | int reg; | |
164 | }; | |
165 | ||
166 | struct percpu_list_node { | |
167 | intptr_t data; | |
168 | struct percpu_list_node *next; | |
169 | }; | |
170 | ||
171 | struct percpu_list_entry { | |
172 | struct percpu_list_node *head; | |
173 | } __attribute__((aligned(128))); | |
174 | ||
175 | struct percpu_list { | |
176 | struct percpu_list_entry c[CPU_SETSIZE]; | |
177 | }; | |
178 | ||
179 | #define BUFFER_ITEM_PER_CPU 100 | |
180 | ||
181 | struct percpu_buffer_node { | |
182 | intptr_t data; | |
183 | }; | |
184 | ||
185 | struct percpu_buffer_entry { | |
186 | intptr_t offset; | |
187 | intptr_t buflen; | |
188 | struct percpu_buffer_node **array; | |
189 | } __attribute__((aligned(128))); | |
190 | ||
191 | struct percpu_buffer { | |
192 | struct percpu_buffer_entry c[CPU_SETSIZE]; | |
193 | }; | |
194 | ||
195 | #define MEMCPY_BUFFER_ITEM_PER_CPU 100 | |
196 | ||
197 | struct percpu_memcpy_buffer_node { | |
198 | intptr_t data1; | |
199 | uint64_t data2; | |
200 | }; | |
201 | ||
202 | struct percpu_memcpy_buffer_entry { | |
203 | intptr_t offset; | |
204 | intptr_t buflen; | |
205 | struct percpu_memcpy_buffer_node *array; | |
206 | } __attribute__((aligned(128))); | |
207 | ||
208 | struct percpu_memcpy_buffer { | |
209 | struct percpu_memcpy_buffer_entry c[CPU_SETSIZE]; | |
210 | }; | |
211 | ||
212 | /* A simple percpu spinlock. Returns the cpu lock was acquired on. */ | |
213 | static int rseq_percpu_lock(struct percpu_lock *lock) | |
214 | { | |
215 | struct rseq_state rseq_state; | |
216 | intptr_t *targetptr, newval; | |
217 | int cpu; | |
218 | bool result; | |
219 | ||
220 | for (;;) { | |
221 | do_rseq(&rseq_lock, rseq_state, cpu, result, targetptr, newval, | |
222 | { | |
223 | if (unlikely(lock->c[cpu].v)) { | |
224 | result = false; | |
225 | } else { | |
226 | newval = 1; | |
227 | targetptr = (intptr_t *)&lock->c[cpu].v; | |
228 | } | |
229 | }); | |
230 | if (likely(result)) | |
231 | break; | |
232 | } | |
233 | /* | |
234 | * Acquire semantic when taking lock after control dependency. | |
235 | * Matches smp_store_release(). | |
236 | */ | |
237 | smp_acquire__after_ctrl_dep(); | |
238 | return cpu; | |
239 | } | |
240 | ||
241 | static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) | |
242 | { | |
243 | assert(lock->c[cpu].v == 1); | |
244 | /* | |
245 | * Release lock, with release semantic. Matches | |
246 | * smp_acquire__after_ctrl_dep(). | |
247 | */ | |
248 | smp_store_release(&lock->c[cpu].v, 0); | |
249 | } | |
250 | ||
251 | void *test_percpu_spinlock_thread(void *arg) | |
252 | { | |
253 | struct spinlock_thread_test_data *thread_data = arg; | |
254 | struct spinlock_test_data *data = thread_data->data; | |
255 | int i, cpu; | |
256 | ||
257 | if (!opt_disable_rseq && thread_data->reg | |
258 | && rseq_register_current_thread()) | |
259 | abort(); | |
260 | for (i = 0; i < thread_data->reps; i++) { | |
261 | cpu = rseq_percpu_lock(&data->lock); | |
262 | data->c[cpu].count++; | |
263 | rseq_percpu_unlock(&data->lock, cpu); | |
264 | #ifndef BENCHMARK | |
265 | if (i != 0 && !(i % (thread_data->reps / 10))) | |
266 | printf("tid %d: count %d\n", (int) gettid(), i); | |
267 | #endif | |
268 | } | |
269 | printf_nobench("tid %d: number of retry: %d, signals delivered: %u, nr_fallback %u, nr_fallback_wait %u\n", | |
270 | (int) gettid(), nr_retry, signals_delivered, | |
271 | rseq_get_fallback_cnt(), | |
272 | rseq_get_fallback_wait_cnt()); | |
273 | if (rseq_unregister_current_thread()) | |
274 | abort(); | |
275 | return NULL; | |
276 | } | |
277 | ||
278 | /* | |
279 | * A simple test which implements a sharded counter using a per-cpu | |
280 | * lock. Obviously real applications might prefer to simply use a | |
281 | * per-cpu increment; however, this is reasonable for a test and the | |
282 | * lock can be extended to synchronize more complicated operations. | |
283 | */ | |
284 | void test_percpu_spinlock(void) | |
285 | { | |
286 | const int num_threads = opt_threads; | |
287 | int i, ret; | |
288 | uint64_t sum; | |
289 | pthread_t test_threads[num_threads]; | |
290 | struct spinlock_test_data data; | |
291 | struct spinlock_thread_test_data thread_data[num_threads]; | |
292 | ||
293 | memset(&data, 0, sizeof(data)); | |
294 | for (i = 0; i < num_threads; i++) { | |
295 | thread_data[i].reps = opt_reps; | |
296 | if (opt_disable_mod <= 0 || (i % opt_disable_mod)) | |
297 | thread_data[i].reg = 1; | |
298 | else | |
299 | thread_data[i].reg = 0; | |
300 | thread_data[i].data = &data; | |
301 | ret = pthread_create(&test_threads[i], NULL, | |
302 | test_percpu_spinlock_thread, &thread_data[i]); | |
303 | if (ret) { | |
304 | errno = ret; | |
305 | perror("pthread_create"); | |
306 | abort(); | |
307 | } | |
308 | } | |
309 | ||
310 | for (i = 0; i < num_threads; i++) { | |
311 | pthread_join(test_threads[i], NULL); | |
312 | if (ret) { | |
313 | errno = ret; | |
314 | perror("pthread_join"); | |
315 | abort(); | |
316 | } | |
317 | } | |
318 | ||
319 | sum = 0; | |
320 | for (i = 0; i < CPU_SETSIZE; i++) | |
321 | sum += data.c[i].count; | |
322 | ||
323 | assert(sum == (uint64_t)opt_reps * num_threads); | |
324 | } | |
325 | ||
326 | void *test_percpu_inc_thread(void *arg) | |
327 | { | |
328 | struct inc_thread_test_data *thread_data = arg; | |
329 | struct inc_test_data *data = thread_data->data; | |
330 | int i; | |
331 | ||
332 | if (!opt_disable_rseq && thread_data->reg | |
333 | && rseq_register_current_thread()) | |
334 | abort(); | |
335 | for (i = 0; i < thread_data->reps; i++) { | |
336 | struct rseq_state rseq_state; | |
337 | intptr_t *targetptr, newval; | |
338 | int cpu; | |
339 | bool result; | |
340 | ||
341 | do_rseq(&rseq_lock, rseq_state, cpu, result, targetptr, newval, | |
342 | { | |
343 | newval = (intptr_t)data->c[cpu].count + 1; | |
344 | targetptr = (intptr_t *)&data->c[cpu].count; | |
345 | }); | |
346 | ||
347 | #ifndef BENCHMARK | |
348 | if (i != 0 && !(i % (thread_data->reps / 10))) | |
349 | printf("tid %d: count %d\n", (int) gettid(), i); | |
350 | #endif | |
351 | } | |
352 | printf_nobench("tid %d: number of retry: %d, signals delivered: %u, nr_fallback %u, nr_fallback_wait %u\n", | |
353 | (int) gettid(), nr_retry, signals_delivered, | |
354 | rseq_get_fallback_cnt(), | |
355 | rseq_get_fallback_wait_cnt()); | |
356 | if (rseq_unregister_current_thread()) | |
357 | abort(); | |
358 | return NULL; | |
359 | } | |
360 | ||
361 | void test_percpu_inc(void) | |
362 | { | |
363 | const int num_threads = opt_threads; | |
364 | int i, ret; | |
365 | uint64_t sum; | |
366 | pthread_t test_threads[num_threads]; | |
367 | struct inc_test_data data; | |
368 | struct inc_thread_test_data thread_data[num_threads]; | |
369 | ||
370 | memset(&data, 0, sizeof(data)); | |
371 | for (i = 0; i < num_threads; i++) { | |
372 | thread_data[i].reps = opt_reps; | |
373 | if (opt_disable_mod <= 0 || (i % opt_disable_mod)) | |
374 | thread_data[i].reg = 1; | |
375 | else | |
376 | thread_data[i].reg = 0; | |
377 | thread_data[i].data = &data; | |
378 | ret = pthread_create(&test_threads[i], NULL, | |
379 | test_percpu_inc_thread, &thread_data[i]); | |
380 | if (ret) { | |
381 | errno = ret; | |
382 | perror("pthread_create"); | |
383 | abort(); | |
384 | } | |
385 | } | |
386 | ||
387 | for (i = 0; i < num_threads; i++) { | |
388 | pthread_join(test_threads[i], NULL); | |
389 | if (ret) { | |
390 | errno = ret; | |
391 | perror("pthread_join"); | |
392 | abort(); | |
393 | } | |
394 | } | |
395 | ||
396 | sum = 0; | |
397 | for (i = 0; i < CPU_SETSIZE; i++) | |
398 | sum += data.c[i].count; | |
399 | ||
400 | assert(sum == (uint64_t)opt_reps * num_threads); | |
401 | } | |
402 | ||
403 | int percpu_list_push(struct percpu_list *list, struct percpu_list_node *node) | |
404 | { | |
405 | struct rseq_state rseq_state; | |
406 | intptr_t *targetptr, newval; | |
407 | int cpu; | |
408 | bool result; | |
409 | ||
410 | do_rseq(&rseq_lock, rseq_state, cpu, result, targetptr, newval, | |
411 | { | |
412 | newval = (intptr_t)node; | |
413 | targetptr = (intptr_t *)&list->c[cpu].head; | |
414 | node->next = list->c[cpu].head; | |
415 | }); | |
416 | ||
417 | return cpu; | |
418 | } | |
419 | ||
420 | /* | |
421 | * Unlike a traditional lock-less linked list; the availability of a | |
422 | * rseq primitive allows us to implement pop without concerns over | |
423 | * ABA-type races. | |
424 | */ | |
425 | struct percpu_list_node *percpu_list_pop(struct percpu_list *list) | |
426 | { | |
427 | struct percpu_list_node *head, *next; | |
428 | struct rseq_state rseq_state; | |
429 | intptr_t *targetptr, newval; | |
430 | int cpu; | |
431 | bool result; | |
432 | ||
433 | do_rseq(&rseq_lock, rseq_state, cpu, result, targetptr, newval, | |
434 | { | |
435 | head = list->c[cpu].head; | |
436 | if (!head) { | |
437 | result = false; | |
438 | } else { | |
439 | next = head->next; | |
440 | newval = (intptr_t) next; | |
441 | targetptr = (intptr_t *) &list->c[cpu].head; | |
442 | } | |
443 | }); | |
444 | ||
445 | return head; | |
446 | } | |
447 | ||
448 | void *test_percpu_list_thread(void *arg) | |
449 | { | |
450 | int i; | |
451 | struct percpu_list *list = (struct percpu_list *)arg; | |
452 | ||
453 | if (rseq_register_current_thread()) | |
454 | abort(); | |
455 | ||
456 | for (i = 0; i < opt_reps; i++) { | |
457 | struct percpu_list_node *node = percpu_list_pop(list); | |
458 | ||
459 | if (opt_yield) | |
460 | sched_yield(); /* encourage shuffling */ | |
461 | if (node) | |
462 | percpu_list_push(list, node); | |
463 | } | |
464 | ||
465 | if (rseq_unregister_current_thread()) | |
466 | abort(); | |
467 | ||
468 | return NULL; | |
469 | } | |
470 | ||
471 | /* Simultaneous modification to a per-cpu linked list from many threads. */ | |
472 | void test_percpu_list(void) | |
473 | { | |
474 | const int num_threads = opt_threads; | |
475 | int i, j, ret; | |
476 | uint64_t sum = 0, expected_sum = 0; | |
477 | struct percpu_list list; | |
478 | pthread_t test_threads[num_threads]; | |
479 | cpu_set_t allowed_cpus; | |
480 | ||
481 | memset(&list, 0, sizeof(list)); | |
482 | ||
483 | /* Generate list entries for every usable cpu. */ | |
484 | sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); | |
485 | for (i = 0; i < CPU_SETSIZE; i++) { | |
486 | if (!CPU_ISSET(i, &allowed_cpus)) | |
487 | continue; | |
488 | for (j = 1; j <= 100; j++) { | |
489 | struct percpu_list_node *node; | |
490 | ||
491 | expected_sum += j; | |
492 | ||
493 | node = malloc(sizeof(*node)); | |
494 | assert(node); | |
495 | node->data = j; | |
496 | node->next = list.c[i].head; | |
497 | list.c[i].head = node; | |
498 | } | |
499 | } | |
500 | ||
501 | for (i = 0; i < num_threads; i++) { | |
502 | ret = pthread_create(&test_threads[i], NULL, | |
503 | test_percpu_list_thread, &list); | |
504 | if (ret) { | |
505 | errno = ret; | |
506 | perror("pthread_create"); | |
507 | abort(); | |
508 | } | |
509 | } | |
510 | ||
511 | for (i = 0; i < num_threads; i++) { | |
512 | pthread_join(test_threads[i], NULL); | |
513 | if (ret) { | |
514 | errno = ret; | |
515 | perror("pthread_join"); | |
516 | abort(); | |
517 | } | |
518 | } | |
519 | ||
520 | for (i = 0; i < CPU_SETSIZE; i++) { | |
521 | cpu_set_t pin_mask; | |
522 | struct percpu_list_node *node; | |
523 | ||
524 | if (!CPU_ISSET(i, &allowed_cpus)) | |
525 | continue; | |
526 | ||
527 | CPU_ZERO(&pin_mask); | |
528 | CPU_SET(i, &pin_mask); | |
529 | sched_setaffinity(0, sizeof(pin_mask), &pin_mask); | |
530 | ||
531 | while ((node = percpu_list_pop(&list))) { | |
532 | sum += node->data; | |
533 | free(node); | |
534 | } | |
535 | } | |
536 | ||
537 | /* | |
538 | * All entries should now be accounted for (unless some external | |
539 | * actor is interfering with our allowed affinity while this | |
540 | * test is running). | |
541 | */ | |
542 | assert(sum == expected_sum); | |
543 | } | |
544 | ||
545 | bool percpu_buffer_push(struct percpu_buffer *buffer, | |
546 | struct percpu_buffer_node *node) | |
547 | { | |
548 | struct rseq_state rseq_state; | |
549 | intptr_t *targetptr_spec, newval_spec; | |
550 | intptr_t *targetptr_final, newval_final; | |
551 | int cpu; | |
552 | bool result; | |
553 | ||
554 | do_rseq2(&rseq_lock, rseq_state, cpu, result, | |
555 | targetptr_spec, newval_spec, targetptr_final, newval_final, | |
556 | { | |
557 | intptr_t offset = buffer->c[cpu].offset; | |
558 | ||
559 | if (offset == buffer->c[cpu].buflen) { | |
560 | result = false; | |
561 | } else { | |
562 | newval_spec = (intptr_t)node; | |
563 | targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; | |
564 | newval_final = offset + 1; | |
565 | targetptr_final = &buffer->c[cpu].offset; | |
566 | } | |
567 | }); | |
568 | ||
569 | return result; | |
570 | } | |
571 | ||
572 | struct percpu_buffer_node *percpu_buffer_pop(struct percpu_buffer *buffer) | |
573 | { | |
574 | struct percpu_buffer_node *head; | |
575 | struct rseq_state rseq_state; | |
576 | intptr_t *targetptr, newval; | |
577 | int cpu; | |
578 | bool result; | |
579 | ||
580 | do_rseq(&rseq_lock, rseq_state, cpu, result, targetptr, newval, | |
581 | { | |
582 | intptr_t offset = buffer->c[cpu].offset; | |
583 | ||
584 | if (offset == 0) { | |
585 | result = false; | |
586 | } else { | |
587 | head = buffer->c[cpu].array[offset - 1]; | |
588 | newval = offset - 1; | |
589 | targetptr = (intptr_t *)&buffer->c[cpu].offset; | |
590 | } | |
591 | }); | |
592 | ||
593 | if (result) | |
594 | return head; | |
595 | else | |
596 | return NULL; | |
597 | } | |
598 | ||
599 | void *test_percpu_buffer_thread(void *arg) | |
600 | { | |
601 | int i; | |
602 | struct percpu_buffer *buffer = (struct percpu_buffer *)arg; | |
603 | ||
604 | if (rseq_register_current_thread()) | |
605 | abort(); | |
606 | ||
607 | for (i = 0; i < opt_reps; i++) { | |
608 | struct percpu_buffer_node *node = percpu_buffer_pop(buffer); | |
609 | ||
610 | if (opt_yield) | |
611 | sched_yield(); /* encourage shuffling */ | |
612 | if (node) { | |
613 | if (!percpu_buffer_push(buffer, node)) { | |
614 | /* Should increase buffer size. */ | |
615 | abort(); | |
616 | } | |
617 | } | |
618 | } | |
619 | ||
620 | if (rseq_unregister_current_thread()) | |
621 | abort(); | |
622 | ||
623 | return NULL; | |
624 | } | |
625 | ||
626 | /* Simultaneous modification to a per-cpu buffer from many threads. */ | |
627 | void test_percpu_buffer(void) | |
628 | { | |
629 | const int num_threads = opt_threads; | |
630 | int i, j, ret; | |
631 | uint64_t sum = 0, expected_sum = 0; | |
632 | struct percpu_buffer buffer; | |
633 | pthread_t test_threads[num_threads]; | |
634 | cpu_set_t allowed_cpus; | |
635 | ||
636 | memset(&buffer, 0, sizeof(buffer)); | |
637 | ||
638 | /* Generate list entries for every usable cpu. */ | |
639 | sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); | |
640 | for (i = 0; i < CPU_SETSIZE; i++) { | |
641 | if (!CPU_ISSET(i, &allowed_cpus)) | |
642 | continue; | |
643 | /* Worse-case is every item in same CPU. */ | |
644 | buffer.c[i].array = | |
645 | malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE | |
646 | * BUFFER_ITEM_PER_CPU); | |
647 | assert(buffer.c[i].array); | |
648 | buffer.c[i].buflen = CPU_SETSIZE * BUFFER_ITEM_PER_CPU; | |
649 | for (j = 1; j <= BUFFER_ITEM_PER_CPU; j++) { | |
650 | struct percpu_buffer_node *node; | |
651 | ||
652 | expected_sum += j; | |
653 | ||
654 | /* | |
655 | * We could theoretically put the word-sized | |
656 | * "data" directly in the buffer. However, we | |
657 | * want to model objects that would not fit | |
658 | * within a single word, so allocate an object | |
659 | * for each node. | |
660 | */ | |
661 | node = malloc(sizeof(*node)); | |
662 | assert(node); | |
663 | node->data = j; | |
664 | buffer.c[i].array[j - 1] = node; | |
665 | buffer.c[i].offset++; | |
666 | } | |
667 | } | |
668 | ||
669 | for (i = 0; i < num_threads; i++) { | |
670 | ret = pthread_create(&test_threads[i], NULL, | |
671 | test_percpu_buffer_thread, &buffer); | |
672 | if (ret) { | |
673 | errno = ret; | |
674 | perror("pthread_create"); | |
675 | abort(); | |
676 | } | |
677 | } | |
678 | ||
679 | for (i = 0; i < num_threads; i++) { | |
680 | pthread_join(test_threads[i], NULL); | |
681 | if (ret) { | |
682 | errno = ret; | |
683 | perror("pthread_join"); | |
684 | abort(); | |
685 | } | |
686 | } | |
687 | ||
688 | for (i = 0; i < CPU_SETSIZE; i++) { | |
689 | cpu_set_t pin_mask; | |
690 | struct percpu_buffer_node *node; | |
691 | ||
692 | if (!CPU_ISSET(i, &allowed_cpus)) | |
693 | continue; | |
694 | ||
695 | CPU_ZERO(&pin_mask); | |
696 | CPU_SET(i, &pin_mask); | |
697 | sched_setaffinity(0, sizeof(pin_mask), &pin_mask); | |
698 | ||
699 | while ((node = percpu_buffer_pop(&buffer))) { | |
700 | sum += node->data; | |
701 | free(node); | |
702 | } | |
703 | free(buffer.c[i].array); | |
704 | } | |
705 | ||
706 | /* | |
707 | * All entries should now be accounted for (unless some external | |
708 | * actor is interfering with our allowed affinity while this | |
709 | * test is running). | |
710 | */ | |
711 | assert(sum == expected_sum); | |
712 | } | |
713 | ||
714 | bool percpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, | |
715 | struct percpu_memcpy_buffer_node item) | |
716 | { | |
717 | struct rseq_state rseq_state; | |
718 | char *destptr, *srcptr; | |
719 | size_t copylen; | |
720 | intptr_t *targetptr_final, newval_final; | |
721 | int cpu; | |
722 | bool result; | |
723 | ||
724 | do_rseq_memcpy(&rseq_lock, rseq_state, cpu, result, | |
725 | destptr, srcptr, copylen, targetptr_final, newval_final, | |
726 | { | |
727 | intptr_t offset = buffer->c[cpu].offset; | |
728 | ||
729 | if (offset == buffer->c[cpu].buflen) { | |
730 | result = false; | |
731 | } else { | |
732 | destptr = (char *)&buffer->c[cpu].array[offset]; | |
733 | srcptr = (char *)&item; | |
734 | copylen = sizeof(item); | |
735 | newval_final = offset + 1; | |
736 | targetptr_final = &buffer->c[cpu].offset; | |
737 | } | |
738 | }); | |
739 | ||
740 | return result; | |
741 | } | |
742 | ||
743 | bool percpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, | |
744 | struct percpu_memcpy_buffer_node *item) | |
745 | { | |
746 | struct rseq_state rseq_state; | |
747 | char *destptr, *srcptr; | |
748 | size_t copylen; | |
749 | intptr_t *targetptr_final, newval_final; | |
750 | int cpu; | |
751 | bool result; | |
752 | ||
753 | do_rseq_memcpy(&rseq_lock, rseq_state, cpu, result, | |
754 | destptr, srcptr, copylen, targetptr_final, newval_final, | |
755 | { | |
756 | intptr_t offset = buffer->c[cpu].offset; | |
757 | ||
758 | if (offset == 0) { | |
759 | result = false; | |
760 | } else { | |
761 | destptr = (char *)item; | |
762 | srcptr = (char *)&buffer->c[cpu].array[offset - 1]; | |
763 | copylen = sizeof(*item); | |
764 | newval_final = offset - 1; | |
765 | targetptr_final = &buffer->c[cpu].offset; | |
766 | } | |
767 | }); | |
768 | ||
769 | return result; | |
770 | } | |
771 | ||
772 | void *test_percpu_memcpy_buffer_thread(void *arg) | |
773 | { | |
774 | int i; | |
775 | struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; | |
776 | ||
777 | if (rseq_register_current_thread()) | |
778 | abort(); | |
779 | ||
780 | for (i = 0; i < opt_reps; i++) { | |
781 | struct percpu_memcpy_buffer_node item; | |
782 | bool result; | |
783 | ||
784 | result = percpu_memcpy_buffer_pop(buffer, &item); | |
785 | if (opt_yield) | |
786 | sched_yield(); /* encourage shuffling */ | |
787 | if (result) { | |
788 | if (!percpu_memcpy_buffer_push(buffer, item)) { | |
789 | /* Should increase buffer size. */ | |
790 | abort(); | |
791 | } | |
792 | } | |
793 | } | |
794 | ||
795 | if (rseq_unregister_current_thread()) | |
796 | abort(); | |
797 | ||
798 | return NULL; | |
799 | } | |
800 | ||
801 | /* Simultaneous modification to a per-cpu buffer from many threads. */ | |
802 | void test_percpu_memcpy_buffer(void) | |
803 | { | |
804 | const int num_threads = opt_threads; | |
805 | int i, j, ret; | |
806 | uint64_t sum = 0, expected_sum = 0; | |
807 | struct percpu_memcpy_buffer buffer; | |
808 | pthread_t test_threads[num_threads]; | |
809 | cpu_set_t allowed_cpus; | |
810 | ||
811 | memset(&buffer, 0, sizeof(buffer)); | |
812 | ||
813 | /* Generate list entries for every usable cpu. */ | |
814 | sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); | |
815 | for (i = 0; i < CPU_SETSIZE; i++) { | |
816 | if (!CPU_ISSET(i, &allowed_cpus)) | |
817 | continue; | |
818 | /* Worse-case is every item in same CPU. */ | |
819 | buffer.c[i].array = | |
820 | malloc(sizeof(*buffer.c[i].array) * CPU_SETSIZE | |
821 | * MEMCPY_BUFFER_ITEM_PER_CPU); | |
822 | assert(buffer.c[i].array); | |
823 | buffer.c[i].buflen = CPU_SETSIZE * MEMCPY_BUFFER_ITEM_PER_CPU; | |
824 | for (j = 1; j <= MEMCPY_BUFFER_ITEM_PER_CPU; j++) { | |
825 | expected_sum += 2 * j + 1; | |
826 | ||
827 | /* | |
828 | * We could theoretically put the word-sized | |
829 | * "data" directly in the buffer. However, we | |
830 | * want to model objects that would not fit | |
831 | * within a single word, so allocate an object | |
832 | * for each node. | |
833 | */ | |
834 | buffer.c[i].array[j - 1].data1 = j; | |
835 | buffer.c[i].array[j - 1].data2 = j + 1; | |
836 | buffer.c[i].offset++; | |
837 | } | |
838 | } | |
839 | ||
840 | for (i = 0; i < num_threads; i++) { | |
841 | ret = pthread_create(&test_threads[i], NULL, | |
842 | test_percpu_memcpy_buffer_thread, &buffer); | |
843 | if (ret) { | |
844 | errno = ret; | |
845 | perror("pthread_create"); | |
846 | abort(); | |
847 | } | |
848 | } | |
849 | ||
850 | for (i = 0; i < num_threads; i++) { | |
851 | pthread_join(test_threads[i], NULL); | |
852 | if (ret) { | |
853 | errno = ret; | |
854 | perror("pthread_join"); | |
855 | abort(); | |
856 | } | |
857 | } | |
858 | ||
859 | for (i = 0; i < CPU_SETSIZE; i++) { | |
860 | cpu_set_t pin_mask; | |
861 | struct percpu_memcpy_buffer_node item; | |
862 | ||
863 | if (!CPU_ISSET(i, &allowed_cpus)) | |
864 | continue; | |
865 | ||
866 | CPU_ZERO(&pin_mask); | |
867 | CPU_SET(i, &pin_mask); | |
868 | sched_setaffinity(0, sizeof(pin_mask), &pin_mask); | |
869 | ||
870 | while (percpu_memcpy_buffer_pop(&buffer, &item)) { | |
871 | sum += item.data1; | |
872 | sum += item.data2; | |
873 | } | |
874 | free(buffer.c[i].array); | |
875 | } | |
876 | ||
877 | /* | |
878 | * All entries should now be accounted for (unless some external | |
879 | * actor is interfering with our allowed affinity while this | |
880 | * test is running). | |
881 | */ | |
882 | assert(sum == expected_sum); | |
883 | } | |
884 | ||
885 | static void test_signal_interrupt_handler(int signo) | |
886 | { | |
887 | signals_delivered++; | |
888 | } | |
889 | ||
890 | static int set_signal_handler(void) | |
891 | { | |
892 | int ret = 0; | |
893 | struct sigaction sa; | |
894 | sigset_t sigset; | |
895 | ||
896 | ret = sigemptyset(&sigset); | |
897 | if (ret < 0) { | |
898 | perror("sigemptyset"); | |
899 | return ret; | |
900 | } | |
901 | ||
902 | sa.sa_handler = test_signal_interrupt_handler; | |
903 | sa.sa_mask = sigset; | |
904 | sa.sa_flags = 0; | |
905 | ret = sigaction(SIGUSR1, &sa, NULL); | |
906 | if (ret < 0) { | |
907 | perror("sigaction"); | |
908 | return ret; | |
909 | } | |
910 | ||
911 | printf_nobench("Signal handler set for SIGUSR1\n"); | |
912 | ||
913 | return ret; | |
914 | } | |
915 | ||
916 | static void show_usage(int argc, char **argv) | |
917 | { | |
918 | printf("Usage : %s <OPTIONS>\n", | |
919 | argv[0]); | |
920 | printf("OPTIONS:\n"); | |
921 | printf(" [-1 loops] Number of loops for delay injection 1\n"); | |
922 | printf(" [-2 loops] Number of loops for delay injection 2\n"); | |
923 | printf(" [-3 loops] Number of loops for delay injection 3\n"); | |
924 | printf(" [-4 loops] Number of loops for delay injection 4\n"); | |
925 | printf(" [-5 loops] Number of loops for delay injection 5\n"); | |
926 | printf(" [-6 loops] Number of loops for delay injection 6 (-1 to enable -m)\n"); | |
927 | printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); | |
928 | printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); | |
929 | printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); | |
930 | printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); | |
931 | printf(" [-y] Yield\n"); | |
932 | printf(" [-k] Kill thread with signal\n"); | |
933 | printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); | |
934 | printf(" [-f N] Use fallback every N failure (>= 1)\n"); | |
935 | printf(" [-t N] Number of threads (default 200)\n"); | |
936 | printf(" [-r N] Number of repetitions per thread (default 5000)\n"); | |
937 | printf(" [-d] Disable rseq system call (no initialization)\n"); | |
938 | printf(" [-D M] Disable rseq for each M threads\n"); | |
939 | printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n"); | |
940 | printf(" [-h] Show this help.\n"); | |
941 | printf("\n"); | |
942 | } | |
943 | ||
944 | int main(int argc, char **argv) | |
945 | { | |
946 | int i; | |
947 | ||
948 | if (rseq_init_lock(&rseq_lock)) { | |
949 | perror("rseq_init_lock"); | |
950 | return -1; | |
951 | } | |
952 | if (set_signal_handler()) | |
953 | goto error; | |
954 | for (i = 1; i < argc; i++) { | |
955 | if (argv[i][0] != '-') | |
956 | continue; | |
957 | switch (argv[i][1]) { | |
958 | case '1': | |
959 | case '2': | |
960 | case '3': | |
961 | case '4': | |
962 | case '5': | |
963 | case '6': | |
964 | case '7': | |
965 | case '8': | |
966 | case '9': | |
967 | if (argc < i + 2) { | |
968 | show_usage(argc, argv); | |
969 | goto error; | |
970 | } | |
971 | loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); | |
972 | i++; | |
973 | break; | |
974 | case 'm': | |
975 | if (argc < i + 2) { | |
976 | show_usage(argc, argv); | |
977 | goto error; | |
978 | } | |
979 | opt_modulo = atol(argv[i + 1]); | |
980 | if (opt_modulo < 0) { | |
981 | show_usage(argc, argv); | |
982 | goto error; | |
983 | } | |
984 | i++; | |
985 | break; | |
986 | case 's': | |
987 | if (argc < i + 2) { | |
988 | show_usage(argc, argv); | |
989 | goto error; | |
990 | } | |
991 | opt_sleep = atol(argv[i + 1]); | |
992 | if (opt_sleep < 0) { | |
993 | show_usage(argc, argv); | |
994 | goto error; | |
995 | } | |
996 | i++; | |
997 | break; | |
998 | case 'y': | |
999 | opt_yield = 1; | |
1000 | break; | |
1001 | case 'k': | |
1002 | opt_signal = 1; | |
1003 | break; | |
1004 | case 'd': | |
1005 | opt_disable_rseq = 1; | |
1006 | break; | |
1007 | case 'D': | |
1008 | if (argc < i + 2) { | |
1009 | show_usage(argc, argv); | |
1010 | goto error; | |
1011 | } | |
1012 | opt_disable_mod = atol(argv[i + 1]); | |
1013 | if (opt_disable_mod < 0) { | |
1014 | show_usage(argc, argv); | |
1015 | goto error; | |
1016 | } | |
1017 | i++; | |
1018 | break; | |
1019 | case 'f': | |
1020 | if (argc < i + 2) { | |
1021 | show_usage(argc, argv); | |
1022 | goto error; | |
1023 | } | |
1024 | opt_fallback_cnt = atol(argv[i + 1]); | |
1025 | if (opt_fallback_cnt < 1) { | |
1026 | show_usage(argc, argv); | |
1027 | goto error; | |
1028 | } | |
1029 | i++; | |
1030 | break; | |
1031 | case 't': | |
1032 | if (argc < i + 2) { | |
1033 | show_usage(argc, argv); | |
1034 | goto error; | |
1035 | } | |
1036 | opt_threads = atol(argv[i + 1]); | |
1037 | if (opt_threads < 0) { | |
1038 | show_usage(argc, argv); | |
1039 | goto error; | |
1040 | } | |
1041 | i++; | |
1042 | break; | |
1043 | case 'r': | |
1044 | if (argc < i + 2) { | |
1045 | show_usage(argc, argv); | |
1046 | goto error; | |
1047 | } | |
1048 | opt_reps = atol(argv[i + 1]); | |
1049 | if (opt_reps < 0) { | |
1050 | show_usage(argc, argv); | |
1051 | goto error; | |
1052 | } | |
1053 | i++; | |
1054 | break; | |
1055 | case 'h': | |
1056 | show_usage(argc, argv); | |
1057 | goto end; | |
1058 | case 'T': | |
1059 | if (argc < i + 2) { | |
1060 | show_usage(argc, argv); | |
1061 | goto error; | |
1062 | } | |
1063 | opt_test = *argv[i + 1]; | |
1064 | switch (opt_test) { | |
1065 | case 's': | |
1066 | case 'l': | |
1067 | case 'i': | |
1068 | case 'b': | |
1069 | case 'm': | |
1070 | break; | |
1071 | default: | |
1072 | show_usage(argc, argv); | |
1073 | goto error; | |
1074 | } | |
1075 | i++; | |
1076 | break; | |
1077 | default: | |
1078 | show_usage(argc, argv); | |
1079 | goto error; | |
1080 | } | |
1081 | } | |
1082 | ||
1083 | if (!opt_disable_rseq && rseq_register_current_thread()) | |
1084 | goto error; | |
1085 | switch (opt_test) { | |
1086 | case 's': | |
1087 | printf_nobench("spinlock\n"); | |
1088 | test_percpu_spinlock(); | |
1089 | break; | |
1090 | case 'l': | |
1091 | printf_nobench("linked list\n"); | |
1092 | test_percpu_list(); | |
1093 | break; | |
1094 | case 'b': | |
1095 | printf_nobench("buffer\n"); | |
1096 | test_percpu_buffer(); | |
1097 | break; | |
1098 | case 'm': | |
1099 | printf_nobench("memcpy buffer\n"); | |
1100 | test_percpu_memcpy_buffer(); | |
1101 | break; | |
1102 | case 'i': | |
1103 | printf_nobench("counter increment\n"); | |
1104 | test_percpu_inc(); | |
1105 | break; | |
1106 | } | |
1107 | if (rseq_unregister_current_thread()) | |
1108 | abort(); | |
1109 | end: | |
1110 | return 0; | |
1111 | ||
1112 | error: | |
1113 | if (rseq_destroy_lock(&rseq_lock)) | |
1114 | perror("rseq_destroy_lock"); | |
1115 | return -1; | |
1116 | } |