Lines Matching +full:cpu +full:- +full:offset
1 // SPDX-License-Identifier: LGPL-2.1
44 static __thread __attribute__((tls_model("initial-exec")))
49 static __thread __attribute__((tls_model("initial-exec"), unused))
113 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
204 "addiu " INJECT_ASM_REG ", -1\n\t" \
226 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
245 if (loc_nr_loops == -1 && opt_modulo) { \
246 if (yield_mod_cnt == opt_modulo - 1) { \
302 int rseq_membarrier_expedited(int cpu) in rseq_membarrier_expedited() argument
327 int rseq_membarrier_expedited(int cpu) in rseq_membarrier_expedited() argument
330 MEMBARRIER_CMD_FLAG_CPU, cpu); in rseq_membarrier_expedited()
388 intptr_t offset; member
405 intptr_t offset; member
414 /* A simple percpu spinlock. Grabs lock on current cpu. */
417 int cpu; in rseq_this_cpu_lock() local
422 cpu = get_current_cpu_id(); in rseq_this_cpu_lock()
423 if (cpu < 0) { in rseq_this_cpu_lock()
424 fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n", in rseq_this_cpu_lock()
425 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu); in rseq_this_cpu_lock()
429 &lock->c[cpu].v, in rseq_this_cpu_lock()
430 0, 1, cpu); in rseq_this_cpu_lock()
440 return cpu; in rseq_this_cpu_lock()
443 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) in rseq_percpu_unlock() argument
445 assert(lock->c[cpu].v == 1); in rseq_percpu_unlock()
450 rseq_smp_store_release(&lock->c[cpu].v, 0); in rseq_percpu_unlock()
456 struct spinlock_test_data *data = thread_data->data; in test_percpu_spinlock_thread()
459 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
462 reps = thread_data->reps; in test_percpu_spinlock_thread()
464 int cpu = rseq_this_cpu_lock(&data->lock); in test_percpu_spinlock_thread() local
465 data->c[cpu].count++; in test_percpu_spinlock_thread()
466 rseq_percpu_unlock(&data->lock, cpu); in test_percpu_spinlock_thread()
475 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
482 * A simple test which implements a sharded counter using a per-cpu
484 * per-cpu increment; however, this is reasonable for a test and the
533 struct inc_test_data *data = thread_data->data; in test_percpu_inc_thread()
536 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
539 reps = thread_data->reps; in test_percpu_inc_thread()
544 int cpu; in test_percpu_inc_thread() local
546 cpu = get_current_cpu_id(); in test_percpu_inc_thread()
548 &data->c[cpu].count, 1, cpu); in test_percpu_inc_thread()
558 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
611 int cpu; in this_cpu_list_push() local
617 cpu = get_current_cpu_id(); in this_cpu_list_push()
618 /* Load list->c[cpu].head with single-copy atomicity. */ in this_cpu_list_push()
619 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); in this_cpu_list_push()
621 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_push()
622 node->next = (struct percpu_list_node *)expect; in this_cpu_list_push()
624 targetptr, expect, newval, cpu); in this_cpu_list_push()
630 *_cpu = cpu; in this_cpu_list_push()
634 * Unlike a traditional lock-less linked list; the availability of a
636 * ABA-type races.
642 int cpu; in this_cpu_list_pop() local
647 long offset; in this_cpu_list_pop() local
650 cpu = get_current_cpu_id(); in this_cpu_list_pop()
651 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_pop()
653 offset = offsetof(struct percpu_list_node, next); in this_cpu_list_pop()
657 offset, load, cpu); in this_cpu_list_pop()
667 *_cpu = cpu; in this_cpu_list_pop()
675 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu) in __percpu_list_pop() argument
679 node = list->c[cpu].head; in __percpu_list_pop()
682 list->c[cpu].head = node->next; in __percpu_list_pop()
713 /* Simultaneous modification to a per-cpu linked list from many threads. */
725 /* Generate list entries for every usable cpu. */ in test_percpu_list()
737 node->data = j; in test_percpu_list()
738 node->next = list.c[i].head; in test_percpu_list()
769 sum += node->data; in test_percpu_list()
787 int cpu; in this_cpu_buffer_push() local
792 intptr_t offset; in this_cpu_buffer_push() local
795 cpu = get_current_cpu_id(); in this_cpu_buffer_push()
796 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_push()
797 if (offset == buffer->c[cpu].buflen) in this_cpu_buffer_push()
800 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; in this_cpu_buffer_push()
801 newval_final = offset + 1; in this_cpu_buffer_push()
802 targetptr_final = &buffer->c[cpu].offset; in this_cpu_buffer_push()
804 targetptr_final, offset, targetptr_spec, in this_cpu_buffer_push()
805 newval_spec, newval_final, cpu); in this_cpu_buffer_push()
813 *_cpu = cpu; in this_cpu_buffer_push()
821 int cpu; in this_cpu_buffer_pop() local
825 intptr_t offset; in this_cpu_buffer_pop() local
828 cpu = get_current_cpu_id(); in this_cpu_buffer_pop()
829 /* Load offset with single-copy atomicity. */ in this_cpu_buffer_pop()
830 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_pop()
831 if (offset == 0) { in this_cpu_buffer_pop()
835 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); in this_cpu_buffer_pop()
836 newval = offset - 1; in this_cpu_buffer_pop()
837 targetptr = (intptr_t *)&buffer->c[cpu].offset; in this_cpu_buffer_pop()
839 targetptr, offset, in this_cpu_buffer_pop()
840 (intptr_t *)&buffer->c[cpu].array[offset - 1], in this_cpu_buffer_pop()
841 (intptr_t)head, newval, cpu); in this_cpu_buffer_pop()
847 *_cpu = cpu; in this_cpu_buffer_pop()
856 int cpu) in __percpu_buffer_pop() argument
859 intptr_t offset; in __percpu_buffer_pop() local
861 offset = buffer->c[cpu].offset; in __percpu_buffer_pop()
862 if (offset == 0) in __percpu_buffer_pop()
864 head = buffer->c[cpu].array[offset - 1]; in __percpu_buffer_pop()
865 buffer->c[cpu].offset = offset - 1; in __percpu_buffer_pop()
900 /* Simultaneous modification to a per-cpu buffer from many threads. */
912 /* Generate list entries for every usable cpu. */ in test_percpu_buffer()
917 /* Worse-case is every item in same CPU. */ in test_percpu_buffer()
929 * We could theoretically put the word-sized in test_percpu_buffer()
937 node->data = j; in test_percpu_buffer()
938 buffer.c[i].array[j - 1] = node; in test_percpu_buffer()
939 buffer.c[i].offset++; in test_percpu_buffer()
969 sum += node->data; in test_percpu_buffer()
988 int cpu; in this_cpu_memcpy_buffer_push() local
991 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_push() local
996 cpu = get_current_cpu_id(); in this_cpu_memcpy_buffer_push()
997 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_push()
998 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_push()
999 if (offset == buffer->c[cpu].buflen) in this_cpu_memcpy_buffer_push()
1001 destptr = (char *)&buffer->c[cpu].array[offset]; in this_cpu_memcpy_buffer_push()
1005 newval_final = offset + 1; in this_cpu_memcpy_buffer_push()
1006 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_push()
1009 targetptr_final, offset, in this_cpu_memcpy_buffer_push()
1011 newval_final, cpu); in this_cpu_memcpy_buffer_push()
1019 *_cpu = cpu; in this_cpu_memcpy_buffer_push()
1028 int cpu; in this_cpu_memcpy_buffer_pop() local
1031 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_pop() local
1036 cpu = get_current_cpu_id(); in this_cpu_memcpy_buffer_pop()
1037 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_pop()
1038 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_pop()
1039 if (offset == 0) in this_cpu_memcpy_buffer_pop()
1042 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; in this_cpu_memcpy_buffer_pop()
1045 newval_final = offset - 1; in this_cpu_memcpy_buffer_pop()
1046 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_pop()
1048 targetptr_final, offset, destptr, srcptr, copylen, in this_cpu_memcpy_buffer_pop()
1049 newval_final, cpu); in this_cpu_memcpy_buffer_pop()
1057 *_cpu = cpu; in this_cpu_memcpy_buffer_pop()
1067 int cpu) in __percpu_memcpy_buffer_pop() argument
1069 intptr_t offset; in __percpu_memcpy_buffer_pop() local
1071 offset = buffer->c[cpu].offset; in __percpu_memcpy_buffer_pop()
1072 if (offset == 0) in __percpu_memcpy_buffer_pop()
1074 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); in __percpu_memcpy_buffer_pop()
1075 buffer->c[cpu].offset = offset - 1; in __percpu_memcpy_buffer_pop()
1111 /* Simultaneous modification to a per-cpu buffer from many threads. */
1123 /* Generate list entries for every usable cpu. */ in test_percpu_memcpy_buffer()
1128 /* Worse-case is every item in same CPU. */ in test_percpu_memcpy_buffer()
1138 * We could theoretically put the word-sized in test_percpu_memcpy_buffer()
1144 buffer.c[i].array[j - 1].data1 = j; in test_percpu_memcpy_buffer()
1145 buffer.c[i].array[j - 1].data2 = j + 1; in test_percpu_memcpy_buffer()
1146 buffer.c[i].offset++; in test_percpu_memcpy_buffer()
1244 while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {} in test_membarrier_worker_thread()
1250 int cpu = get_current_cpu_id(); in test_membarrier_worker_thread() local
1253 &args->percpu_list_ptr, in test_membarrier_worker_thread()
1254 sizeof(struct percpu_list_entry) * cpu, 1, cpu); in test_membarrier_worker_thread()
1276 node->data = 0; in test_membarrier_init_percpu_list()
1277 node->next = NULL; in test_membarrier_init_percpu_list()
1278 list->c[i].head = node; in test_membarrier_init_percpu_list()
1287 free(list->c[i].head); in test_membarrier_free_percpu_list()
1291 * The manager thread swaps per-cpu lists that worker threads see,
1312 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1314 while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1321 if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1327 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1329 errno != ENXIO /* missing CPU */) { in test_membarrier_manager_thread()
1334 * Cpu A should now only modify list_b, so the values in test_membarrier_manager_thread()
1337 expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE); in test_membarrier_manager_thread()
1344 if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1350 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1352 errno != ENXIO /* missing CPU*/) { in test_membarrier_manager_thread()
1357 expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE); in test_membarrier_manager_thread()
1435 printf(" [-1 loops] Number of loops for delay injection 1\n"); in show_usage()
1436 printf(" [-2 loops] Number of loops for delay injection 2\n"); in show_usage()
1437 printf(" [-3 loops] Number of loops for delay injection 3\n"); in show_usage()
1438 printf(" [-4 loops] Number of loops for delay injection 4\n"); in show_usage()
1439 printf(" [-5 loops] Number of loops for delay injection 5\n"); in show_usage()
1440 printf(" [-6 loops] Number of loops for delay injection 6\n"); in show_usage()
1441 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); in show_usage()
1442 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); in show_usage()
1443 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); in show_usage()
1444 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); in show_usage()
1445 printf(" [-y] Yield\n"); in show_usage()
1446 printf(" [-k] Kill thread with signal\n"); in show_usage()
1447 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); in show_usage()
1448 printf(" [-t N] Number of threads (default 200)\n"); in show_usage()
1449 printf(" [-r N] Number of repetitions per thread (default 5000)\n"); in show_usage()
1450 printf(" [-d] Disable rseq system call (no initialization)\n"); in show_usage()
1451 printf(" [-D M] Disable rseq for each M threads\n"); in show_usage()
1452 …printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\… in show_usage()
1453 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); in show_usage()
1454 printf(" [-v] Verbose output.\n"); in show_usage()
1455 printf(" [-h] Show this help.\n"); in show_usage()
1464 if (argv[i][0] != '-') in main()
1480 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); in main()
1600 fprintf(stderr, "Error: cpu id getter unavailable\n"); in main()
1635 return -1; in main()