Lines Matching +full:cpu +full:- +full:offset
1 // SPDX-License-Identifier: LGPL-2.1
47 static __thread __attribute__((tls_model("initial-exec")))
52 static __thread __attribute__((tls_model("initial-exec"), unused))
116 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
207 "addiu " INJECT_ASM_REG ", -1\n\t" \
225 if (loc_nr_loops == -1 && opt_modulo) { \
226 if (yield_mod_cnt == opt_modulo - 1) { \
301 intptr_t offset; member
318 intptr_t offset; member
327 /* A simple percpu spinlock. Grabs lock on current cpu. */
330 int cpu; in rseq_this_cpu_lock() local
335 cpu = rseq_cpu_start(); in rseq_this_cpu_lock()
336 ret = rseq_cmpeqv_storev(&lock->c[cpu].v, in rseq_this_cpu_lock()
337 0, 1, cpu); in rseq_this_cpu_lock()
347 return cpu; in rseq_this_cpu_lock()
350 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) in rseq_percpu_unlock() argument
352 assert(lock->c[cpu].v == 1); in rseq_percpu_unlock()
357 rseq_smp_store_release(&lock->c[cpu].v, 0); in rseq_percpu_unlock()
363 struct spinlock_test_data *data = thread_data->data; in test_percpu_spinlock_thread()
366 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
369 reps = thread_data->reps; in test_percpu_spinlock_thread()
371 int cpu = rseq_cpu_start(); in test_percpu_spinlock_thread() local
373 cpu = rseq_this_cpu_lock(&data->lock); in test_percpu_spinlock_thread()
374 data->c[cpu].count++; in test_percpu_spinlock_thread()
375 rseq_percpu_unlock(&data->lock, cpu); in test_percpu_spinlock_thread()
384 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
391 * A simple test which implements a sharded counter using a per-cpu
393 * per-cpu increment; however, this is reasonable for a test and the
442 struct inc_test_data *data = thread_data->data; in test_percpu_inc_thread()
445 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
448 reps = thread_data->reps; in test_percpu_inc_thread()
453 int cpu; in test_percpu_inc_thread() local
455 cpu = rseq_cpu_start(); in test_percpu_inc_thread()
456 ret = rseq_addv(&data->c[cpu].count, 1, cpu); in test_percpu_inc_thread()
466 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
519 int cpu; in this_cpu_list_push() local
525 cpu = rseq_cpu_start(); in this_cpu_list_push()
526 /* Load list->c[cpu].head with single-copy atomicity. */ in this_cpu_list_push()
527 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); in this_cpu_list_push()
529 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_push()
530 node->next = (struct percpu_list_node *)expect; in this_cpu_list_push()
531 ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); in this_cpu_list_push()
537 *_cpu = cpu; in this_cpu_list_push()
541 * Unlike a traditional lock-less linked list; the availability of a
543 * ABA-type races.
549 int cpu; in this_cpu_list_pop() local
554 off_t offset; in this_cpu_list_pop() local
557 cpu = rseq_cpu_start(); in this_cpu_list_pop()
558 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_pop()
560 offset = offsetof(struct percpu_list_node, next); in this_cpu_list_pop()
563 offset, load, cpu); in this_cpu_list_pop()
573 *_cpu = cpu; in this_cpu_list_pop()
581 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu) in __percpu_list_pop() argument
585 node = list->c[cpu].head; in __percpu_list_pop()
588 list->c[cpu].head = node->next; in __percpu_list_pop()
619 /* Simultaneous modification to a per-cpu linked list from many threads. */
631 /* Generate list entries for every usable cpu. */ in test_percpu_list()
643 node->data = j; in test_percpu_list()
644 node->next = list.c[i].head; in test_percpu_list()
675 sum += node->data; in test_percpu_list()
693 int cpu; in this_cpu_buffer_push() local
698 intptr_t offset; in this_cpu_buffer_push() local
701 cpu = rseq_cpu_start(); in this_cpu_buffer_push()
702 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_push()
703 if (offset == buffer->c[cpu].buflen) in this_cpu_buffer_push()
706 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; in this_cpu_buffer_push()
707 newval_final = offset + 1; in this_cpu_buffer_push()
708 targetptr_final = &buffer->c[cpu].offset; in this_cpu_buffer_push()
711 targetptr_final, offset, targetptr_spec, in this_cpu_buffer_push()
712 newval_spec, newval_final, cpu); in this_cpu_buffer_push()
715 offset, targetptr_spec, newval_spec, in this_cpu_buffer_push()
716 newval_final, cpu); in this_cpu_buffer_push()
724 *_cpu = cpu; in this_cpu_buffer_push()
732 int cpu; in this_cpu_buffer_pop() local
736 intptr_t offset; in this_cpu_buffer_pop() local
739 cpu = rseq_cpu_start(); in this_cpu_buffer_pop()
740 /* Load offset with single-copy atomicity. */ in this_cpu_buffer_pop()
741 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_pop()
742 if (offset == 0) { in this_cpu_buffer_pop()
746 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); in this_cpu_buffer_pop()
747 newval = offset - 1; in this_cpu_buffer_pop()
748 targetptr = (intptr_t *)&buffer->c[cpu].offset; in this_cpu_buffer_pop()
749 ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, in this_cpu_buffer_pop()
750 (intptr_t *)&buffer->c[cpu].array[offset - 1], in this_cpu_buffer_pop()
751 (intptr_t)head, newval, cpu); in this_cpu_buffer_pop()
757 *_cpu = cpu; in this_cpu_buffer_pop()
766 int cpu) in __percpu_buffer_pop() argument
769 intptr_t offset; in __percpu_buffer_pop() local
771 offset = buffer->c[cpu].offset; in __percpu_buffer_pop()
772 if (offset == 0) in __percpu_buffer_pop()
774 head = buffer->c[cpu].array[offset - 1]; in __percpu_buffer_pop()
775 buffer->c[cpu].offset = offset - 1; in __percpu_buffer_pop()
810 /* Simultaneous modification to a per-cpu buffer from many threads. */
822 /* Generate list entries for every usable cpu. */ in test_percpu_buffer()
827 /* Worse-case is every item in same CPU. */ in test_percpu_buffer()
839 * We could theoretically put the word-sized in test_percpu_buffer()
847 node->data = j; in test_percpu_buffer()
848 buffer.c[i].array[j - 1] = node; in test_percpu_buffer()
849 buffer.c[i].offset++; in test_percpu_buffer()
879 sum += node->data; in test_percpu_buffer()
898 int cpu; in this_cpu_memcpy_buffer_push() local
901 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_push() local
906 cpu = rseq_cpu_start(); in this_cpu_memcpy_buffer_push()
907 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_push()
908 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_push()
909 if (offset == buffer->c[cpu].buflen) in this_cpu_memcpy_buffer_push()
911 destptr = (char *)&buffer->c[cpu].array[offset]; in this_cpu_memcpy_buffer_push()
915 newval_final = offset + 1; in this_cpu_memcpy_buffer_push()
916 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_push()
919 targetptr_final, offset, in this_cpu_memcpy_buffer_push()
921 newval_final, cpu); in this_cpu_memcpy_buffer_push()
924 offset, destptr, srcptr, copylen, in this_cpu_memcpy_buffer_push()
925 newval_final, cpu); in this_cpu_memcpy_buffer_push()
933 *_cpu = cpu; in this_cpu_memcpy_buffer_push()
942 int cpu; in this_cpu_memcpy_buffer_pop() local
945 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_pop() local
950 cpu = rseq_cpu_start(); in this_cpu_memcpy_buffer_pop()
951 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_pop()
952 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_pop()
953 if (offset == 0) in this_cpu_memcpy_buffer_pop()
956 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; in this_cpu_memcpy_buffer_pop()
959 newval_final = offset - 1; in this_cpu_memcpy_buffer_pop()
960 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_pop()
962 offset, destptr, srcptr, copylen, in this_cpu_memcpy_buffer_pop()
963 newval_final, cpu); in this_cpu_memcpy_buffer_pop()
971 *_cpu = cpu; in this_cpu_memcpy_buffer_pop()
981 int cpu) in __percpu_memcpy_buffer_pop() argument
983 intptr_t offset; in __percpu_memcpy_buffer_pop() local
985 offset = buffer->c[cpu].offset; in __percpu_memcpy_buffer_pop()
986 if (offset == 0) in __percpu_memcpy_buffer_pop()
988 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); in __percpu_memcpy_buffer_pop()
989 buffer->c[cpu].offset = offset - 1; in __percpu_memcpy_buffer_pop()
1025 /* Simultaneous modification to a per-cpu buffer from many threads. */
1037 /* Generate list entries for every usable cpu. */ in test_percpu_memcpy_buffer()
1042 /* Worse-case is every item in same CPU. */ in test_percpu_memcpy_buffer()
1052 * We could theoretically put the word-sized in test_percpu_memcpy_buffer()
1058 buffer.c[i].array[j - 1].data1 = j; in test_percpu_memcpy_buffer()
1059 buffer.c[i].array[j - 1].data2 = j + 1; in test_percpu_memcpy_buffer()
1060 buffer.c[i].offset++; in test_percpu_memcpy_buffer()
1156 while (!atomic_load(&args->percpu_list_ptr)) {} in test_membarrier_worker_thread()
1162 int cpu = rseq_cpu_start(); in test_membarrier_worker_thread() local
1164 ret = rseq_offset_deref_addv(&args->percpu_list_ptr, in test_membarrier_worker_thread()
1165 sizeof(struct percpu_list_entry) * cpu, 1, cpu); in test_membarrier_worker_thread()
1187 node->data = 0; in test_membarrier_init_percpu_list()
1188 node->next = NULL; in test_membarrier_init_percpu_list()
1189 list->c[i].head = node; in test_membarrier_init_percpu_list()
1198 free(list->c[i].head); in test_membarrier_free_percpu_list()
1207 * The manager thread swaps per-cpu lists that worker threads see,
1228 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a); in test_membarrier_manager_thread()
1230 while (!atomic_load(&args->stop)) { in test_membarrier_manager_thread()
1237 if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) { in test_membarrier_manager_thread()
1243 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b); in test_membarrier_manager_thread()
1246 errno != ENXIO /* missing CPU */) { in test_membarrier_manager_thread()
1251 * Cpu A should now only modify list_b, so the values in test_membarrier_manager_thread()
1254 expect_a = atomic_load(&list_a.c[cpu_a].head->data); in test_membarrier_manager_thread()
1261 if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) { in test_membarrier_manager_thread()
1267 atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a); in test_membarrier_manager_thread()
1270 errno != ENXIO /* missing CPU*/) { in test_membarrier_manager_thread()
1275 expect_b = atomic_load(&list_b.c[cpu_b].head->data); in test_membarrier_manager_thread()
1355 printf(" [-1 loops] Number of loops for delay injection 1\n"); in show_usage()
1356 printf(" [-2 loops] Number of loops for delay injection 2\n"); in show_usage()
1357 printf(" [-3 loops] Number of loops for delay injection 3\n"); in show_usage()
1358 printf(" [-4 loops] Number of loops for delay injection 4\n"); in show_usage()
1359 printf(" [-5 loops] Number of loops for delay injection 5\n"); in show_usage()
1360 printf(" [-6 loops] Number of loops for delay injection 6\n"); in show_usage()
1361 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); in show_usage()
1362 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); in show_usage()
1363 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); in show_usage()
1364 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); in show_usage()
1365 printf(" [-y] Yield\n"); in show_usage()
1366 printf(" [-k] Kill thread with signal\n"); in show_usage()
1367 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); in show_usage()
1368 printf(" [-t N] Number of threads (default 200)\n"); in show_usage()
1369 printf(" [-r N] Number of repetitions per thread (default 5000)\n"); in show_usage()
1370 printf(" [-d] Disable rseq system call (no initialization)\n"); in show_usage()
1371 printf(" [-D M] Disable rseq for each M threads\n"); in show_usage()
1372 …printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\… in show_usage()
1373 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); in show_usage()
1374 printf(" [-v] Verbose output.\n"); in show_usage()
1375 printf(" [-h] Show this help.\n"); in show_usage()
1384 if (argv[i][0] != '-') in main()
1400 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); in main()
1551 return -1; in main()