Lines Matching +full:we +full:- +full:on +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0
9 * SMP-threaded, sysctl's added
11 * Enforced range limit on SEM_UNDO
30 * - FIFO ordering for semop() operations (just FIFO, not starvation
32 * - multiple semaphore operations that alter the same semaphore in
34 * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
36 * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
37 * - undo adjustments at process exit are limited to 0..SEMVMX.
38 * - namespace are supported.
39 * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtime by writing
41 * - statistics about the usage are reported in /proc/sysvipc/sem.
44 * - scalability:
45 * - all global variables are read-mostly.
46 * - semop() calls and semctl(RMID) are synchronized by RCU.
47 * - most operations do write operations (actually: spin_lock calls) to
48 * the per-semaphore array structure.
51 * trashing on the semaphore array spinlock will limit the scaling.
52 * - semncnt and semzcnt are calculated on demand in count_semcnt()
53 * - the task that performs a successful semop() scans the list of all
57 * - To improve the scalability, the actual wake-up calls are performed after
59 * - All work is done by the waker, the woken up task does not have to do
60 * anything - not even acquiring a lock or dropping a refcount.
61 * - A woken up task may not even touch the semaphore array anymore, it may
63 * - UNDO values are stored in an array (one per process and per
67 * - There are two lists of the pending operations: a per-array list
68 * and per-semaphore list (stored in the array). This allows to achieve FIFO
70 * The worst-case behavior is nevertheless O(N^2) for N wakeups.
100 * - semop
101 * - semctl, via SETVAL and SETALL.
102 * - at task exit when performing undo adjustments (see exit_sem).
105 spinlock_t lock; /* spinlock for fine-grained semtimedop */
106 struct list_head pending_alter; /* pending single-sop operations */
108 struct list_head pending_const; /* pending single-sop operations */
121 struct list_head list_id; /* undo requests on this array */
140 bool dupsop; /* sops on more than one sem_num */
147 struct list_head list_proc; /* per-process list: *
169 #define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS]) argument
177 #define SEMMSL_FAST 256 /* 512 bytes on stack */
178 #define SEMOPM_FAST 64 /* ~ 372 bytes on stack */
200 * * undo_list->lock for write
211 * Setting it from non-zero to 0 is a RELEASE, this is ensured by
214 * Testing if it is non-zero is an ACQUIRE, this is ensured by using
216 * Setting it from 0 to non-zero must be ordered with regards to
218 * is inside a spin_lock() and after a write from 0 to non-zero a
235 * 3) current->state:
236 * current->state is set to TASK_INTERRUPTIBLE while holding sem_lock().
241 * See also ipc/mqueue.c for more details on the covered races.
249 void sem_init_ns(struct ipc_namespace *ns) in sem_init_ns() argument
251 ns->sc_semmsl = SEMMSL; in sem_init_ns()
252 ns->sc_semmns = SEMMNS; in sem_init_ns()
253 ns->sc_semopm = SEMOPM; in sem_init_ns()
254 ns->sc_semmni = SEMMNI; in sem_init_ns()
255 ns->used_sems = 0; in sem_init_ns()
256 ipc_init_ids(&ns->ids[IPC_SEM_IDS]); in sem_init_ns()
260 void sem_exit_ns(struct ipc_namespace *ns) in sem_exit_ns() argument
262 free_ipcs(ns, &sem_ids(ns), freeary); in sem_exit_ns()
263 idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr); in sem_exit_ns()
264 rhashtable_destroy(&ns->ids[IPC_SEM_IDS].key_ht); in sem_exit_ns()
277 * unmerge_queues - unmerge queues, if possible.
288 if (sma->complex_count) in unmerge_queues()
291 * We will switch back to simple mode. in unmerge_queues()
292 * Move all pending operation back into the per-semaphore in unmerge_queues()
295 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in unmerge_queues()
297 curr = &sma->sems[q->sops[0].sem_num]; in unmerge_queues()
299 list_add_tail(&q->list, &curr->pending_alter); in unmerge_queues()
301 INIT_LIST_HEAD(&sma->pending_alter); in unmerge_queues()
305 * merge_queues - merge single semop queues into global queue
308 * This function merges all per-semaphore queues into the global queue.
309 * It is necessary to achieve FIFO ordering for the pending single-sop
310 * operations when a multi-semop operation must sleep.
316 for (i = 0; i < sma->sem_nsems; i++) { in merge_queues()
317 struct sem *sem = &sma->sems[i]; in merge_queues()
319 list_splice_init(&sem->pending_alter, &sma->pending_alter); in merge_queues()
328 security_sem_free(&sma->sem_perm); in sem_rcu_free()
333 * Enter the mode suitable for non-simple operations:
341 if (sma->use_global_lock > 0) { in complexmode_enter()
343 * We are already in global lock mode. in complexmode_enter()
345 * counter until we return to simple mode. in complexmode_enter()
347 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS); in complexmode_enter()
350 WRITE_ONCE(sma->use_global_lock, USE_GLOBAL_LOCK_HYSTERESIS); in complexmode_enter()
352 for (i = 0; i < sma->sem_nsems; i++) { in complexmode_enter()
353 sem = &sma->sems[i]; in complexmode_enter()
354 spin_lock(&sem->lock); in complexmode_enter()
355 spin_unlock(&sem->lock); in complexmode_enter()
365 if (sma->complex_count) { in complexmode_tryleave()
367 * We must stay in complex mode in complexmode_tryleave()
371 if (sma->use_global_lock == 1) { in complexmode_tryleave()
374 smp_store_release(&sma->use_global_lock, 0); in complexmode_tryleave()
376 WRITE_ONCE(sma->use_global_lock, in complexmode_tryleave()
377 sma->use_global_lock-1); in complexmode_tryleave()
381 #define SEM_GLOBAL_LOCK (-1)
385 * Otherwise, lock the entire semaphore array, since we either have
386 * multiple semaphores in our own semops, or we need to look at
396 /* Complex operation - acquire a full lock */ in sem_lock()
397 ipc_lock_object(&sma->sem_perm); in sem_lock()
405 * Only one semaphore affected - try to optimize locking. in sem_lock()
411 idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in sem_lock()
412 sem = &sma->sems[idx]; in sem_lock()
418 if (!READ_ONCE(sma->use_global_lock)) { in sem_lock()
421 * Acquire the per-semaphore lock. in sem_lock()
423 spin_lock(&sem->lock); in sem_lock()
426 if (!smp_load_acquire(&sma->use_global_lock)) { in sem_lock()
428 return sops->sem_num; in sem_lock()
430 spin_unlock(&sem->lock); in sem_lock()
434 ipc_lock_object(&sma->sem_perm); in sem_lock()
436 if (sma->use_global_lock == 0) { in sem_lock()
438 * The use_global_lock mode ended while we waited for in sem_lock()
439 * sma->sem_perm.lock. Thus we must switch to locking in sem_lock()
440 * with sem->lock. in sem_lock()
442 * sma->use_global_lock after we have acquired sem->lock: in sem_lock()
443 * We own sma->sem_perm.lock, thus use_global_lock cannot in sem_lock()
446 spin_lock(&sem->lock); in sem_lock()
448 ipc_unlock_object(&sma->sem_perm); in sem_lock()
449 return sops->sem_num; in sem_lock()
454 * the caller that has set use_global_mode to non-zero. in sem_lock()
465 ipc_unlock_object(&sma->sem_perm); in sem_unlock()
467 struct sem *sem = &sma->sems[locknum]; in sem_unlock()
468 spin_unlock(&sem->lock); in sem_unlock()
478 static inline struct sem_array *sem_obtain_object(struct ipc_namespace *ns, int id) in sem_obtain_object() argument
480 struct kern_ipc_perm *ipcp = ipc_obtain_object_idr(&sem_ids(ns), id); in sem_obtain_object()
488 static inline struct sem_array *sem_obtain_object_check(struct ipc_namespace *ns, in sem_obtain_object_check() argument
491 struct kern_ipc_perm *ipcp = ipc_obtain_object_check(&sem_ids(ns), id); in sem_obtain_object_check()
501 sem_lock(sma, NULL, -1); in sem_lock_and_putref()
502 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in sem_lock_and_putref()
505 static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s) in sem_rmid() argument
507 ipc_rmid(&sem_ids(ns), &s->sem_perm); in sem_rmid()
514 if (nsems > (INT_MAX - sizeof(*sma)) / sizeof(sma->sems[0])) in sem_alloc()
525 * newary - Create a new semaphore set
526 * @ns: namespace
531 static int newary(struct ipc_namespace *ns, struct ipc_params *params) in newary() argument
535 key_t key = params->key; in newary()
536 int nsems = params->u.nsems; in newary()
537 int semflg = params->flg; in newary()
541 return -EINVAL; in newary()
542 if (ns->used_sems + nsems > ns->sc_semmns) in newary()
543 return -ENOSPC; in newary()
547 return -ENOMEM; in newary()
549 sma->sem_perm.mode = (semflg & S_IRWXUGO); in newary()
550 sma->sem_perm.key = key; in newary()
552 sma->sem_perm.security = NULL; in newary()
553 retval = security_sem_alloc(&sma->sem_perm); in newary()
560 INIT_LIST_HEAD(&sma->sems[i].pending_alter); in newary()
561 INIT_LIST_HEAD(&sma->sems[i].pending_const); in newary()
562 spin_lock_init(&sma->sems[i].lock); in newary()
565 sma->complex_count = 0; in newary()
566 sma->use_global_lock = USE_GLOBAL_LOCK_HYSTERESIS; in newary()
567 INIT_LIST_HEAD(&sma->pending_alter); in newary()
568 INIT_LIST_HEAD(&sma->pending_const); in newary()
569 INIT_LIST_HEAD(&sma->list_id); in newary()
570 sma->sem_nsems = nsems; in newary()
571 sma->sem_ctime = ktime_get_real_seconds(); in newary()
574 retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); in newary()
576 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in newary()
579 ns->used_sems += nsems; in newary()
581 sem_unlock(sma, -1); in newary()
584 return sma->sem_perm.id; in newary()
596 if (params->u.nsems > sma->sem_nsems) in sem_more_checks()
597 return -EINVAL; in sem_more_checks()
604 struct ipc_namespace *ns; in ksys_semget() local
612 ns = current->nsproxy->ipc_ns; in ksys_semget()
614 if (nsems < 0 || nsems > ns->sc_semmsl) in ksys_semget()
615 return -EINVAL; in ksys_semget()
621 return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params); in ksys_semget()
630 * perform_atomic_semop[_slow] - Attempt to perform semaphore
631 * operations on a given array.
639 * (2) 0 (wait-for-zero operation): semval is non-zero.
655 sops = q->sops; in perform_atomic_semop_slow()
656 nsops = q->nsops; in perform_atomic_semop_slow()
657 un = q->undo; in perform_atomic_semop_slow()
660 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop_slow()
661 curr = &sma->sems[idx]; in perform_atomic_semop_slow()
662 sem_op = sop->sem_op; in perform_atomic_semop_slow()
663 result = curr->semval; in perform_atomic_semop_slow()
674 if (sop->sem_flg & SEM_UNDO) { in perform_atomic_semop_slow()
675 int undo = un->semadj[sop->sem_num] - sem_op; in perform_atomic_semop_slow()
677 if (undo < (-SEMAEM - 1) || undo > SEMAEM) in perform_atomic_semop_slow()
679 un->semadj[sop->sem_num] = undo; in perform_atomic_semop_slow()
682 curr->semval = result; in perform_atomic_semop_slow()
685 sop--; in perform_atomic_semop_slow()
686 pid = q->pid; in perform_atomic_semop_slow()
688 ipc_update_pid(&sma->sems[sop->sem_num].sempid, pid); in perform_atomic_semop_slow()
689 sop--; in perform_atomic_semop_slow()
695 result = -ERANGE; in perform_atomic_semop_slow()
699 q->blocking = sop; in perform_atomic_semop_slow()
701 if (sop->sem_flg & IPC_NOWAIT) in perform_atomic_semop_slow()
702 result = -EAGAIN; in perform_atomic_semop_slow()
707 sop--; in perform_atomic_semop_slow()
709 sem_op = sop->sem_op; in perform_atomic_semop_slow()
710 sma->sems[sop->sem_num].semval -= sem_op; in perform_atomic_semop_slow()
711 if (sop->sem_flg & SEM_UNDO) in perform_atomic_semop_slow()
712 un->semadj[sop->sem_num] += sem_op; in perform_atomic_semop_slow()
713 sop--; in perform_atomic_semop_slow()
727 sops = q->sops; in perform_atomic_semop()
728 nsops = q->nsops; in perform_atomic_semop()
729 un = q->undo; in perform_atomic_semop()
731 if (unlikely(q->dupsop)) in perform_atomic_semop()
735 * We scan the semaphore set twice, first to ensure that the entire in perform_atomic_semop()
741 int idx = array_index_nospec(sop->sem_num, sma->sem_nsems); in perform_atomic_semop()
743 curr = &sma->sems[idx]; in perform_atomic_semop()
744 sem_op = sop->sem_op; in perform_atomic_semop()
745 result = curr->semval; in perform_atomic_semop()
748 goto would_block; /* wait-for-zero */ in perform_atomic_semop()
755 return -ERANGE; in perform_atomic_semop()
757 if (sop->sem_flg & SEM_UNDO) { in perform_atomic_semop()
758 int undo = un->semadj[sop->sem_num] - sem_op; in perform_atomic_semop()
761 if (undo < (-SEMAEM - 1) || undo > SEMAEM) in perform_atomic_semop()
762 return -ERANGE; in perform_atomic_semop()
767 curr = &sma->sems[sop->sem_num]; in perform_atomic_semop()
768 sem_op = sop->sem_op; in perform_atomic_semop()
770 if (sop->sem_flg & SEM_UNDO) { in perform_atomic_semop()
771 int undo = un->semadj[sop->sem_num] - sem_op; in perform_atomic_semop()
773 un->semadj[sop->sem_num] = undo; in perform_atomic_semop()
775 curr->semval += sem_op; in perform_atomic_semop()
776 ipc_update_pid(&curr->sempid, q->pid); in perform_atomic_semop()
782 q->blocking = sop; in perform_atomic_semop()
783 return sop->sem_flg & IPC_NOWAIT ? -EAGAIN : 1; in perform_atomic_semop()
791 sleeper = get_task_struct(q->sleeper); in wake_up_sem_queue_prepare()
794 smp_store_release(&q->status, error); in wake_up_sem_queue_prepare()
801 list_del(&q->list); in unlink_queue()
802 if (q->nsops > 1) in unlink_queue()
803 sma->complex_count--; in unlink_queue()
814 * Note that wait-for-zero operations are handled without restart.
819 if (!list_empty(&sma->pending_alter)) in check_restart()
822 /* we were a sleeping complex operation. Too difficult */ in check_restart()
823 if (q->nsops > 1) in check_restart()
827 * - complex operations always restart. in check_restart()
828 * - wait-for-zero are handled separately. in check_restart()
829 * - q is a previously sleeping simple operation that in check_restart()
832 * - If there are older (higher priority) decrements in check_restart()
835 * decremented to value - thus they won't proceed either. in check_restart()
841 * wake_const_ops - wake up non-alter tasks
844 * @wake_q: lockless wake-queue head.
848 * be called with semnum = -1, as well as with the number of each modified
851 * is stored in q->pid.
861 if (semnum == -1) in wake_const_ops()
862 pending_list = &sma->pending_const; in wake_const_ops()
864 pending_list = &sma->sems[semnum].pending_const; in wake_const_ops()
883 * do_smart_wakeup_zero - wakeup all wait for zero tasks
887 * @wake_q: lockless wake-queue head
889 * Checks all required queue for wait-for-zero operations, based
890 * on the actual changes that were performed on the semaphore array.
900 /* first: the per-semaphore queues, if known */ in do_smart_wakeup_zero()
905 if (sma->sems[num].semval == 0) { in do_smart_wakeup_zero()
915 for (i = 0; i < sma->sem_nsems; i++) { in do_smart_wakeup_zero()
916 if (sma->sems[i].semval == 0) { in do_smart_wakeup_zero()
927 semop_completed |= wake_const_ops(sma, -1, wake_q); in do_smart_wakeup_zero()
934 * update_queue - look for tasks that can be completed.
937 * @wake_q: lockless wake-queue head.
941 * be called with semnum = -1, as well as with the number of each modified
944 * is stored in q->pid.
955 if (semnum == -1) in update_queue()
956 pending_list = &sma->pending_alter; in update_queue()
958 pending_list = &sma->sems[semnum].pending_alter; in update_queue()
964 /* If we are scanning the single sop, per-semaphore list of in update_queue()
971 if (semnum != -1 && sma->sems[semnum].semval == 0) in update_queue()
976 /* Does q->sleeper still need to sleep? */ in update_queue()
986 do_smart_wakeup_zero(sma, q->sops, q->nsops, wake_q); in update_queue()
998 * set_semotime - set sem_otime
1008 sma->sems[0].sem_otime = ktime_get_real_seconds(); in set_semotime()
1010 sma->sems[sops[0].sem_num].sem_otime = in set_semotime()
1016 * do_smart_update - optimized update_queue
1021 * @wake_q: lockless wake-queue head
1024 * based on the actual changes that were performed on the semaphore array.
1025 * Note that the function does not do the actual wake-up: the caller is
1036 if (!list_empty(&sma->pending_alter)) { in do_smart_update()
1037 /* semaphore array uses the global queue - just process it. */ in do_smart_update()
1038 otime |= update_queue(sma, -1, wake_q); in do_smart_update()
1045 for (i = 0; i < sma->sem_nsems; i++) in do_smart_update()
1050 * - No complex ops, thus all sleeping ops are in do_smart_update()
1052 * - if we decreased the value, then any sleeping in do_smart_update()
1070 * check_qop: Test if a queued operation sleeps on the semaphore semnum
1075 struct sembuf *sop = q->blocking; in check_qop()
1078 * Linux always (since 0.99.10) reported a task as sleeping on all in check_qop()
1082 * might misbehave because it relies on the Linux behavior. in check_qop()
1086 current->comm, task_pid_nr(current)); in check_qop()
1088 if (sop->sem_num != semnum) in check_qop()
1091 if (count_zero && sop->sem_op == 0) in check_qop()
1093 if (!count_zero && sop->sem_op < 0) in check_qop()
1100 * semncnt number of tasks waiting on semval being nonzero
1101 * semzcnt number of tasks waiting on semval being zero
1103 * Per definition, a task waits only on the semaphore of the first semop
1116 l = &sma->sems[semnum].pending_const; in count_semcnt()
1118 l = &sma->sems[semnum].pending_alter; in count_semcnt()
1121 /* all task on a per-semaphore list sleep on exactly in count_semcnt()
1128 list_for_each_entry(q, &sma->pending_alter, list) { in count_semcnt()
1132 list_for_each_entry(q, &sma->pending_const, list) { in count_semcnt()
1141 * remains locked on exit.
1143 static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) in freeary() argument
1152 ipc_assert_locked_object(&sma->sem_perm); in freeary()
1153 list_for_each_entry_safe(un, tu, &sma->list_id, list_id) { in freeary()
1154 list_del(&un->list_id); in freeary()
1155 spin_lock(&un->ulp->lock); in freeary()
1156 un->semid = -1; in freeary()
1157 list_del_rcu(&un->list_proc); in freeary()
1158 spin_unlock(&un->ulp->lock); in freeary()
1163 list_for_each_entry_safe(q, tq, &sma->pending_const, list) { in freeary()
1165 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); in freeary()
1168 list_for_each_entry_safe(q, tq, &sma->pending_alter, list) { in freeary()
1170 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); in freeary()
1172 for (i = 0; i < sma->sem_nsems; i++) { in freeary()
1173 struct sem *sem = &sma->sems[i]; in freeary()
1174 list_for_each_entry_safe(q, tq, &sem->pending_const, list) { in freeary()
1176 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); in freeary()
1178 list_for_each_entry_safe(q, tq, &sem->pending_alter, list) { in freeary()
1180 wake_up_sem_queue_prepare(q, -EIDRM, &wake_q); in freeary()
1182 ipc_update_pid(&sem->sempid, NULL); in freeary()
1186 sem_rmid(ns, sma); in freeary()
1187 sem_unlock(sma, -1); in freeary()
1191 ns->used_sems -= sma->sem_nsems; in freeary()
1192 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in freeary()
1206 ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm); in copy_semid_to_user()
1208 out.sem_otime = in->sem_otime; in copy_semid_to_user()
1209 out.sem_ctime = in->sem_ctime; in copy_semid_to_user()
1210 out.sem_nsems = in->sem_nsems; in copy_semid_to_user()
1215 return -EINVAL; in copy_semid_to_user()
1224 res = sma->sems[0].sem_otime; in get_semotime()
1225 for (i = 1; i < sma->sem_nsems; i++) { in get_semotime()
1226 time64_t to = sma->sems[i].sem_otime; in get_semotime()
1234 static int semctl_stat(struct ipc_namespace *ns, int semid, in semctl_stat() argument
1245 sma = sem_obtain_object(ns, semid); in semctl_stat()
1251 sma = sem_obtain_object_check(ns, semid); in semctl_stat()
1260 audit_ipc_obj(&sma->sem_perm); in semctl_stat()
1262 err = -EACCES; in semctl_stat()
1263 if (ipcperms(ns, &sma->sem_perm, S_IRUGO)) in semctl_stat()
1267 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_stat()
1271 ipc_lock_object(&sma->sem_perm); in semctl_stat()
1273 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_stat()
1274 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1275 err = -EIDRM; in semctl_stat()
1279 kernel_to_ipc64_perm(&sma->sem_perm, &semid64->sem_perm); in semctl_stat()
1281 semid64->sem_otime = semotime; in semctl_stat()
1282 semid64->sem_ctime = sma->sem_ctime; in semctl_stat()
1284 semid64->sem_otime_high = semotime >> 32; in semctl_stat()
1285 semid64->sem_ctime_high = sma->sem_ctime >> 32; in semctl_stat()
1287 semid64->sem_nsems = sma->sem_nsems; in semctl_stat()
1292 * Return 0 on success in semctl_stat()
1300 err = sma->sem_perm.id; in semctl_stat()
1302 ipc_unlock_object(&sma->sem_perm); in semctl_stat()
1308 static int semctl_info(struct ipc_namespace *ns, int semid, in semctl_info() argument
1320 seminfo.semmni = ns->sc_semmni; in semctl_info()
1321 seminfo.semmns = ns->sc_semmns; in semctl_info()
1322 seminfo.semmsl = ns->sc_semmsl; in semctl_info()
1323 seminfo.semopm = ns->sc_semopm; in semctl_info()
1328 down_read(&sem_ids(ns).rwsem); in semctl_info()
1330 seminfo.semusz = sem_ids(ns).in_use; in semctl_info()
1331 seminfo.semaem = ns->used_sems; in semctl_info()
1336 max_idx = ipc_get_maxidx(&sem_ids(ns)); in semctl_info()
1337 up_read(&sem_ids(ns).rwsem); in semctl_info()
1339 return -EFAULT; in semctl_info()
1343 static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum, in semctl_setval() argument
1353 return -ERANGE; in semctl_setval()
1356 sma = sem_obtain_object_check(ns, semid); in semctl_setval()
1362 if (semnum < 0 || semnum >= sma->sem_nsems) { in semctl_setval()
1364 return -EINVAL; in semctl_setval()
1368 if (ipcperms(ns, &sma->sem_perm, S_IWUGO)) { in semctl_setval()
1370 return -EACCES; in semctl_setval()
1373 err = security_sem_semctl(&sma->sem_perm, SETVAL); in semctl_setval()
1376 return -EACCES; in semctl_setval()
1379 sem_lock(sma, NULL, -1); in semctl_setval()
1381 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_setval()
1382 sem_unlock(sma, -1); in semctl_setval()
1384 return -EIDRM; in semctl_setval()
1387 semnum = array_index_nospec(semnum, sma->sem_nsems); in semctl_setval()
1388 curr = &sma->sems[semnum]; in semctl_setval()
1390 ipc_assert_locked_object(&sma->sem_perm); in semctl_setval()
1391 list_for_each_entry(un, &sma->list_id, list_id) in semctl_setval()
1392 un->semadj[semnum] = 0; in semctl_setval()
1394 curr->semval = val; in semctl_setval()
1395 ipc_update_pid(&curr->sempid, task_tgid(current)); in semctl_setval()
1396 sma->sem_ctime = ktime_get_real_seconds(); in semctl_setval()
1397 /* maybe some queued-up processes were waiting for this */ in semctl_setval()
1399 sem_unlock(sma, -1); in semctl_setval()
1405 static int semctl_main(struct ipc_namespace *ns, int semid, int semnum, in semctl_main() argument
1416 sma = sem_obtain_object_check(ns, semid); in semctl_main()
1422 nsems = sma->sem_nsems; in semctl_main()
1424 err = -EACCES; in semctl_main()
1425 if (ipcperms(ns, &sma->sem_perm, cmd == SETALL ? S_IWUGO : S_IRUGO)) in semctl_main()
1428 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_main()
1438 sem_lock(sma, NULL, -1); in semctl_main()
1439 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1440 err = -EIDRM; in semctl_main()
1444 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1445 err = -EIDRM; in semctl_main()
1448 sem_unlock(sma, -1); in semctl_main()
1453 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1454 return -ENOMEM; in semctl_main()
1459 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1460 err = -EIDRM; in semctl_main()
1464 for (i = 0; i < sma->sem_nsems; i++) in semctl_main()
1465 sem_io[i] = sma->sems[i].semval; in semctl_main()
1466 sem_unlock(sma, -1); in semctl_main()
1470 err = -EFAULT; in semctl_main()
1478 if (!ipc_rcu_getref(&sma->sem_perm)) { in semctl_main()
1479 err = -EIDRM; in semctl_main()
1488 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1489 return -ENOMEM; in semctl_main()
1494 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1495 err = -EFAULT; in semctl_main()
1501 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in semctl_main()
1502 err = -ERANGE; in semctl_main()
1508 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1509 err = -EIDRM; in semctl_main()
1514 sma->sems[i].semval = sem_io[i]; in semctl_main()
1515 ipc_update_pid(&sma->sems[i].sempid, task_tgid(current)); in semctl_main()
1518 ipc_assert_locked_object(&sma->sem_perm); in semctl_main()
1519 list_for_each_entry(un, &sma->list_id, list_id) { in semctl_main()
1521 un->semadj[i] = 0; in semctl_main()
1523 sma->sem_ctime = ktime_get_real_seconds(); in semctl_main()
1524 /* maybe some queued-up processes were waiting for this */ in semctl_main()
1529 /* GETVAL, GETPID, GETNCTN, GETZCNT: fall-through */ in semctl_main()
1531 err = -EINVAL; in semctl_main()
1535 sem_lock(sma, NULL, -1); in semctl_main()
1536 if (!ipc_valid_object(&sma->sem_perm)) { in semctl_main()
1537 err = -EIDRM; in semctl_main()
1542 curr = &sma->sems[semnum]; in semctl_main()
1546 err = curr->semval; in semctl_main()
1549 err = pid_vnr(curr->sempid); in semctl_main()
1560 sem_unlock(sma, -1); in semctl_main()
1576 return -EFAULT; in copy_semid_from_user()
1583 return -EFAULT; in copy_semid_from_user()
1585 out->sem_perm.uid = tbuf_old.sem_perm.uid; in copy_semid_from_user()
1586 out->sem_perm.gid = tbuf_old.sem_perm.gid; in copy_semid_from_user()
1587 out->sem_perm.mode = tbuf_old.sem_perm.mode; in copy_semid_from_user()
1592 return -EINVAL; in copy_semid_from_user()
1601 static int semctl_down(struct ipc_namespace *ns, int semid, in semctl_down() argument
1608 down_write(&sem_ids(ns).rwsem); in semctl_down()
1611 ipcp = ipcctl_obtain_check(ns, &sem_ids(ns), semid, cmd, in semctl_down()
1612 &semid64->sem_perm, 0); in semctl_down()
1620 err = security_sem_semctl(&sma->sem_perm, cmd); in semctl_down()
1626 sem_lock(sma, NULL, -1); in semctl_down()
1628 freeary(ns, ipcp); in semctl_down()
1631 sem_lock(sma, NULL, -1); in semctl_down()
1632 err = ipc_update_perm(&semid64->sem_perm, ipcp); in semctl_down()
1635 sma->sem_ctime = ktime_get_real_seconds(); in semctl_down()
1638 err = -EINVAL; in semctl_down()
1643 sem_unlock(sma, -1); in semctl_down()
1647 up_write(&sem_ids(ns).rwsem); in semctl_down()
1653 struct ipc_namespace *ns; in ksys_semctl() local
1659 return -EINVAL; in ksys_semctl()
1661 ns = current->nsproxy->ipc_ns; in ksys_semctl()
1666 return semctl_info(ns, semid, cmd, p); in ksys_semctl()
1670 err = semctl_stat(ns, semid, cmd, &semid64); in ksys_semctl()
1674 err = -EFAULT; in ksys_semctl()
1682 return semctl_main(ns, semid, semnum, cmd, p); in ksys_semctl()
1686 /* big-endian 64bit */ in ksys_semctl()
1689 /* 32bit or little-endian 64bit */ in ksys_semctl()
1692 return semctl_setval(ns, semid, semnum, val); in ksys_semctl()
1696 return -EFAULT; in ksys_semctl()
1699 return semctl_down(ns, semid, cmd, &semid64); in ksys_semctl()
1701 return -EINVAL; in ksys_semctl()
1743 return get_compat_ipc64_perm(&out->sem_perm, &p->sem_perm); in copy_compat_semid_from_user()
1746 return get_compat_ipc_perm(&out->sem_perm, &p->sem_perm); in copy_compat_semid_from_user()
1756 to_compat_ipc64_perm(&v.sem_perm, &in->sem_perm); in copy_compat_semid_to_user()
1757 v.sem_otime = lower_32_bits(in->sem_otime); in copy_compat_semid_to_user()
1758 v.sem_otime_high = upper_32_bits(in->sem_otime); in copy_compat_semid_to_user()
1759 v.sem_ctime = lower_32_bits(in->sem_ctime); in copy_compat_semid_to_user()
1760 v.sem_ctime_high = upper_32_bits(in->sem_ctime); in copy_compat_semid_to_user()
1761 v.sem_nsems = in->sem_nsems; in copy_compat_semid_to_user()
1766 to_compat_ipc_perm(&v.sem_perm, &in->sem_perm); in copy_compat_semid_to_user()
1767 v.sem_otime = in->sem_otime; in copy_compat_semid_to_user()
1768 v.sem_ctime = in->sem_ctime; in copy_compat_semid_to_user()
1769 v.sem_nsems = in->sem_nsems; in copy_compat_semid_to_user()
1777 struct ipc_namespace *ns; in compat_ksys_semctl() local
1781 ns = current->nsproxy->ipc_ns; in compat_ksys_semctl()
1784 return -EINVAL; in compat_ksys_semctl()
1789 return semctl_info(ns, semid, cmd, p); in compat_ksys_semctl()
1793 err = semctl_stat(ns, semid, cmd, &semid64); in compat_ksys_semctl()
1797 err = -EFAULT; in compat_ksys_semctl()
1805 return semctl_main(ns, semid, semnum, cmd, p); in compat_ksys_semctl()
1807 return semctl_setval(ns, semid, semnum, arg); in compat_ksys_semctl()
1810 return -EFAULT; in compat_ksys_semctl()
1813 return semctl_down(ns, semid, cmd, &semid64); in compat_ksys_semctl()
1815 return -EINVAL; in compat_ksys_semctl()
1840 * here. We guarantee there is only one thread using this undo list,
1854 undo_list = current->sysvsem.undo_list; in get_undo_list()
1858 return -ENOMEM; in get_undo_list()
1859 spin_lock_init(&undo_list->lock); in get_undo_list()
1860 refcount_set(&undo_list->refcnt, 1); in get_undo_list()
1861 INIT_LIST_HEAD(&undo_list->list_proc); in get_undo_list()
1863 current->sysvsem.undo_list = undo_list; in get_undo_list()
1873 list_for_each_entry_rcu(un, &ulp->list_proc, list_proc, in __lookup_undo()
1874 spin_is_locked(&ulp->lock)) { in __lookup_undo()
1875 if (un->semid == semid) in __lookup_undo()
1885 assert_spin_locked(&ulp->lock); in lookup_undo()
1889 list_del_rcu(&un->list_proc); in lookup_undo()
1890 list_add_rcu(&un->list_proc, &ulp->list_proc); in lookup_undo()
1896 * find_alloc_undo - lookup (and if not present create) undo array
1897 * @ns: namespace
1901 * The size of the undo structure depends on the size of the semaphore
1903 * Lifetime-rules: sem_undo is rcu-protected, on success, the function
1906 static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid) in find_alloc_undo() argument
1918 spin_lock(&ulp->lock); in find_alloc_undo()
1920 spin_unlock(&ulp->lock); in find_alloc_undo()
1924 /* no undo structure around - allocate one. */ in find_alloc_undo()
1926 sma = sem_obtain_object_check(ns, semid); in find_alloc_undo()
1932 nsems = sma->sem_nsems; in find_alloc_undo()
1933 if (!ipc_rcu_getref(&sma->sem_perm)) { in find_alloc_undo()
1935 un = ERR_PTR(-EIDRM); in find_alloc_undo()
1943 ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); in find_alloc_undo()
1944 return ERR_PTR(-ENOMEM); in find_alloc_undo()
1947 /* step 3: Acquire the lock on semaphore array */ in find_alloc_undo()
1950 if (!ipc_valid_object(&sma->sem_perm)) { in find_alloc_undo()
1951 sem_unlock(sma, -1); in find_alloc_undo()
1954 un = ERR_PTR(-EIDRM); in find_alloc_undo()
1957 spin_lock(&ulp->lock); in find_alloc_undo()
1964 spin_unlock(&ulp->lock); in find_alloc_undo()
1969 new->ulp = ulp; in find_alloc_undo()
1970 new->semid = semid; in find_alloc_undo()
1971 assert_spin_locked(&ulp->lock); in find_alloc_undo()
1972 list_add_rcu(&new->list_proc, &ulp->list_proc); in find_alloc_undo()
1973 ipc_assert_locked_object(&sma->sem_perm); in find_alloc_undo()
1974 list_add(&new->list_id, &sma->list_id); in find_alloc_undo()
1976 spin_unlock(&ulp->lock); in find_alloc_undo()
1978 sem_unlock(sma, -1); in find_alloc_undo()
1985 struct ipc_namespace *ns) in __do_semtimedop() argument
1987 int error = -EINVAL; in __do_semtimedop()
1999 return -EINVAL; in __do_semtimedop()
2000 if (nsops > ns->sc_semopm) in __do_semtimedop()
2001 return -E2BIG; in __do_semtimedop()
2005 return -EINVAL; in __do_semtimedop()
2014 unsigned long mask = 1ULL << ((sop->sem_num) % BITS_PER_LONG); in __do_semtimedop()
2016 if (sop->sem_num >= max) in __do_semtimedop()
2017 max = sop->sem_num; in __do_semtimedop()
2018 if (sop->sem_flg & SEM_UNDO) in __do_semtimedop()
2029 if (sop->sem_op != 0) { in __do_semtimedop()
2036 /* On success, find_alloc_undo takes the rcu_read_lock */ in __do_semtimedop()
2037 un = find_alloc_undo(ns, semid); in __do_semtimedop()
2047 sma = sem_obtain_object_check(ns, semid); in __do_semtimedop()
2054 error = -EFBIG; in __do_semtimedop()
2055 if (max >= sma->sem_nsems) { in __do_semtimedop()
2060 error = -EACCES; in __do_semtimedop()
2061 if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO)) { in __do_semtimedop()
2066 error = security_sem_semop(&sma->sem_perm, sops, nsops, alter); in __do_semtimedop()
2072 error = -EIDRM; in __do_semtimedop()
2075 * We eventually might perform the following check in a lockless in __do_semtimedop()
2078 * only a per-semaphore lock is held and it's OK to proceed with the in __do_semtimedop()
2079 * check below. More details on the fine grained locking scheme in __do_semtimedop()
2080 * entangled here and why it's RMID race safe on comments at sem_lock() in __do_semtimedop()
2082 if (!ipc_valid_object(&sma->sem_perm)) in __do_semtimedop()
2085 * semid identifiers are not unique - find_alloc_undo may have in __do_semtimedop()
2088 * This case can be detected checking un->semid. The existence of in __do_semtimedop()
2091 if (un && un->semid == -1) in __do_semtimedop()
2102 if (error == 0) { /* non-blocking successful path */ in __do_semtimedop()
2120 if (error < 0) /* non-blocking error path */ in __do_semtimedop()
2124 * We need to sleep on this operation, so we put the current in __do_semtimedop()
2129 int idx = array_index_nospec(sops->sem_num, sma->sem_nsems); in __do_semtimedop()
2130 curr = &sma->sems[idx]; in __do_semtimedop()
2133 if (sma->complex_count) { in __do_semtimedop()
2135 &sma->pending_alter); in __do_semtimedop()
2139 &curr->pending_alter); in __do_semtimedop()
2142 list_add_tail(&queue.list, &curr->pending_const); in __do_semtimedop()
2145 if (!sma->complex_count) in __do_semtimedop()
2149 list_add_tail(&queue.list, &sma->pending_alter); in __do_semtimedop()
2151 list_add_tail(&queue.list, &sma->pending_const); in __do_semtimedop()
2153 sma->complex_count++; in __do_semtimedop()
2158 WRITE_ONCE(queue.status, -EINTR); in __do_semtimedop()
2167 current->timer_slack_ns, HRTIMER_MODE_ABS); in __do_semtimedop()
2172 * point; we're done. in __do_semtimedop()
2174 * We _do_ care, nonetheless, about being awoken by a signal or in __do_semtimedop()
2176 * slowpath (aka after taking sem_lock), such that we can detect in __do_semtimedop()
2177 * scenarios where we were awakened externally, during the in __do_semtimedop()
2182 if (error != -EINTR) { in __do_semtimedop()
2191 if (!ipc_valid_object(&sma->sem_perm)) in __do_semtimedop()
2195 * No necessity for any barrier: We are protect by sem_lock() in __do_semtimedop()
2200 * If queue.status != -EINTR we are woken up by another process. in __do_semtimedop()
2203 if (error != -EINTR) in __do_semtimedop()
2207 * If an interrupt occurred we have to clean up the queue. in __do_semtimedop()
2210 error = -EAGAIN; in __do_semtimedop()
2211 } while (error == -EINTR && !signal_pending(current)); /* spurious */ in __do_semtimedop()
2227 struct ipc_namespace *ns; in do_semtimedop() local
2230 ns = current->nsproxy->ipc_ns; in do_semtimedop()
2231 if (nsops > ns->sc_semopm) in do_semtimedop()
2232 return -E2BIG; in do_semtimedop()
2234 return -EINVAL; in do_semtimedop()
2239 return -ENOMEM; in do_semtimedop()
2243 ret = -EFAULT; in do_semtimedop()
2247 ret = __do_semtimedop(semid, sops, nsops, timeout, ns); in do_semtimedop()
2262 return -EFAULT; in ksys_semtimedop()
2282 return -EFAULT; in compat_ksys_semtimedop()
2315 refcount_inc(&undo_list->refcnt); in copy_semundo()
2316 tsk->sysvsem.undo_list = undo_list; in copy_semundo()
2318 tsk->sysvsem.undo_list = NULL; in copy_semundo()
2329 * manner or not. That is, if we are attempting to decrement the semval
2330 * should we queue up and wait until we can do so legally?
2339 ulp = tsk->sysvsem.undo_list; in exit_sem()
2342 tsk->sysvsem.undo_list = NULL; in exit_sem()
2344 if (!refcount_dec_and_test(&ulp->refcnt)) in exit_sem()
2356 un = list_entry_rcu(ulp->list_proc.next, in exit_sem()
2358 if (&un->list_proc == &ulp->list_proc) { in exit_sem()
2360 * We must wait for freeary() before freeing this ulp, in exit_sem()
2361 * in case we raced with last sem_undo. There is a small in exit_sem()
2362 * possibility where we exit while freeary() didn't in exit_sem()
2365 spin_lock(&ulp->lock); in exit_sem()
2366 spin_unlock(&ulp->lock); in exit_sem()
2370 spin_lock(&ulp->lock); in exit_sem()
2371 semid = un->semid; in exit_sem()
2372 spin_unlock(&ulp->lock); in exit_sem()
2375 if (semid == -1) { in exit_sem()
2380 sma = sem_obtain_object_check(tsk->nsproxy->ipc_ns, semid); in exit_sem()
2387 sem_lock(sma, NULL, -1); in exit_sem()
2389 if (!ipc_valid_object(&sma->sem_perm)) { in exit_sem()
2390 sem_unlock(sma, -1); in exit_sem()
2399 sem_unlock(sma, -1); in exit_sem()
2405 ipc_assert_locked_object(&sma->sem_perm); in exit_sem()
2406 list_del(&un->list_id); in exit_sem()
2408 spin_lock(&ulp->lock); in exit_sem()
2409 list_del_rcu(&un->list_proc); in exit_sem()
2410 spin_unlock(&ulp->lock); in exit_sem()
2413 for (i = 0; i < sma->sem_nsems; i++) { in exit_sem()
2414 struct sem *semaphore = &sma->sems[i]; in exit_sem()
2415 if (un->semadj[i]) { in exit_sem()
2416 semaphore->semval += un->semadj[i]; in exit_sem()
2420 * - Some unices ignore the undo entirely in exit_sem()
2422 * - some cap the value (e.g. FreeBSD caps in exit_sem()
2430 if (semaphore->semval < 0) in exit_sem()
2431 semaphore->semval = 0; in exit_sem()
2432 if (semaphore->semval > SEMVMX) in exit_sem()
2433 semaphore->semval = SEMVMX; in exit_sem()
2434 ipc_update_pid(&semaphore->sempid, task_tgid(current)); in exit_sem()
2437 /* maybe some queued-up processes were waiting for this */ in exit_sem()
2439 sem_unlock(sma, -1); in exit_sem()
2458 * ipc_lock_object(), i.e. spin_lock(&sma->sem_perm.lock). in sysvipc_sem_proc_show()
2460 * In order to stay compatible with sem_lock(), we must in sysvipc_sem_proc_show()
2469 sma->sem_perm.key, in sysvipc_sem_proc_show()
2470 sma->sem_perm.id, in sysvipc_sem_proc_show()
2471 sma->sem_perm.mode, in sysvipc_sem_proc_show()
2472 sma->sem_nsems, in sysvipc_sem_proc_show()
2473 from_kuid_munged(user_ns, sma->sem_perm.uid), in sysvipc_sem_proc_show()
2474 from_kgid_munged(user_ns, sma->sem_perm.gid), in sysvipc_sem_proc_show()
2475 from_kuid_munged(user_ns, sma->sem_perm.cuid), in sysvipc_sem_proc_show()
2476 from_kgid_munged(user_ns, sma->sem_perm.cgid), in sysvipc_sem_proc_show()
2478 sma->sem_ctime); in sysvipc_sem_proc_show()