Lines Matching +full:wait +full:- +full:monitoring +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0-only
9 * 'fork.c' contains the help-routines for the 'fork' system call
83 #include <linux/posix-timers.h>
84 #include <linux/user-return-notifier.h>
217 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) in thread_stack_free_rcu()
225 struct vm_stack *vm_stack = tsk->stack; in thread_stack_delayed_free()
227 vm_stack->stack_vm_area = tsk->stack_vm_area; in thread_stack_delayed_free()
228 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); in thread_stack_delayed_free()
242 vfree(vm_stack->addr); in free_vm_stack_cache()
255 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); in memcg_charge_kernel_stack()
258 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0); in memcg_charge_kernel_stack()
266 memcg_kmem_uncharge_page(vm->pages[i], 0); in memcg_charge_kernel_stack()
285 kasan_unpoison_range(s->addr, THREAD_SIZE); in alloc_thread_stack_node()
287 stack = kasan_reset_tag(s->addr); in alloc_thread_stack_node()
293 vfree(s->addr); in alloc_thread_stack_node()
294 return -ENOMEM; in alloc_thread_stack_node()
297 tsk->stack_vm_area = s; in alloc_thread_stack_node()
298 tsk->stack = stack; in alloc_thread_stack_node()
313 return -ENOMEM; in alloc_thread_stack_node()
318 return -ENOMEM; in alloc_thread_stack_node()
325 tsk->stack_vm_area = vm; in alloc_thread_stack_node()
327 tsk->stack = stack; in alloc_thread_stack_node()
333 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) in free_thread_stack()
336 tsk->stack = NULL; in free_thread_stack()
337 tsk->stack_vm_area = NULL; in free_thread_stack()
349 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
360 tsk->stack = kasan_reset_tag(page_address(page)); in alloc_thread_stack_node()
363 return -ENOMEM; in alloc_thread_stack_node()
369 tsk->stack = NULL; in free_thread_stack()
384 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
394 tsk->stack = stack; in alloc_thread_stack_node()
395 return stack ? 0 : -ENOMEM; in alloc_thread_stack_node()
401 tsk->stack = NULL; in free_thread_stack()
414 /* SLAB cache for signal_struct structures (tsk->signal) */
417 /* SLAB cache for sighand_struct structures (tsk->sighand) */
420 /* SLAB cache for files_struct structures (tsk->files) */
423 /* SLAB cache for fs_struct structures (tsk->fs) */
429 /* SLAB cache for mm_struct structures (tsk->mm) */
439 vma->vm_lock = kmem_cache_alloc(vma_lock_cachep, GFP_KERNEL); in vma_lock_alloc()
440 if (!vma->vm_lock) in vma_lock_alloc()
443 init_rwsem(&vma->vm_lock->lock); in vma_lock_alloc()
444 vma->vm_lock_seq = -1; in vma_lock_alloc()
451 kmem_cache_free(vma_lock_cachep, vma->vm_lock); in vma_lock_free()
485 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); in vm_area_dup()
486 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); in vm_area_dup()
488 * orig->shared.rb may be modified concurrently, but the clone in vm_area_dup()
496 INIT_LIST_HEAD(&new->anon_vma_chain); in vm_area_dup()
518 VM_BUG_ON_VMA(rwsem_is_locked(&vma->vm_lock->lock), vma); in vm_area_free_rcu_cb()
526 call_rcu(&vma->vm_rcu, vm_area_free_rcu_cb); in vm_area_free()
539 mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB, in account_kernel_stack()
552 account_kernel_stack(tsk, -1); in exit_task_stack_account()
560 memcg_kmem_uncharge_page(vm->pages[i], 0); in exit_task_stack_account()
566 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) in release_task_stack()
575 if (refcount_dec_and_test(&tsk->stack_refcount)) in put_task_stack()
583 WARN_ON_ONCE(tsk->seccomp.filter); in free_task()
599 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); in free_task()
604 if (tsk->flags & PF_KTHREAD) in free_task()
616 RCU_INIT_POINTER(mm->exe_file, exe_file); in dup_mm_exe_file()
637 retval = -EINTR; in dup_mmap()
643 * Not linked in yet - no deadlock potential: in dup_mmap()
650 mm->total_vm = oldmm->total_vm; in dup_mmap()
651 mm->data_vm = oldmm->data_vm; in dup_mmap()
652 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
653 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
661 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); in dup_mmap()
670 if (mpnt->vm_flags & VM_DONTCOPY) { in dup_mmap()
671 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, in dup_mmap()
672 mpnt->vm_end, GFP_KERNEL); in dup_mmap()
676 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); in dup_mmap()
681 * Don't duplicate many vmas if we've been oom-killed (for in dup_mmap()
685 retval = -EINTR; in dup_mmap()
688 if (mpnt->vm_flags & VM_ACCOUNT) { in dup_mmap()
701 tmp->vm_mm = mm; in dup_mmap()
705 if (tmp->vm_flags & VM_WIPEONFORK) { in dup_mmap()
711 tmp->anon_vma = NULL; in dup_mmap()
715 file = tmp->vm_file; in dup_mmap()
717 struct address_space *mapping = file->f_mapping; in dup_mmap()
726 &mapping->i_mmap); in dup_mmap()
743 mm->map_count++; in dup_mmap()
744 if (!(tmp->vm_flags & VM_WIPEONFORK)) in dup_mmap()
747 if (tmp->vm_ops && tmp->vm_ops->open) in dup_mmap()
748 tmp->vm_ops->open(tmp); in dup_mmap()
769 mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1); in dup_mmap()
786 retval = -ENOMEM; in dup_mmap()
793 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
794 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
795 return -ENOMEM; in mm_alloc_pgd()
801 pgd_free(mm, mm->pgd); in mm_free_pgd()
823 long x = percpu_counter_sum(&mm->rss_stat[i]); in check_mm()
826 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", in check_mm()
831 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", in check_mm()
835 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
846 WARN_ON_ONCE(current->active_mm == mm); in do_check_lazy_tlb()
853 if (current->active_mm == mm) { in do_shoot_lazy_tlb()
854 WARN_ON_ONCE(current->mm); in do_shoot_lazy_tlb()
855 current->active_mm = &init_mm; in do_shoot_lazy_tlb()
876 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm in cleanup_lazy_tlbs()
886 * - The last lazy reference created by exit_mm() could instead switch in cleanup_lazy_tlbs()
889 * - A batch of mms requiring IPIs could be gathered and freed at once. in cleanup_lazy_tlbs()
890 * - CPUs store active_mm where it can be remotely checked without a in cleanup_lazy_tlbs()
891 * lock, to filter out false-positives in the cpumask. in cleanup_lazy_tlbs()
892 * - After mm_users or mm_count reaches zero, switching away from the in cleanup_lazy_tlbs()
895 * - A delayed freeing and RCU-like quiescing sequence based on mm in cleanup_lazy_tlbs()
911 WARN_ON_ONCE(mm == current->mm); in __mmdrop()
916 WARN_ON_ONCE(mm == current->active_mm); in __mmdrop()
921 put_user_ns(mm->user_ns); in __mmdrop()
924 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); in __mmdrop()
940 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { in mmdrop_async()
941 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); in mmdrop_async()
942 schedule_work(&mm->async_put_work); in mmdrop_async()
954 if (sig->oom_mm) in free_signal_struct()
955 mmdrop_async(sig->oom_mm); in free_signal_struct()
961 if (refcount_dec_and_test(&sig->sigcnt)) in put_signal_struct()
967 WARN_ON(!tsk->exit_state); in __put_task_struct()
968 WARN_ON(refcount_read(&tsk->usage)); in __put_task_struct()
977 put_signal_struct(tsk->signal); in __put_task_struct()
1028 * Handle zero-sized whitelist or empty thread_struct, otherwise in task_struct_whitelist()
1058 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; in fork_init()
1059 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; in fork_init()
1060 init_task.signal->rlim[RLIMIT_SIGPENDING] = in fork_init()
1061 init_task.signal->rlim[RLIMIT_NPROC]; in fork_init()
1117 refcount_set(&tsk->stack_refcount, 1); in dup_task_struct()
1132 tsk->seccomp.filter = NULL; in dup_task_struct()
1142 tsk->stack_canary = get_random_canary(); in dup_task_struct()
1144 if (orig->cpus_ptr == &orig->cpus_mask) in dup_task_struct()
1145 tsk->cpus_ptr = &tsk->cpus_mask; in dup_task_struct()
1152 refcount_set(&tsk->rcu_users, 2); in dup_task_struct()
1154 refcount_set(&tsk->usage, 1); in dup_task_struct()
1156 tsk->btrace_seq = 0; in dup_task_struct()
1158 tsk->splice_pipe = NULL; in dup_task_struct()
1159 tsk->task_frag.page = NULL; in dup_task_struct()
1160 tsk->wake_q.next = NULL; in dup_task_struct()
1161 tsk->worker_private = NULL; in dup_task_struct()
1168 tsk->fail_nth = 0; in dup_task_struct()
1172 tsk->throttle_disk = NULL; in dup_task_struct()
1173 tsk->use_memdelay = 0; in dup_task_struct()
1177 tsk->pasid_activated = 0; in dup_task_struct()
1181 tsk->active_memcg = NULL; in dup_task_struct()
1185 tsk->reported_split_lock = 0; in dup_task_struct()
1189 tsk->mm_cid = -1; in dup_task_struct()
1190 tsk->last_mm_cid = -1; in dup_task_struct()
1191 tsk->mm_cid_active = 0; in dup_task_struct()
1192 tsk->migrate_from_cpu = -1; in dup_task_struct()
1223 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
1224 mm->ioctx_table = NULL; in mm_init_aio()
1232 if (mm->owner == p) in mm_clear_owner()
1233 WRITE_ONCE(mm->owner, NULL); in mm_clear_owner()
1240 mm->owner = p; in mm_init_owner()
1247 mm->uprobes_state.xol_area = NULL; in mm_init_uprobes_state()
1254 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); in mm_init()
1255 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); in mm_init()
1256 atomic_set(&mm->mm_users, 1); in mm_init()
1257 atomic_set(&mm->mm_count, 1); in mm_init()
1258 seqcount_init(&mm->write_protect_seq); in mm_init()
1260 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
1262 mm->mm_lock_seq = 0; in mm_init()
1265 mm->map_count = 0; in mm_init()
1266 mm->locked_vm = 0; in mm_init()
1267 atomic64_set(&mm->pinned_vm, 0); in mm_init()
1268 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
1269 spin_lock_init(&mm->page_table_lock); in mm_init()
1270 spin_lock_init(&mm->arg_lock); in mm_init()
1275 RCU_INIT_POINTER(mm->exe_file, NULL); in mm_init()
1279 mm->pmd_huge_pte = NULL; in mm_init()
1284 if (current->mm) { in mm_init()
1285 mm->flags = mmf_init_flags(current->mm->flags); in mm_init()
1286 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
1288 mm->flags = default_dump_filter; in mm_init()
1289 mm->def_flags = 0; in mm_init()
1301 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, in mm_init()
1305 mm->user_ns = get_user_ns(user_ns); in mm_init()
1337 VM_BUG_ON(atomic_read(&mm->mm_users)); in __mmput()
1346 if (!list_empty(&mm->mmlist)) { in __mmput()
1348 list_del(&mm->mmlist); in __mmput()
1351 if (mm->binfmt) in __mmput()
1352 module_put(mm->binfmt->module); in __mmput()
1364 if (atomic_dec_and_test(&mm->mm_users)) in mmput()
1380 if (atomic_dec_and_test(&mm->mm_users)) { in mmput_async()
1381 INIT_WORK(&mm->async_put_work, mmput_async_fn); in mmput_async()
1382 schedule_work(&mm->async_put_work); in mmput_async()
1389 * set_mm_exe_file - change a reference to the mm's executable file
1408 * this mm -- see comment above for justification. in set_mm_exe_file()
1410 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
1418 return -EACCES; in set_mm_exe_file()
1421 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
1430 * replace_mm_exe_file - replace a reference to the mm's executable file
1444 /* Forbid mm->exe_file change if old file still mapped. */ in replace_mm_exe_file()
1450 if (!vma->vm_file) in replace_mm_exe_file()
1452 if (path_equal(&vma->vm_file->f_path, in replace_mm_exe_file()
1453 &old_exe_file->f_path)) { in replace_mm_exe_file()
1454 ret = -EBUSY; in replace_mm_exe_file()
1466 return -EACCES; in replace_mm_exe_file()
1471 old_exe_file = rcu_dereference_raw(mm->exe_file); in replace_mm_exe_file()
1472 rcu_assign_pointer(mm->exe_file, new_exe_file); in replace_mm_exe_file()
1483 * get_mm_exe_file - acquire a reference to the mm's executable file
1494 exe_file = get_file_rcu(&mm->exe_file); in get_mm_exe_file()
1500 * get_task_exe_file - acquire a reference to the task's executable file
1513 mm = task->mm; in get_task_exe_file()
1515 if (!(task->flags & PF_KTHREAD)) in get_task_exe_file()
1523 * get_task_mm - acquire a reference to the task's mm
1537 mm = task->mm; in get_task_mm()
1539 if (task->flags & PF_KTHREAD) in get_task_mm()
1554 err = down_read_killable(&task->signal->exec_update_lock); in mm_access()
1559 if (mm && mm != current->mm && in mm_access()
1562 mm = ERR_PTR(-EACCES); in mm_access()
1564 up_read(&task->signal->exec_update_lock); in mm_access()
1574 vfork = tsk->vfork_done; in complete_vfork_done()
1576 tsk->vfork_done = NULL; in complete_vfork_done()
1594 child->vfork_done = NULL; in wait_for_vfork_done()
1627 if (tsk->clear_child_tid) { in mm_release()
1628 if (atomic_read(&mm->mm_users) > 1) { in mm_release()
1630 * We don't check the error code - if userspace has in mm_release()
1633 put_user(0, tsk->clear_child_tid); in mm_release()
1634 do_futex(tsk->clear_child_tid, FUTEX_WAKE, in mm_release()
1637 tsk->clear_child_tid = NULL; in mm_release()
1644 if (tsk->vfork_done) in mm_release()
1661 * dup_mm() - duplicates an existing mm structure
1682 if (!mm_init(mm, tsk, mm->user_ns)) in dup_mm()
1689 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
1690 mm->hiwater_vm = mm->total_vm; in dup_mm()
1692 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
1699 mm->binfmt = NULL; in dup_mm()
1711 tsk->min_flt = tsk->maj_flt = 0; in copy_mm()
1712 tsk->nvcsw = tsk->nivcsw = 0; in copy_mm()
1714 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; in copy_mm()
1715 tsk->last_switch_time = 0; in copy_mm()
1718 tsk->mm = NULL; in copy_mm()
1719 tsk->active_mm = NULL; in copy_mm()
1726 oldmm = current->mm; in copy_mm()
1734 mm = dup_mm(tsk, current->mm); in copy_mm()
1736 return -ENOMEM; in copy_mm()
1739 tsk->mm = mm; in copy_mm()
1740 tsk->active_mm = mm; in copy_mm()
1747 struct fs_struct *fs = current->fs; in copy_fs()
1749 /* tsk->fs is already what we want */ in copy_fs()
1750 spin_lock(&fs->lock); in copy_fs()
1752 if (fs->in_exec) { in copy_fs()
1753 spin_unlock(&fs->lock); in copy_fs()
1754 return -EAGAIN; in copy_fs()
1756 fs->users++; in copy_fs()
1757 spin_unlock(&fs->lock); in copy_fs()
1760 tsk->fs = copy_fs_struct(fs); in copy_fs()
1761 if (!tsk->fs) in copy_fs()
1762 return -ENOMEM; in copy_fs()
1775 oldf = current->files; in copy_files()
1780 tsk->files = NULL; in copy_files()
1785 atomic_inc(&oldf->count); in copy_files()
1793 tsk->files = newf; in copy_files()
1804 refcount_inc(¤t->sighand->count); in copy_sighand()
1808 RCU_INIT_POINTER(tsk->sighand, sig); in copy_sighand()
1810 return -ENOMEM; in copy_sighand()
1812 refcount_set(&sig->count, 1); in copy_sighand()
1813 spin_lock_irq(¤t->sighand->siglock); in copy_sighand()
1814 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); in copy_sighand()
1815 spin_unlock_irq(¤t->sighand->siglock); in copy_sighand()
1826 if (refcount_dec_and_test(&sighand->count)) { in __cleanup_sighand()
1841 struct posix_cputimers *pct = &sig->posix_cputimers; in posix_cpu_timers_init_group()
1844 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); in posix_cpu_timers_init_group()
1856 tsk->signal = sig; in copy_signal()
1858 return -ENOMEM; in copy_signal()
1860 sig->nr_threads = 1; in copy_signal()
1861 sig->quick_threads = 1; in copy_signal()
1862 atomic_set(&sig->live, 1); in copy_signal()
1863 refcount_set(&sig->sigcnt, 1); in copy_signal()
1866 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); in copy_signal()
1867 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); in copy_signal()
1869 init_waitqueue_head(&sig->wait_chldexit); in copy_signal()
1870 sig->curr_target = tsk; in copy_signal()
1871 init_sigpending(&sig->shared_pending); in copy_signal()
1872 INIT_HLIST_HEAD(&sig->multiprocess); in copy_signal()
1873 seqlock_init(&sig->stats_lock); in copy_signal()
1874 prev_cputime_init(&sig->prev_cputime); in copy_signal()
1877 INIT_LIST_HEAD(&sig->posix_timers); in copy_signal()
1878 hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in copy_signal()
1879 sig->real_timer.function = it_real_fn; in copy_signal()
1882 task_lock(current->group_leader); in copy_signal()
1883 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); in copy_signal()
1884 task_unlock(current->group_leader); in copy_signal()
1891 sig->oom_score_adj = current->signal->oom_score_adj; in copy_signal()
1892 sig->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_signal()
1894 mutex_init(&sig->cred_guard_mutex); in copy_signal()
1895 init_rwsem(&sig->exec_update_lock); in copy_signal()
1904 * Must be called with sighand->lock held, which is common to in copy_seccomp()
1909 assert_spin_locked(¤t->sighand->siglock); in copy_seccomp()
1911 /* Ref-count the new filter user, and assign it. */ in copy_seccomp()
1913 p->seccomp = current->seccomp; in copy_seccomp()
1928 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) in copy_seccomp()
1935 current->clear_child_tid = tidptr; in SYSCALL_DEFINE1()
1942 raw_spin_lock_init(&p->pi_lock); in rt_mutex_init_task()
1944 p->pi_waiters = RB_ROOT_CACHED; in rt_mutex_init_task()
1945 p->pi_top_task = NULL; in rt_mutex_init_task()
1946 p->pi_blocked_on = NULL; in rt_mutex_init_task()
1955 INIT_HLIST_NODE(&task->pid_links[type]); in init_task_pid_links()
1962 task->thread_pid = pid; in init_task_pid()
1964 task->signal->pids[type] = pid; in init_task_pid()
1970 p->rcu_read_lock_nesting = 0; in rcu_copy_process()
1971 p->rcu_read_unlock_special.s = 0; in rcu_copy_process()
1972 p->rcu_blocked_node = NULL; in rcu_copy_process()
1973 INIT_LIST_HEAD(&p->rcu_node_entry); in rcu_copy_process()
1976 p->rcu_tasks_holdout = false; in rcu_copy_process()
1977 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); in rcu_copy_process()
1978 p->rcu_tasks_idle_cpu = -1; in rcu_copy_process()
1981 p->trc_reader_nesting = 0; in rcu_copy_process()
1982 p->trc_reader_special.s = 0; in rcu_copy_process()
1983 INIT_LIST_HEAD(&p->trc_holdout_list); in rcu_copy_process()
1984 INIT_LIST_HEAD(&p->trc_blkd_node); in rcu_copy_process()
1990 if (file->f_op == &pidfd_fops) in pidfd_pid()
1991 return file->private_data; in pidfd_pid()
1993 return ERR_PTR(-EBADF); in pidfd_pid()
1998 struct pid *pid = file->private_data; in pidfd_release()
2000 file->private_data = NULL; in pidfd_release()
2007 * pidfd_show_fdinfo - print information about a pidfd
2033 * - create two new pid namespaces ns1 and ns2 in the initial pid
2036 * - create a process with a pidfd in ns1
2037 * - send pidfd from ns1 to ns2
2038 * - read /proc/self/fdinfo/<pidfd> and observe that both Pid and NSpid
2043 struct pid *pid = f->private_data; in pidfd_show_fdinfo()
2044 struct pid_namespace *ns; in pidfd_show_fdinfo() local
2045 pid_t nr = -1; in pidfd_show_fdinfo()
2048 ns = proc_pid_ns(file_inode(m->file)->i_sb); in pidfd_show_fdinfo()
2049 nr = pid_nr_ns(pid, ns); in pidfd_show_fdinfo()
2059 /* If nr is non-zero it means that 'pid' is valid and that in pidfd_show_fdinfo()
2060 * ns, i.e. the pid namespace associated with the procfs in pidfd_show_fdinfo()
2064 for (i = ns->level + 1; i <= pid->level; i++) in pidfd_show_fdinfo()
2065 seq_put_decimal_ll(m, "\t", pid->numbers[i].nr); in pidfd_show_fdinfo()
2077 struct pid *pid = file->private_data; in pidfd_poll()
2080 poll_wait(file, &pid->wait_pidfd, pts); in pidfd_poll()
2085 * group, then poll(2) should block, similar to the wait(2) family. in pidfd_poll()
2102 * __pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2135 return -EINVAL; in __pidfd_prepare()
2153 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2181 return -EINVAL; in pidfd_prepare()
2196 call_rcu(&tsk->rcu, __delayed_free_task); in delayed_free_task()
2204 if (!tsk->mm) in copy_oom_score_adj()
2213 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); in copy_oom_score_adj()
2215 tsk->signal->oom_score_adj = current->signal->oom_score_adj; in copy_oom_score_adj()
2216 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_oom_score_adj()
2226 p->rv[i].da_mon.monitoring = false; in rv_task_fork()
2238 * flags). The actual kick-off is left to the caller.
2246 int pidfd = -1, retval; in copy_process()
2250 const u64 clone_flags = args->flags; in copy_process()
2251 struct nsproxy *nsp = current->nsproxy; in copy_process()
2258 return ERR_PTR(-EINVAL); in copy_process()
2261 return ERR_PTR(-EINVAL); in copy_process()
2268 return ERR_PTR(-EINVAL); in copy_process()
2276 return ERR_PTR(-EINVAL); in copy_process()
2281 * multi-rooted process trees, prevent global and container-inits in copy_process()
2285 current->signal->flags & SIGNAL_UNKILLABLE) in copy_process()
2286 return ERR_PTR(-EINVAL); in copy_process()
2294 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) in copy_process()
2295 return ERR_PTR(-EINVAL); in copy_process()
2300 * - CLONE_DETACHED is blocked so that we can potentially in copy_process()
2302 * - CLONE_THREAD is blocked until someone really needs it. in copy_process()
2305 return ERR_PTR(-EINVAL); in copy_process()
2317 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2319 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); in copy_process()
2321 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2322 retval = -ERESTARTNOINTR; in copy_process()
2326 retval = -ENOMEM; in copy_process()
2330 p->flags &= ~PF_KTHREAD; in copy_process()
2331 if (args->kthread) in copy_process()
2332 p->flags |= PF_KTHREAD; in copy_process()
2333 if (args->user_worker) { in copy_process()
2338 p->flags |= PF_USER_WORKER; in copy_process()
2339 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); in copy_process()
2341 if (args->io_thread) in copy_process()
2342 p->flags |= PF_IO_WORKER; in copy_process()
2344 if (args->name) in copy_process()
2345 strscpy_pad(p->comm, args->name, sizeof(p->comm)); in copy_process()
2347 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; in copy_process()
2351 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; in copy_process()
2359 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); in copy_process()
2365 retval = -EAGAIN; in copy_process()
2367 if (p->real_cred->user != INIT_USER && in copy_process()
2371 current->flags &= ~PF_NPROC_EXCEEDED; in copy_process()
2378 retval = -EAGAIN; in copy_process()
2383 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); in copy_process()
2384 p->flags |= PF_FORKNOEXEC; in copy_process()
2385 INIT_LIST_HEAD(&p->children); in copy_process()
2386 INIT_LIST_HEAD(&p->sibling); in copy_process()
2388 p->vfork_done = NULL; in copy_process()
2389 spin_lock_init(&p->alloc_lock); in copy_process()
2391 init_sigpending(&p->pending); in copy_process()
2393 p->utime = p->stime = p->gtime = 0; in copy_process()
2395 p->utimescaled = p->stimescaled = 0; in copy_process()
2397 prev_cputime_init(&p->prev_cputime); in copy_process()
2400 seqcount_init(&p->vtime.seqcount); in copy_process()
2401 p->vtime.starttime = 0; in copy_process()
2402 p->vtime.state = VTIME_INACTIVE; in copy_process()
2406 p->io_uring = NULL; in copy_process()
2409 p->default_timer_slack_ns = current->timer_slack_ns; in copy_process()
2412 p->psi_flags = 0; in copy_process()
2415 task_io_accounting_init(&p->ioac); in copy_process()
2418 posix_cputimers_init(&p->posix_cputimers); in copy_process()
2420 p->io_context = NULL; in copy_process()
2423 if (args->kthread) { in copy_process()
2428 p->mempolicy = mpol_dup(p->mempolicy); in copy_process()
2429 if (IS_ERR(p->mempolicy)) { in copy_process()
2430 retval = PTR_ERR(p->mempolicy); in copy_process()
2431 p->mempolicy = NULL; in copy_process()
2436 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; in copy_process()
2437 p->cpuset_slab_spread_rotor = NUMA_NO_NODE; in copy_process()
2438 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); in copy_process()
2441 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); in copy_process()
2442 p->irqtrace.hardirq_disable_ip = _THIS_IP_; in copy_process()
2443 p->irqtrace.softirq_enable_ip = _THIS_IP_; in copy_process()
2444 p->softirqs_enabled = 1; in copy_process()
2445 p->softirq_context = 0; in copy_process()
2448 p->pagefault_disabled = 0; in copy_process()
2455 p->blocked_on = NULL; /* not blocked yet */ in copy_process()
2458 p->sequential_io = 0; in copy_process()
2459 p->sequential_io_avg = 0; in copy_process()
2462 RCU_INIT_POINTER(p->bpf_storage, NULL); in copy_process()
2463 p->bpf_ctx = NULL; in copy_process()
2485 retval = copy_files(clone_flags, p, args->no_files); in copy_process()
2513 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, in copy_process()
2514 args->set_tid_size); in copy_process()
2533 retval = put_user(pidfd, args->pidfd); in copy_process()
2539 p->plug = NULL; in copy_process()
2561 p->pid = pid_nr(pid); in copy_process()
2563 p->group_leader = current->group_leader; in copy_process()
2564 p->tgid = current->tgid; in copy_process()
2566 p->group_leader = p; in copy_process()
2567 p->tgid = p->pid; in copy_process()
2570 p->nr_dirtied = 0; in copy_process()
2571 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); in copy_process()
2572 p->dirty_paused_when = 0; in copy_process()
2574 p->pdeath_signal = 0; in copy_process()
2575 p->task_works = NULL; in copy_process()
2579 p->kretprobe_instances.first = NULL; in copy_process()
2582 p->rethooks.first = NULL; in copy_process()
2596 * Now that the cgroups are pinned, re-clone the parent cgroup and put in copy_process()
2600 * This isn't part of ->can_fork() because while the re-cloning is in copy_process()
2607 * From this point on we must avoid any synchronous user-space in copy_process()
2608 * communication until we take the tasklist-lock. In particular, we do in copy_process()
2609 * not want user-space to be able to predict the process start-time by in copy_process()
2614 p->start_time = ktime_get_ns(); in copy_process()
2615 p->start_boottime = ktime_get_boottime_ns(); in copy_process()
2623 /* CLONE_PARENT re-uses the old parent */ in copy_process()
2625 p->real_parent = current->real_parent; in copy_process()
2626 p->parent_exec_id = current->parent_exec_id; in copy_process()
2628 p->exit_signal = -1; in copy_process()
2630 p->exit_signal = current->group_leader->exit_signal; in copy_process()
2632 p->real_parent = current; in copy_process()
2633 p->parent_exec_id = current->self_exec_id; in copy_process()
2634 p->exit_signal = args->exit_signal; in copy_process()
2641 spin_lock(¤t->sighand->siglock); in copy_process()
2648 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { in copy_process()
2649 retval = -ENOMEM; in copy_process()
2655 retval = -EINTR; in copy_process()
2668 if (likely(p->pid)) { in copy_process()
2678 ns_of_pid(pid)->child_reaper = p; in copy_process()
2679 p->signal->flags |= SIGNAL_UNKILLABLE; in copy_process()
2681 p->signal->shared_pending.signal = delayed.signal; in copy_process()
2682 p->signal->tty = tty_kref_get(current->signal->tty); in copy_process()
2688 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || in copy_process()
2689 p->real_parent->signal->is_child_subreaper; in copy_process()
2690 list_add_tail(&p->sibling, &p->real_parent->children); in copy_process()
2691 list_add_tail_rcu(&p->tasks, &init_task.tasks); in copy_process()
2697 current->signal->nr_threads++; in copy_process()
2698 current->signal->quick_threads++; in copy_process()
2699 atomic_inc(¤t->signal->live); in copy_process()
2700 refcount_inc(¤t->signal->sigcnt); in copy_process()
2702 list_add_tail_rcu(&p->thread_node, in copy_process()
2703 &p->signal->thread_head); in copy_process()
2710 spin_unlock(¤t->sighand->siglock); in copy_process()
2732 spin_unlock(¤t->sighand->siglock); in copy_process()
2746 if (p->io_context) in copy_process()
2751 if (p->mm) { in copy_process()
2752 mm_clear_owner(p->mm, p); in copy_process()
2753 mmput(p->mm); in copy_process()
2757 free_signal_struct(p->signal); in copy_process()
2759 __cleanup_sighand(p->sighand); in copy_process()
2775 mpol_put(p->mempolicy); in copy_process()
2783 WRITE_ONCE(p->__state, TASK_DEAD); in copy_process()
2788 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2790 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2799 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ in init_idle_pids()
2854 * Ok, this is the main fork-routine.
2856 * It copies the process, and if successful kick-starts
2859 * args->exit_signal is expected to be checked for sanity by the caller.
2863 u64 clone_flags = args->flags; in kernel_clone()
2879 if ((args->flags & CLONE_PIDFD) && in kernel_clone()
2880 (args->flags & CLONE_PARENT_SETTID) && in kernel_clone()
2881 (args->pidfd == args->parent_tid)) in kernel_clone()
2882 return -EINVAL; in kernel_clone()
2893 else if (args->exit_signal != SIGCHLD) in kernel_clone()
2909 * Do this prior waking up the new thread - the thread pointer in kernel_clone()
2918 put_user(nr, args->parent_tid); in kernel_clone()
2921 p->vfork_done = &vfork; in kernel_clone()
2929 lru_gen_add_mm(p->mm); in kernel_clone()
2994 return -EINVAL; in SYSCALL_DEFINE0()
3057 pid_t *kset_tid = kargs->set_tid; in copy_clone_args_from_user()
3068 return -E2BIG; in copy_clone_args_from_user()
3070 return -EINVAL; in copy_clone_args_from_user()
3077 return -EINVAL; in copy_clone_args_from_user()
3080 return -EINVAL; in copy_clone_args_from_user()
3083 return -EINVAL; in copy_clone_args_from_user()
3091 return -EINVAL; in copy_clone_args_from_user()
3095 return -EINVAL; in copy_clone_args_from_user()
3112 (kargs->set_tid_size * sizeof(pid_t)))) in copy_clone_args_from_user()
3113 return -EFAULT; in copy_clone_args_from_user()
3115 kargs->set_tid = kset_tid; in copy_clone_args_from_user()
3121 * clone3_stack_valid - check and prepare stack
3130 if (kargs->stack == 0) { in clone3_stack_valid()
3131 if (kargs->stack_size > 0) in clone3_stack_valid()
3134 if (kargs->stack_size == 0) in clone3_stack_valid()
3137 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) in clone3_stack_valid()
3141 kargs->stack += kargs->stack_size; in clone3_stack_valid()
3151 if (kargs->flags & in clone3_args_valid()
3156 * - make the CLONE_DETACHED bit reusable for clone3 in clone3_args_valid()
3157 * - make the CSIGNAL bits reusable for clone3 in clone3_args_valid()
3159 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) in clone3_args_valid()
3162 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == in clone3_args_valid()
3166 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && in clone3_args_valid()
3167 kargs->exit_signal) in clone3_args_valid()
3177 * sys_clone3 - create a new process with specific properties
3201 return -EINVAL; in SYSCALL_DEFINE2()
3213 leader = top = top->group_leader; in walk_process_tree()
3216 list_for_each_entry(child, &parent->children, sibling) { in walk_process_tree()
3231 parent = child->real_parent; in walk_process_tree()
3232 leader = parent->group_leader; in walk_process_tree()
3247 spin_lock_init(&sighand->siglock); in sighand_ctor()
3248 init_waitqueue_head(&sighand->signalfd_wqh); in sighand_ctor()
3307 return -EINVAL; in check_unshare_flags()
3316 return -EINVAL; in check_unshare_flags()
3319 if (refcount_read(¤t->sighand->count) > 1) in check_unshare_flags()
3320 return -EINVAL; in check_unshare_flags()
3324 return -EINVAL; in check_unshare_flags()
3335 struct fs_struct *fs = current->fs; in unshare_fs()
3341 if (fs->users == 1) in unshare_fs()
3346 return -ENOMEM; in unshare_fs()
3357 struct files_struct *fd = current->files; in unshare_fd()
3361 (fd && atomic_read(&fd->count) > 1)) { in unshare_fd()
3447 /* Orphan segments in old ns (see sem above). */ in ksys_unshare()
3458 fs = current->fs; in ksys_unshare()
3459 spin_lock(&fs->lock); in ksys_unshare()
3460 current->fs = new_fs; in ksys_unshare()
3461 if (--fs->users) in ksys_unshare()
3465 spin_unlock(&fs->lock); in ksys_unshare()
3469 swap(current->files, new_fd); in ksys_unshare()
3518 old = task->files; in unshare_files()
3520 task->files = copy; in unshare_files()