Lines Matching +full:mm +full:- +full:0
1 // SPDX-License-Identifier: GPL-2.0-only
9 * 'fork.c' contains the help-routines for the 'fork' system call
12 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
18 #include <linux/sched/mm.h>
45 #include <linux/mm.h>
84 #include <linux/posix-timers.h>
85 #include <linux/user-return-notifier.h>
149 DEFINE_PER_CPU(unsigned long, process_counts) = 0;
164 int total = 0; in nr_processes()
211 for (i = 0; i < NR_CACHED_STACKS; i++) { in try_release_thread_stack_to_cache()
224 if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area)) in thread_stack_free_rcu()
232 struct vm_stack *vm_stack = tsk->stack; in thread_stack_delayed_free()
234 vm_stack->stack_vm_area = tsk->stack_vm_area; in thread_stack_delayed_free()
235 call_rcu(&vm_stack->rcu, thread_stack_free_rcu); in thread_stack_delayed_free()
243 for (i = 0; i < NR_CACHED_STACKS; i++) { in free_vm_stack_cache()
249 vfree(vm_stack->addr); in free_vm_stack_cache()
253 return 0; in free_vm_stack_cache()
260 int nr_charged = 0; in memcg_charge_kernel_stack()
262 BUG_ON(vm->nr_pages != THREAD_SIZE / PAGE_SIZE); in memcg_charge_kernel_stack()
264 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) { in memcg_charge_kernel_stack()
265 ret = memcg_kmem_charge_page(vm->pages[i], GFP_KERNEL, 0); in memcg_charge_kernel_stack()
270 return 0; in memcg_charge_kernel_stack()
272 for (i = 0; i < nr_charged; i++) in memcg_charge_kernel_stack()
273 memcg_kmem_uncharge_page(vm->pages[i], 0); in memcg_charge_kernel_stack()
283 for (i = 0; i < NR_CACHED_STACKS; i++) { in alloc_thread_stack_node()
292 kasan_unpoison_range(s->addr, THREAD_SIZE); in alloc_thread_stack_node()
294 stack = kasan_reset_tag(s->addr); in alloc_thread_stack_node()
297 memset(stack, 0, THREAD_SIZE); in alloc_thread_stack_node()
300 vfree(s->addr); in alloc_thread_stack_node()
301 return -ENOMEM; in alloc_thread_stack_node()
304 tsk->stack_vm_area = s; in alloc_thread_stack_node()
305 tsk->stack = stack; in alloc_thread_stack_node()
306 return 0; in alloc_thread_stack_node()
316 node, __builtin_return_address(0)); in alloc_thread_stack_node()
318 return -ENOMEM; in alloc_thread_stack_node()
323 return -ENOMEM; in alloc_thread_stack_node()
330 tsk->stack_vm_area = vm; in alloc_thread_stack_node()
332 tsk->stack = stack; in alloc_thread_stack_node()
333 return 0; in alloc_thread_stack_node()
338 if (!try_release_thread_stack_to_cache(tsk->stack_vm_area)) in free_thread_stack()
341 tsk->stack = NULL; in free_thread_stack()
342 tsk->stack_vm_area = NULL; in free_thread_stack()
354 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
365 tsk->stack = kasan_reset_tag(page_address(page)); in alloc_thread_stack_node()
366 return 0; in alloc_thread_stack_node()
368 return -ENOMEM; in alloc_thread_stack_node()
374 tsk->stack = NULL; in free_thread_stack()
389 struct rcu_head *rh = tsk->stack; in thread_stack_delayed_free()
399 tsk->stack = stack; in alloc_thread_stack_node()
400 return stack ? 0 : -ENOMEM; in alloc_thread_stack_node()
406 tsk->stack = NULL; in free_thread_stack()
412 THREAD_SIZE, THREAD_SIZE, 0, 0, in thread_stack_cache_init()
419 /* SLAB cache for signal_struct structures (tsk->signal) */
422 /* SLAB cache for sighand_struct structures (tsk->sighand) */
425 /* SLAB cache for files_struct structures (tsk->files) */
428 /* SLAB cache for fs_struct structures (tsk->fs) */
434 /* SLAB cache for mm_struct structures (tsk->mm) */
437 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) in vm_area_alloc() argument
445 vma_init(vma, mm); in vm_area_alloc()
453 dest->vm_mm = src->vm_mm; in vm_area_init_from()
454 dest->vm_ops = src->vm_ops; in vm_area_init_from()
455 dest->vm_start = src->vm_start; in vm_area_init_from()
456 dest->vm_end = src->vm_end; in vm_area_init_from()
457 dest->anon_vma = src->anon_vma; in vm_area_init_from()
458 dest->vm_pgoff = src->vm_pgoff; in vm_area_init_from()
459 dest->vm_file = src->vm_file; in vm_area_init_from()
460 dest->vm_private_data = src->vm_private_data; in vm_area_init_from()
461 vm_flags_init(dest, src->vm_flags); in vm_area_init_from()
462 memcpy(&dest->vm_page_prot, &src->vm_page_prot, in vm_area_init_from()
463 sizeof(dest->vm_page_prot)); in vm_area_init_from()
465 * src->shared.rb may be modified concurrently when called from in vm_area_init_from()
468 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared))); in vm_area_init_from()
469 memcpy(&dest->vm_userfaultfd_ctx, &src->vm_userfaultfd_ctx, in vm_area_init_from()
470 sizeof(dest->vm_userfaultfd_ctx)); in vm_area_init_from()
472 dest->anon_name = src->anon_name; in vm_area_init_from()
475 memcpy(&dest->swap_readahead_info, &src->swap_readahead_info, in vm_area_init_from()
476 sizeof(dest->swap_readahead_info)); in vm_area_init_from()
479 dest->vm_region = src->vm_region; in vm_area_init_from()
482 dest->vm_policy = src->vm_policy; in vm_area_init_from()
493 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags); in vm_area_dup()
494 ASSERT_EXCLUSIVE_WRITER(orig->vm_file); in vm_area_dup()
497 INIT_LIST_HEAD(&new->anon_vma_chain); in vm_area_dup()
519 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in account_kernel_stack()
520 mod_lruvec_page_state(vm->pages[i], NR_KERNEL_STACK_KB, in account_kernel_stack()
533 account_kernel_stack(tsk, -1); in exit_task_stack_account()
540 for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) in exit_task_stack_account()
541 memcg_kmem_uncharge_page(vm->pages[i], 0); in exit_task_stack_account()
547 if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD)) in release_task_stack()
556 if (refcount_dec_and_test(&tsk->stack_refcount)) in put_task_stack()
564 WARN_ON_ONCE(tsk->seccomp.filter); in free_task()
580 WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0); in free_task()
585 if (tsk->flags & PF_KTHREAD) in free_task()
592 static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mm_exe_file() argument
597 RCU_INIT_POINTER(mm->exe_file, exe_file); in dup_mm_exe_file()
607 static __latent_entropy int dup_mmap(struct mm_struct *mm, in dup_mmap() argument
612 unsigned long charge = 0; in dup_mmap()
614 VMA_ITERATOR(vmi, mm, 0); in dup_mmap()
617 return -EINTR; in dup_mmap()
619 uprobe_dup_mmap(oldmm, mm); in dup_mmap()
621 * Not linked in yet - no deadlock potential: in dup_mmap()
623 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); in dup_mmap()
626 dup_mm_exe_file(mm, oldmm); in dup_mmap()
628 mm->total_vm = oldmm->total_vm; in dup_mmap()
629 mm->data_vm = oldmm->data_vm; in dup_mmap()
630 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
631 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
634 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); in dup_mmap()
643 if (mpnt->vm_flags & VM_DONTCOPY) { in dup_mmap()
644 retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start, in dup_mmap()
645 mpnt->vm_end, GFP_KERNEL); in dup_mmap()
649 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); in dup_mmap()
652 charge = 0; in dup_mmap()
654 * Don't duplicate many vmas if we've been oom-killed (for in dup_mmap()
658 retval = -EINTR; in dup_mmap()
661 if (mpnt->vm_flags & VM_ACCOUNT) { in dup_mmap()
673 if (unlikely(tmp->vm_flags & VM_PFNMAP)) in dup_mmap()
679 tmp->vm_mm = mm; in dup_mmap()
683 if (tmp->vm_flags & VM_WIPEONFORK) { in dup_mmap()
689 tmp->anon_vma = NULL; in dup_mmap()
705 mm->map_count++; in dup_mmap()
707 if (tmp->vm_ops && tmp->vm_ops->open) in dup_mmap()
708 tmp->vm_ops->open(tmp); in dup_mmap()
710 file = tmp->vm_file; in dup_mmap()
712 struct address_space *mapping = file->f_mapping; in dup_mmap()
721 &mapping->i_mmap); in dup_mmap()
726 if (!(tmp->vm_flags & VM_WIPEONFORK)) in dup_mmap()
734 /* a new mm has just been created */ in dup_mmap()
735 retval = arch_dup_mmap(oldmm, mm); in dup_mmap()
740 ksm_fork(mm, oldmm); in dup_mmap()
741 khugepaged_fork(mm, oldmm); in dup_mmap()
752 mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1); in dup_mmap()
755 set_bit(MMF_OOM_SKIP, &mm->flags); in dup_mmap()
762 set_bit(MMF_UNSTABLE, &mm->flags); in dup_mmap()
765 mmap_write_unlock(mm); in dup_mmap()
779 retval = -ENOMEM; in dup_mmap()
784 static inline int mm_alloc_pgd(struct mm_struct *mm) in mm_alloc_pgd() argument
786 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
787 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
788 return -ENOMEM; in mm_alloc_pgd()
789 return 0; in mm_alloc_pgd()
792 static inline void mm_free_pgd(struct mm_struct *mm) in mm_free_pgd() argument
794 pgd_free(mm, mm->pgd); in mm_free_pgd()
797 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
800 dup_mm_exe_file(mm, oldmm); in dup_mmap()
802 return 0; in dup_mmap()
804 #define mm_alloc_pgd(mm) (0) argument
805 #define mm_free_pgd(mm) argument
811 static inline int mm_alloc_id(struct mm_struct *mm) in mm_alloc_id() argument
816 if (ret < 0) in mm_alloc_id()
818 mm->mm_id = ret; in mm_alloc_id()
819 return 0; in mm_alloc_id()
822 static inline void mm_free_id(struct mm_struct *mm) in mm_free_id() argument
824 const mm_id_t id = mm->mm_id; in mm_free_id()
826 mm->mm_id = MM_ID_DUMMY; in mm_free_id()
834 static inline int mm_alloc_id(struct mm_struct *mm) { return 0; } in mm_alloc_id() argument
835 static inline void mm_free_id(struct mm_struct *mm) {} in mm_free_id() argument
838 static void check_mm(struct mm_struct *mm) in check_mm() argument
845 for (i = 0; i < NR_MM_COUNTERS; i++) { in check_mm()
846 long x = percpu_counter_sum(&mm->rss_stat[i]); in check_mm()
849 pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", in check_mm()
850 mm, resident_page_types[i], x); in check_mm()
853 if (mm_pgtables_bytes(mm)) in check_mm()
854 pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", in check_mm()
855 mm_pgtables_bytes(mm)); in check_mm()
858 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
863 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) argument
867 struct mm_struct *mm = arg; in do_check_lazy_tlb() local
869 WARN_ON_ONCE(current->active_mm == mm); in do_check_lazy_tlb()
874 struct mm_struct *mm = arg; in do_shoot_lazy_tlb() local
876 if (current->active_mm == mm) { in do_shoot_lazy_tlb()
877 WARN_ON_ONCE(current->mm); in do_shoot_lazy_tlb()
878 current->active_mm = &init_mm; in do_shoot_lazy_tlb()
879 switch_mm(mm, &init_mm, current); in do_shoot_lazy_tlb()
883 static void cleanup_lazy_tlbs(struct mm_struct *mm) in cleanup_lazy_tlbs() argument
894 * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it in cleanup_lazy_tlbs()
895 * requires lazy mm users to switch to another mm when the refcount in cleanup_lazy_tlbs()
896 * drops to zero, before the mm is freed. This requires IPIs here to in cleanup_lazy_tlbs()
899 * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm in cleanup_lazy_tlbs()
901 * mm lazy on this CPU but no others, reducing the need for additional in cleanup_lazy_tlbs()
904 * one exiting, or kernel threads using the mm when userspace exits. in cleanup_lazy_tlbs()
909 * - The last lazy reference created by exit_mm() could instead switch in cleanup_lazy_tlbs()
912 * - A batch of mms requiring IPIs could be gathered and freed at once. in cleanup_lazy_tlbs()
913 * - CPUs store active_mm where it can be remotely checked without a in cleanup_lazy_tlbs()
914 * lock, to filter out false-positives in the cpumask. in cleanup_lazy_tlbs()
915 * - After mm_users or mm_count reaches zero, switching away from the in cleanup_lazy_tlbs()
916 * mm could clear mm_cpumask to reduce some IPIs, perhaps together in cleanup_lazy_tlbs()
918 * - A delayed freeing and RCU-like quiescing sequence based on mm in cleanup_lazy_tlbs()
921 on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
923 on_each_cpu(do_check_lazy_tlb, (void *)mm, 1); in cleanup_lazy_tlbs()
927 * Called when the last reference to the mm
929 * mmput. Free the page directory and the mm.
931 void __mmdrop(struct mm_struct *mm) in __mmdrop() argument
933 BUG_ON(mm == &init_mm); in __mmdrop()
934 WARN_ON_ONCE(mm == current->mm); in __mmdrop()
936 /* Ensure no CPUs are using this as their lazy tlb mm */ in __mmdrop()
937 cleanup_lazy_tlbs(mm); in __mmdrop()
939 WARN_ON_ONCE(mm == current->active_mm); in __mmdrop()
940 mm_free_pgd(mm); in __mmdrop()
941 mm_free_id(mm); in __mmdrop()
942 destroy_context(mm); in __mmdrop()
943 mmu_notifier_subscriptions_destroy(mm); in __mmdrop()
944 check_mm(mm); in __mmdrop()
945 put_user_ns(mm->user_ns); in __mmdrop()
946 mm_pasid_drop(mm); in __mmdrop()
947 mm_destroy_cid(mm); in __mmdrop()
948 percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS); in __mmdrop()
950 free_mm(mm); in __mmdrop()
956 struct mm_struct *mm; in mmdrop_async_fn() local
958 mm = container_of(work, struct mm_struct, async_put_work); in mmdrop_async_fn()
959 __mmdrop(mm); in mmdrop_async_fn()
962 static void mmdrop_async(struct mm_struct *mm) in mmdrop_async() argument
964 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { in mmdrop_async()
965 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); in mmdrop_async()
966 schedule_work(&mm->async_put_work); in mmdrop_async()
978 if (sig->oom_mm) in free_signal_struct()
979 mmdrop_async(sig->oom_mm); in free_signal_struct()
985 if (refcount_dec_and_test(&sig->sigcnt)) in put_signal_struct()
991 WARN_ON(!tsk->exit_state); in __put_task_struct()
992 WARN_ON(refcount_read(&tsk->usage)); in __put_task_struct()
1002 put_signal_struct(tsk->signal); in __put_task_struct()
1053 * Handle zero-sized whitelist or empty thread_struct, otherwise in task_struct_whitelist()
1056 if (unlikely(*size == 0)) in task_struct_whitelist()
1057 *offset = 0; in task_struct_whitelist()
1066 #define ARCH_MIN_TASKALIGN 0 in fork_init()
1083 init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; in fork_init()
1084 init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; in fork_init()
1085 init_task.signal->rlim[RLIMIT_SIGPENDING] = in fork_init()
1086 init_task.signal->rlim[RLIMIT_NPROC]; in fork_init()
1088 for (i = 0; i < UCOUNT_COUNTS; i++) in fork_init()
1111 return 0; in arch_dup_task_struct()
1142 refcount_set(&tsk->stack_refcount, 1); in dup_task_struct()
1157 tsk->seccomp.filter = NULL; in dup_task_struct()
1167 tsk->stack_canary = get_random_canary(); in dup_task_struct()
1169 if (orig->cpus_ptr == &orig->cpus_mask) in dup_task_struct()
1170 tsk->cpus_ptr = &tsk->cpus_mask; in dup_task_struct()
1177 refcount_set(&tsk->rcu_users, 2); in dup_task_struct()
1179 refcount_set(&tsk->usage, 1); in dup_task_struct()
1181 tsk->btrace_seq = 0; in dup_task_struct()
1183 tsk->splice_pipe = NULL; in dup_task_struct()
1184 tsk->task_frag.page = NULL; in dup_task_struct()
1185 tsk->wake_q.next = NULL; in dup_task_struct()
1186 tsk->worker_private = NULL; in dup_task_struct()
1193 tsk->fail_nth = 0; in dup_task_struct()
1197 tsk->throttle_disk = NULL; in dup_task_struct()
1198 tsk->use_memdelay = 0; in dup_task_struct()
1202 tsk->pasid_activated = 0; in dup_task_struct()
1206 tsk->active_memcg = NULL; in dup_task_struct()
1210 tsk->reported_split_lock = 0; in dup_task_struct()
1214 tsk->mm_cid = -1; in dup_task_struct()
1215 tsk->last_mm_cid = -1; in dup_task_struct()
1216 tsk->mm_cid_active = 0; in dup_task_struct()
1217 tsk->migrate_from_cpu = -1; in dup_task_struct()
1236 (simple_strtoul(s, NULL, 0) << MMF_DUMP_FILTER_SHIFT) & in coredump_filter_setup()
1245 static void mm_init_aio(struct mm_struct *mm) in mm_init_aio() argument
1248 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
1249 mm->ioctx_table = NULL; in mm_init_aio()
1253 static __always_inline void mm_clear_owner(struct mm_struct *mm, in mm_clear_owner() argument
1257 if (mm->owner == p) in mm_clear_owner()
1258 WRITE_ONCE(mm->owner, NULL); in mm_clear_owner()
1262 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) in mm_init_owner() argument
1265 mm->owner = p; in mm_init_owner()
1269 static void mm_init_uprobes_state(struct mm_struct *mm) in mm_init_uprobes_state() argument
1272 mm->uprobes_state.xol_area = NULL; in mm_init_uprobes_state()
1276 static void mmap_init_lock(struct mm_struct *mm) in mmap_init_lock() argument
1278 init_rwsem(&mm->mmap_lock); in mmap_init_lock()
1279 mm_lock_seqcount_init(mm); in mmap_init_lock()
1281 rcuwait_init(&mm->vma_writer_wait); in mmap_init_lock()
1285 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, in mm_init() argument
1288 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); in mm_init()
1289 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); in mm_init()
1290 atomic_set(&mm->mm_users, 1); in mm_init()
1291 atomic_set(&mm->mm_count, 1); in mm_init()
1292 seqcount_init(&mm->write_protect_seq); in mm_init()
1293 mmap_init_lock(mm); in mm_init()
1294 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
1295 mm_pgtables_bytes_init(mm); in mm_init()
1296 mm->map_count = 0; in mm_init()
1297 mm->locked_vm = 0; in mm_init()
1298 atomic64_set(&mm->pinned_vm, 0); in mm_init()
1299 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
1300 spin_lock_init(&mm->page_table_lock); in mm_init()
1301 spin_lock_init(&mm->arg_lock); in mm_init()
1302 mm_init_cpumask(mm); in mm_init()
1303 mm_init_aio(mm); in mm_init()
1304 mm_init_owner(mm, p); in mm_init()
1305 mm_pasid_init(mm); in mm_init()
1306 RCU_INIT_POINTER(mm->exe_file, NULL); in mm_init()
1307 mmu_notifier_subscriptions_init(mm); in mm_init()
1308 init_tlb_flush_pending(mm); in mm_init()
1310 mm->pmd_huge_pte = NULL; in mm_init()
1312 mm_init_uprobes_state(mm); in mm_init()
1313 hugetlb_count_init(mm); in mm_init()
1315 if (current->mm) { in mm_init()
1316 mm->flags = mmf_init_flags(current->mm->flags); in mm_init()
1317 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
1319 mm->flags = default_dump_filter; in mm_init()
1320 mm->def_flags = 0; in mm_init()
1323 if (mm_alloc_pgd(mm)) in mm_init()
1326 if (mm_alloc_id(mm)) in mm_init()
1329 if (init_new_context(p, mm)) in mm_init()
1332 if (mm_alloc_cid(mm, p)) in mm_init()
1335 if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT, in mm_init()
1339 mm->user_ns = get_user_ns(user_ns); in mm_init()
1340 lru_gen_init_mm(mm); in mm_init()
1341 return mm; in mm_init()
1344 mm_destroy_cid(mm); in mm_init()
1346 destroy_context(mm); in mm_init()
1348 mm_free_id(mm); in mm_init()
1350 mm_free_pgd(mm); in mm_init()
1352 free_mm(mm); in mm_init()
1361 struct mm_struct *mm; in mm_alloc() local
1363 mm = allocate_mm(); in mm_alloc()
1364 if (!mm) in mm_alloc()
1367 memset(mm, 0, sizeof(*mm)); in mm_alloc()
1368 return mm_init(mm, current, current_user_ns()); in mm_alloc()
1372 static inline void __mmput(struct mm_struct *mm) in __mmput() argument
1374 VM_BUG_ON(atomic_read(&mm->mm_users)); in __mmput()
1376 uprobe_clear_state(mm); in __mmput()
1377 exit_aio(mm); in __mmput()
1378 ksm_exit(mm); in __mmput()
1379 khugepaged_exit(mm); /* must run before exit_mmap */ in __mmput()
1380 exit_mmap(mm); in __mmput()
1381 mm_put_huge_zero_folio(mm); in __mmput()
1382 set_mm_exe_file(mm, NULL); in __mmput()
1383 if (!list_empty(&mm->mmlist)) { in __mmput()
1385 list_del(&mm->mmlist); in __mmput()
1388 if (mm->binfmt) in __mmput()
1389 module_put(mm->binfmt->module); in __mmput()
1390 lru_gen_del_mm(mm); in __mmput()
1391 mmdrop(mm); in __mmput()
1395 * Decrement the use count and release all resources for an mm.
1397 void mmput(struct mm_struct *mm) in mmput() argument
1401 if (atomic_dec_and_test(&mm->mm_users)) in mmput()
1402 __mmput(mm); in mmput()
1409 struct mm_struct *mm = container_of(work, struct mm_struct, in mmput_async_fn() local
1412 __mmput(mm); in mmput_async_fn()
1415 void mmput_async(struct mm_struct *mm) in mmput_async() argument
1417 if (atomic_dec_and_test(&mm->mm_users)) { in mmput_async()
1418 INIT_WORK(&mm->async_put_work, mmput_async_fn); in mmput_async()
1419 schedule_work(&mm->async_put_work); in mmput_async()
1426 * set_mm_exe_file - change a reference to the mm's executable file
1427 * @mm: The mm to change.
1430 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1434 * the new mm is made visible to anyone.
1438 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in set_mm_exe_file() argument
1445 * this mm -- see comment above for justification. in set_mm_exe_file()
1447 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
1455 return -EACCES; in set_mm_exe_file()
1458 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
1463 return 0; in set_mm_exe_file()
1467 * replace_mm_exe_file - replace a reference to the mm's executable file
1468 * @mm: The mm to change.
1471 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
1475 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in replace_mm_exe_file() argument
1479 int ret = 0; in replace_mm_exe_file()
1481 /* Forbid mm->exe_file change if old file still mapped. */ in replace_mm_exe_file()
1482 old_exe_file = get_mm_exe_file(mm); in replace_mm_exe_file()
1484 VMA_ITERATOR(vmi, mm, 0); in replace_mm_exe_file()
1485 mmap_read_lock(mm); in replace_mm_exe_file()
1487 if (!vma->vm_file) in replace_mm_exe_file()
1489 if (path_equal(&vma->vm_file->f_path, in replace_mm_exe_file()
1490 &old_exe_file->f_path)) { in replace_mm_exe_file()
1491 ret = -EBUSY; in replace_mm_exe_file()
1495 mmap_read_unlock(mm); in replace_mm_exe_file()
1503 return -EACCES; in replace_mm_exe_file()
1507 mmap_write_lock(mm); in replace_mm_exe_file()
1508 old_exe_file = rcu_dereference_raw(mm->exe_file); in replace_mm_exe_file()
1509 rcu_assign_pointer(mm->exe_file, new_exe_file); in replace_mm_exe_file()
1510 mmap_write_unlock(mm); in replace_mm_exe_file()
1516 return 0; in replace_mm_exe_file()
1520 * get_mm_exe_file - acquire a reference to the mm's executable file
1521 * @mm: The mm of interest.
1523 * Returns %NULL if mm has no associated executable file.
1526 struct file *get_mm_exe_file(struct mm_struct *mm) in get_mm_exe_file() argument
1531 exe_file = get_file_rcu(&mm->exe_file); in get_mm_exe_file()
1537 * get_task_exe_file - acquire a reference to the task's executable file
1540 * Returns %NULL if task's mm (if any) has no associated executable file or
1541 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
1547 struct mm_struct *mm; in get_task_exe_file() local
1549 if (task->flags & PF_KTHREAD) in get_task_exe_file()
1553 mm = task->mm; in get_task_exe_file()
1554 if (mm) in get_task_exe_file()
1555 exe_file = get_mm_exe_file(mm); in get_task_exe_file()
1561 * get_task_mm - acquire a reference to the task's mm
1564 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
1565 * this kernel workthread has transiently adopted a user mm with use_mm,
1567 * bumping up the use count. User must release the mm via mmput()
1572 struct mm_struct *mm; in get_task_mm() local
1574 if (task->flags & PF_KTHREAD) in get_task_mm()
1578 mm = task->mm; in get_task_mm()
1579 if (mm) in get_task_mm()
1580 mmget(mm); in get_task_mm()
1582 return mm; in get_task_mm()
1586 static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode) in may_access_mm() argument
1588 if (mm == current->mm) in may_access_mm()
1599 struct mm_struct *mm; in mm_access() local
1602 err = down_read_killable(&task->signal->exec_update_lock); in mm_access()
1606 mm = get_task_mm(task); in mm_access()
1607 if (!mm) { in mm_access()
1608 mm = ERR_PTR(-ESRCH); in mm_access()
1609 } else if (!may_access_mm(mm, task, mode)) { in mm_access()
1610 mmput(mm); in mm_access()
1611 mm = ERR_PTR(-EACCES); in mm_access()
1613 up_read(&task->signal->exec_update_lock); in mm_access()
1615 return mm; in mm_access()
1623 vfork = tsk->vfork_done; in complete_vfork_done()
1625 tsk->vfork_done = NULL; in complete_vfork_done()
1643 child->vfork_done = NULL; in wait_for_vfork_done()
1664 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) in mm_release() argument
1669 deactivate_mm(tsk, mm); in mm_release()
1676 if (tsk->clear_child_tid) { in mm_release()
1677 if (atomic_read(&mm->mm_users) > 1) { in mm_release()
1679 * We don't check the error code - if userspace has in mm_release()
1682 put_user(0, tsk->clear_child_tid); in mm_release()
1683 do_futex(tsk->clear_child_tid, FUTEX_WAKE, in mm_release()
1684 1, NULL, NULL, 0, 0); in mm_release()
1686 tsk->clear_child_tid = NULL; in mm_release()
1690 * All done, finally we can wake up parent and return this mm to him. in mm_release()
1693 if (tsk->vfork_done) in mm_release()
1697 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exit_mm_release() argument
1700 mm_release(tsk, mm); in exit_mm_release()
1703 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exec_mm_release() argument
1706 mm_release(tsk, mm); in exec_mm_release()
1710 * dup_mm() - duplicates an existing mm structure
1711 * @tsk: the task_struct with which the new mm will be associated.
1712 * @oldmm: the mm to duplicate.
1714 * Allocates a new mm structure and duplicates the provided @oldmm structure
1717 * Return: the duplicated mm or NULL on failure.
1722 struct mm_struct *mm; in dup_mm() local
1725 mm = allocate_mm(); in dup_mm()
1726 if (!mm) in dup_mm()
1729 memcpy(mm, oldmm, sizeof(*mm)); in dup_mm()
1731 if (!mm_init(mm, tsk, mm->user_ns)) in dup_mm()
1735 err = dup_mmap(mm, oldmm); in dup_mm()
1740 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
1741 mm->hiwater_vm = mm->total_vm; in dup_mm()
1743 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
1746 return mm; in dup_mm()
1750 mm->binfmt = NULL; in dup_mm()
1751 mm_init_owner(mm, NULL); in dup_mm()
1752 mmput(mm); in dup_mm()
1762 struct mm_struct *mm, *oldmm; in copy_mm() local
1764 tsk->min_flt = tsk->maj_flt = 0; in copy_mm()
1765 tsk->nvcsw = tsk->nivcsw = 0; in copy_mm()
1767 tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw; in copy_mm()
1768 tsk->last_switch_time = 0; in copy_mm()
1771 tsk->mm = NULL; in copy_mm()
1772 tsk->active_mm = NULL; in copy_mm()
1779 oldmm = current->mm; in copy_mm()
1781 return 0; in copy_mm()
1785 mm = oldmm; in copy_mm()
1787 mm = dup_mm(tsk, current->mm); in copy_mm()
1788 if (!mm) in copy_mm()
1789 return -ENOMEM; in copy_mm()
1792 tsk->mm = mm; in copy_mm()
1793 tsk->active_mm = mm; in copy_mm()
1795 return 0; in copy_mm()
1800 struct fs_struct *fs = current->fs; in copy_fs()
1802 /* tsk->fs is already what we want */ in copy_fs()
1803 spin_lock(&fs->lock); in copy_fs()
1805 if (fs->in_exec) { in copy_fs()
1806 spin_unlock(&fs->lock); in copy_fs()
1807 return -EAGAIN; in copy_fs()
1809 fs->users++; in copy_fs()
1810 spin_unlock(&fs->lock); in copy_fs()
1811 return 0; in copy_fs()
1813 tsk->fs = copy_fs_struct(fs); in copy_fs()
1814 if (!tsk->fs) in copy_fs()
1815 return -ENOMEM; in copy_fs()
1816 return 0; in copy_fs()
1827 oldf = current->files; in copy_files()
1829 return 0; in copy_files()
1832 tsk->files = NULL; in copy_files()
1833 return 0; in copy_files()
1837 atomic_inc(&oldf->count); in copy_files()
1838 return 0; in copy_files()
1845 tsk->files = newf; in copy_files()
1846 return 0; in copy_files()
1854 refcount_inc(¤t->sighand->count); in copy_sighand()
1855 return 0; in copy_sighand()
1858 RCU_INIT_POINTER(tsk->sighand, sig); in copy_sighand()
1860 return -ENOMEM; in copy_sighand()
1862 refcount_set(&sig->count, 1); in copy_sighand()
1863 spin_lock_irq(¤t->sighand->siglock); in copy_sighand()
1864 memcpy(sig->action, current->sighand->action, sizeof(sig->action)); in copy_sighand()
1865 spin_unlock_irq(¤t->sighand->siglock); in copy_sighand()
1869 flush_signal_handlers(tsk, 0); in copy_sighand()
1871 return 0; in copy_sighand()
1876 if (refcount_dec_and_test(&sighand->count)) { in __cleanup_sighand()
1891 struct posix_cputimers *pct = &sig->posix_cputimers; in posix_cpu_timers_init_group()
1894 cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); in posix_cpu_timers_init_group()
1903 return 0; in copy_signal()
1906 tsk->signal = sig; in copy_signal()
1908 return -ENOMEM; in copy_signal()
1910 sig->nr_threads = 1; in copy_signal()
1911 sig->quick_threads = 1; in copy_signal()
1912 atomic_set(&sig->live, 1); in copy_signal()
1913 refcount_set(&sig->sigcnt, 1); in copy_signal()
1916 sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node); in copy_signal()
1917 tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head); in copy_signal()
1919 init_waitqueue_head(&sig->wait_chldexit); in copy_signal()
1920 sig->curr_target = tsk; in copy_signal()
1921 init_sigpending(&sig->shared_pending); in copy_signal()
1922 INIT_HLIST_HEAD(&sig->multiprocess); in copy_signal()
1923 seqlock_init(&sig->stats_lock); in copy_signal()
1924 prev_cputime_init(&sig->prev_cputime); in copy_signal()
1927 INIT_HLIST_HEAD(&sig->posix_timers); in copy_signal()
1928 INIT_HLIST_HEAD(&sig->ignored_posix_timers); in copy_signal()
1929 hrtimer_setup(&sig->real_timer, it_real_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in copy_signal()
1932 task_lock(current->group_leader); in copy_signal()
1933 memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim); in copy_signal()
1934 task_unlock(current->group_leader); in copy_signal()
1941 sig->oom_score_adj = current->signal->oom_score_adj; in copy_signal()
1942 sig->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_signal()
1944 mutex_init(&sig->cred_guard_mutex); in copy_signal()
1945 init_rwsem(&sig->exec_update_lock); in copy_signal()
1947 return 0; in copy_signal()
1954 * Must be called with sighand->lock held, which is common to in copy_seccomp()
1959 assert_spin_locked(¤t->sighand->siglock); in copy_seccomp()
1961 /* Ref-count the new filter user, and assign it. */ in copy_seccomp()
1963 p->seccomp = current->seccomp; in copy_seccomp()
1978 if (p->seccomp.mode != SECCOMP_MODE_DISABLED) in copy_seccomp()
1985 current->clear_child_tid = tidptr; in SYSCALL_DEFINE1()
1992 raw_spin_lock_init(&p->pi_lock); in rt_mutex_init_task()
1994 p->pi_waiters = RB_ROOT_CACHED; in rt_mutex_init_task()
1995 p->pi_top_task = NULL; in rt_mutex_init_task()
1996 p->pi_blocked_on = NULL; in rt_mutex_init_task()
2005 INIT_HLIST_NODE(&task->pid_links[type]); in init_task_pid_links()
2012 task->thread_pid = pid; in init_task_pid()
2014 task->signal->pids[type] = pid; in init_task_pid()
2020 p->rcu_read_lock_nesting = 0; in rcu_copy_process()
2021 p->rcu_read_unlock_special.s = 0; in rcu_copy_process()
2022 p->rcu_blocked_node = NULL; in rcu_copy_process()
2023 INIT_LIST_HEAD(&p->rcu_node_entry); in rcu_copy_process()
2026 p->rcu_tasks_holdout = false; in rcu_copy_process()
2027 INIT_LIST_HEAD(&p->rcu_tasks_holdout_list); in rcu_copy_process()
2028 p->rcu_tasks_idle_cpu = -1; in rcu_copy_process()
2029 INIT_LIST_HEAD(&p->rcu_tasks_exit_list); in rcu_copy_process()
2032 p->trc_reader_nesting = 0; in rcu_copy_process()
2033 p->trc_reader_special.s = 0; in rcu_copy_process()
2034 INIT_LIST_HEAD(&p->trc_holdout_list); in rcu_copy_process()
2035 INIT_LIST_HEAD(&p->trc_blkd_node); in rcu_copy_process()
2040 * __pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2072 if (pidfd < 0) in __pidfd_prepare()
2084 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
2093 * task identified by @pid must be a thread-group leader.
2115 return -EINVAL; in pidfd_prepare()
2130 call_rcu(&tsk->rcu, __delayed_free_task); in delayed_free_task()
2138 if (!tsk->mm) in copy_oom_score_adj()
2147 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); in copy_oom_score_adj()
2149 tsk->signal->oom_score_adj = current->signal->oom_score_adj; in copy_oom_score_adj()
2150 tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min; in copy_oom_score_adj()
2159 for (i = 0; i < RV_PER_TASK_MONITORS; i++) in rv_task_fork()
2160 p->rv[i].da_mon.monitoring = false; in rv_task_fork()
2163 #define rv_task_fork(p) do {} while (0)
2172 * flags). The actual kick-off is left to the caller.
2180 int pidfd = -1, retval; in copy_process()
2184 const u64 clone_flags = args->flags; in copy_process()
2185 struct nsproxy *nsp = current->nsproxy; in copy_process()
2192 return ERR_PTR(-EINVAL); in copy_process()
2195 return ERR_PTR(-EINVAL); in copy_process()
2202 return ERR_PTR(-EINVAL); in copy_process()
2210 return ERR_PTR(-EINVAL); in copy_process()
2215 * multi-rooted process trees, prevent global and container-inits in copy_process()
2219 current->signal->flags & SIGNAL_UNKILLABLE) in copy_process()
2220 return ERR_PTR(-EINVAL); in copy_process()
2228 (task_active_pid_ns(current) != nsp->pid_ns_for_children)) in copy_process()
2229 return ERR_PTR(-EINVAL); in copy_process()
2234 * - CLONE_DETACHED is blocked so that we can potentially in copy_process()
2238 return ERR_PTR(-EINVAL); in copy_process()
2250 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2252 hlist_add_head(&delayed.node, ¤t->signal->multiprocess); in copy_process()
2254 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2255 retval = -ERESTARTNOINTR; in copy_process()
2259 retval = -ENOMEM; in copy_process()
2263 p->flags &= ~PF_KTHREAD; in copy_process()
2264 if (args->kthread) in copy_process()
2265 p->flags |= PF_KTHREAD; in copy_process()
2266 if (args->user_worker) { in copy_process()
2271 p->flags |= PF_USER_WORKER; in copy_process()
2272 siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP)); in copy_process()
2274 if (args->io_thread) in copy_process()
2275 p->flags |= PF_IO_WORKER; in copy_process()
2277 if (args->name) in copy_process()
2278 strscpy_pad(p->comm, args->name, sizeof(p->comm)); in copy_process()
2280 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL; in copy_process()
2284 p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; in copy_process()
2292 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); in copy_process()
2295 if (retval < 0) in copy_process()
2298 retval = -EAGAIN; in copy_process()
2300 if (p->real_cred->user != INIT_USER && in copy_process()
2304 current->flags &= ~PF_NPROC_EXCEEDED; in copy_process()
2311 retval = -EAGAIN; in copy_process()
2316 p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY); in copy_process()
2317 p->flags |= PF_FORKNOEXEC; in copy_process()
2318 INIT_LIST_HEAD(&p->children); in copy_process()
2319 INIT_LIST_HEAD(&p->sibling); in copy_process()
2321 p->vfork_done = NULL; in copy_process()
2322 spin_lock_init(&p->alloc_lock); in copy_process()
2324 init_sigpending(&p->pending); in copy_process()
2326 p->utime = p->stime = p->gtime = 0; in copy_process()
2328 p->utimescaled = p->stimescaled = 0; in copy_process()
2330 prev_cputime_init(&p->prev_cputime); in copy_process()
2333 seqcount_init(&p->vtime.seqcount); in copy_process()
2334 p->vtime.starttime = 0; in copy_process()
2335 p->vtime.state = VTIME_INACTIVE; in copy_process()
2339 p->io_uring = NULL; in copy_process()
2342 p->default_timer_slack_ns = current->timer_slack_ns; in copy_process()
2345 p->psi_flags = 0; in copy_process()
2348 task_io_accounting_init(&p->ioac); in copy_process()
2351 posix_cputimers_init(&p->posix_cputimers); in copy_process()
2354 p->io_context = NULL; in copy_process()
2357 if (args->kthread) { in copy_process()
2362 p->mempolicy = mpol_dup(p->mempolicy); in copy_process()
2363 if (IS_ERR(p->mempolicy)) { in copy_process()
2364 retval = PTR_ERR(p->mempolicy); in copy_process()
2365 p->mempolicy = NULL; in copy_process()
2370 p->cpuset_mem_spread_rotor = NUMA_NO_NODE; in copy_process()
2371 seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock); in copy_process()
2374 memset(&p->irqtrace, 0, sizeof(p->irqtrace)); in copy_process()
2375 p->irqtrace.hardirq_disable_ip = _THIS_IP_; in copy_process()
2376 p->irqtrace.softirq_enable_ip = _THIS_IP_; in copy_process()
2377 p->softirqs_enabled = 1; in copy_process()
2378 p->softirq_context = 0; in copy_process()
2381 p->pagefault_disabled = 0; in copy_process()
2388 p->blocked_on = NULL; /* not blocked yet */ in copy_process()
2391 p->sequential_io = 0; in copy_process()
2392 p->sequential_io_avg = 0; in copy_process()
2395 RCU_INIT_POINTER(p->bpf_storage, NULL); in copy_process()
2396 p->bpf_ctx = NULL; in copy_process()
2418 retval = copy_files(clone_flags, p, args->no_files); in copy_process()
2446 pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid, in copy_process()
2447 args->set_tid_size); in copy_process()
2460 int flags = (clone_flags & CLONE_THREAD) ? PIDFD_THREAD : 0; in copy_process()
2467 if (retval < 0) in copy_process()
2471 retval = put_user(pidfd, args->pidfd); in copy_process()
2477 p->plug = NULL; in copy_process()
2499 p->pid = pid_nr(pid); in copy_process()
2501 p->group_leader = current->group_leader; in copy_process()
2502 p->tgid = current->tgid; in copy_process()
2504 p->group_leader = p; in copy_process()
2505 p->tgid = p->pid; in copy_process()
2508 p->nr_dirtied = 0; in copy_process()
2509 p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10); in copy_process()
2510 p->dirty_paused_when = 0; in copy_process()
2512 p->pdeath_signal = 0; in copy_process()
2513 p->task_works = NULL; in copy_process()
2517 p->kretprobe_instances.first = NULL; in copy_process()
2520 p->rethooks.first = NULL; in copy_process()
2534 * Now that the cgroups are pinned, re-clone the parent cgroup and put in copy_process()
2538 * This isn't part of ->can_fork() because while the re-cloning is in copy_process()
2547 * From this point on we must avoid any synchronous user-space in copy_process()
2548 * communication until we take the tasklist-lock. In particular, we do in copy_process()
2549 * not want user-space to be able to predict the process start-time by in copy_process()
2554 p->start_time = ktime_get_ns(); in copy_process()
2555 p->start_boottime = ktime_get_boottime_ns(); in copy_process()
2563 /* CLONE_PARENT re-uses the old parent */ in copy_process()
2565 p->real_parent = current->real_parent; in copy_process()
2566 p->parent_exec_id = current->parent_exec_id; in copy_process()
2568 p->exit_signal = -1; in copy_process()
2570 p->exit_signal = current->group_leader->exit_signal; in copy_process()
2572 p->real_parent = current; in copy_process()
2573 p->parent_exec_id = current->self_exec_id; in copy_process()
2574 p->exit_signal = args->exit_signal; in copy_process()
2581 spin_lock(¤t->sighand->siglock); in copy_process()
2588 if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) { in copy_process()
2589 retval = -ENOMEM; in copy_process()
2595 retval = -EINTR; in copy_process()
2608 if (likely(p->pid)) { in copy_process()
2618 ns_of_pid(pid)->child_reaper = p; in copy_process()
2619 p->signal->flags |= SIGNAL_UNKILLABLE; in copy_process()
2621 p->signal->shared_pending.signal = delayed.signal; in copy_process()
2622 p->signal->tty = tty_kref_get(current->signal->tty); in copy_process()
2628 p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || in copy_process()
2629 p->real_parent->signal->is_child_subreaper; in copy_process()
2630 list_add_tail(&p->sibling, &p->real_parent->children); in copy_process()
2631 list_add_tail_rcu(&p->tasks, &init_task.tasks); in copy_process()
2637 current->signal->nr_threads++; in copy_process()
2638 current->signal->quick_threads++; in copy_process()
2639 atomic_inc(¤t->signal->live); in copy_process()
2640 refcount_inc(¤t->signal->sigcnt); in copy_process()
2642 list_add_tail_rcu(&p->thread_node, in copy_process()
2643 &p->signal->thread_head); in copy_process()
2650 spin_unlock(¤t->sighand->siglock); in copy_process()
2672 spin_unlock(¤t->sighand->siglock); in copy_process()
2687 if (p->io_context) in copy_process()
2692 if (p->mm) { in copy_process()
2693 mm_clear_owner(p->mm, p); in copy_process()
2694 mmput(p->mm); in copy_process()
2698 free_signal_struct(p->signal); in copy_process()
2700 __cleanup_sighand(p->sighand); in copy_process()
2718 mpol_put(p->mempolicy); in copy_process()
2726 WRITE_ONCE(p->__state, TASK_DEAD); in copy_process()
2731 spin_lock_irq(¤t->sighand->siglock); in copy_process()
2733 spin_unlock_irq(¤t->sighand->siglock); in copy_process()
2742 INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */ in init_idle_pids()
2750 return 0; in idle_dummy()
2764 task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args); in fork_idle()
2793 return copy_process(NULL, 0, node, &args); in create_io_thread()
2797 * Ok, this is the main fork-routine.
2799 * It copies the process, and if successful kick-starts
2802 * args->exit_signal is expected to be checked for sanity by the caller.
2806 u64 clone_flags = args->flags; in kernel_clone()
2810 int trace = 0; in kernel_clone()
2824 (args->pidfd == args->parent_tid)) in kernel_clone()
2825 return -EINVAL; in kernel_clone()
2836 else if (args->exit_signal != SIGCHLD) in kernel_clone()
2842 trace = 0; in kernel_clone()
2852 * Do this prior waking up the new thread - the thread pointer in kernel_clone()
2861 put_user(nr, args->parent_tid); in kernel_clone()
2864 p->vfork_done = &vfork; in kernel_clone()
2872 lru_gen_add_mm(p->mm); in kernel_clone()
2937 return -EINVAL; in SYSCALL_DEFINE0()
2998 pid_t *kset_tid = kargs->set_tid; in copy_clone_args_from_user()
3009 return -E2BIG; in copy_clone_args_from_user()
3011 return -EINVAL; in copy_clone_args_from_user()
3018 return -EINVAL; in copy_clone_args_from_user()
3020 if (unlikely(!args.set_tid && args.set_tid_size > 0)) in copy_clone_args_from_user()
3021 return -EINVAL; in copy_clone_args_from_user()
3023 if (unlikely(args.set_tid && args.set_tid_size == 0)) in copy_clone_args_from_user()
3024 return -EINVAL; in copy_clone_args_from_user()
3032 return -EINVAL; in copy_clone_args_from_user()
3036 return -EINVAL; in copy_clone_args_from_user()
3053 (kargs->set_tid_size * sizeof(pid_t)))) in copy_clone_args_from_user()
3054 return -EFAULT; in copy_clone_args_from_user()
3056 kargs->set_tid = kset_tid; in copy_clone_args_from_user()
3058 return 0; in copy_clone_args_from_user()
3062 * clone3_stack_valid - check and prepare stack
3071 if (kargs->stack == 0) { in clone3_stack_valid()
3072 if (kargs->stack_size > 0) in clone3_stack_valid()
3075 if (kargs->stack_size == 0) in clone3_stack_valid()
3078 if (!access_ok((void __user *)kargs->stack, kargs->stack_size)) in clone3_stack_valid()
3082 kargs->stack += kargs->stack_size; in clone3_stack_valid()
3092 if (kargs->flags & in clone3_args_valid()
3097 * - make the CLONE_DETACHED bit reusable for clone3 in clone3_args_valid()
3098 * - make the CSIGNAL bits reusable for clone3 in clone3_args_valid()
3100 if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME)))) in clone3_args_valid()
3103 if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) == in clone3_args_valid()
3107 if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) && in clone3_args_valid()
3108 kargs->exit_signal) in clone3_args_valid()
3118 * sys_clone3 - create a new process with specific properties
3137 return -ENOSYS; in SYSCALL_DEFINE2()
3147 return -EINVAL; in SYSCALL_DEFINE2()
3158 leader = top = top->group_leader; in walk_process_tree()
3161 list_for_each_entry(child, &parent->children, sibling) { in walk_process_tree()
3164 if (res < 0) in walk_process_tree()
3176 parent = child->real_parent; in walk_process_tree()
3177 leader = parent->group_leader; in walk_process_tree()
3185 #define ARCH_MIN_MMSTRUCT_ALIGN 0
3192 spin_lock_init(&sighand->siglock); in sighand_ctor()
3193 init_waitqueue_head(&sighand->signalfd_wqh); in sighand_ctor()
3223 sizeof(struct sighand_struct), 0, in proc_caches_init()
3227 sizeof(struct signal_struct), 0, in proc_caches_init()
3231 sizeof(struct files_struct), 0, in proc_caches_init()
3235 sizeof(struct fs_struct), 0, in proc_caches_init()
3256 return -EINVAL; in check_unshare_flags()
3265 return -EINVAL; in check_unshare_flags()
3268 if (refcount_read(¤t->sighand->count) > 1) in check_unshare_flags()
3269 return -EINVAL; in check_unshare_flags()
3273 return -EINVAL; in check_unshare_flags()
3276 return 0; in check_unshare_flags()
3284 struct fs_struct *fs = current->fs; in unshare_fs()
3287 return 0; in unshare_fs()
3290 if (fs->users == 1) in unshare_fs()
3291 return 0; in unshare_fs()
3295 return -ENOMEM; in unshare_fs()
3297 return 0; in unshare_fs()
3305 struct files_struct *fd = current->files; in unshare_fd()
3308 (fd && atomic_read(&fd->count) > 1)) { in unshare_fd()
3315 return 0; in unshare_fd()
3332 int do_sysvsem = 0; in ksys_unshare()
3406 fs = current->fs; in ksys_unshare()
3407 spin_lock(&fs->lock); in ksys_unshare()
3408 current->fs = new_fs; in ksys_unshare()
3409 if (--fs->users) in ksys_unshare()
3413 spin_unlock(&fs->lock); in ksys_unshare()
3417 swap(current->files, new_fd); in ksys_unshare()
3466 old = task->files; in unshare_files()
3468 task->files = copy; in unshare_files()
3471 return 0; in unshare_files()
3494 return 0; in sysctl_max_threads()