linux/kernel/fork.c

176 void __weak arch_release_task_struct(struct task_struct *tsk)
187 static inline void free_task_struct(struct task_struct *tsk)
189 	kmem_cache_free(task_struct_cachep, tsk);
235 static void thread_stack_delayed_free(struct task_struct *tsk)
237 	struct vm_stack *vm_stack = tsk->stack;
239 	vm_stack->stack_vm_area = tsk->stack_vm_area;
282 static int alloc_thread_stack_node(struct task_struct *tsk, int node)
306 		tsk->stack_vm_area = vm_area;
307 		tsk->stack = stack;
327 	tsk->stack_vm_area = vm_area;
329 	tsk->stack = stack;
333 static void free_thread_stack(struct task_struct *tsk)
335 	if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))
336 		thread_stack_delayed_free(tsk);
338 	tsk->stack = NULL;
339 	tsk->stack_vm_area = NULL;
355 static void thread_stack_delayed_free(struct task_struct *tsk)
357 	struct rcu_head *rh = tsk->stack;
362 static int alloc_thread_stack_node(struct task_struct *tsk, int node)
368 		tsk->stack = kasan_reset_tag(page_address(page));
374 static void free_thread_stack(struct task_struct *tsk)
376 	thread_stack_delayed_free(tsk);
377 	tsk->stack = NULL;
389 static void thread_stack_delayed_free(struct task_struct *tsk)
391 	struct rcu_head *rh = tsk->stack;
396 static int alloc_thread_stack_node(struct task_struct *tsk, int node)
401 	tsk->stack = stack;
405 static void free_thread_stack(struct task_struct *tsk)
407 	thread_stack_delayed_free(tsk);
408 	tsk->stack = NULL;
422 /* SLAB cache for signal_struct structures (tsk->signal) */
425 /* SLAB cache for sighand_struct structures (tsk->sighand) */
428 /* SLAB cache for files_struct structures (tsk->files) */
431 /* SLAB cache for fs_struct structures (tsk->fs) */
434 /* SLAB cache for mm_struct structures (tsk->mm) */
437 static void account_kernel_stack(struct task_struct *tsk, int account)
440 		struct vm_struct *vm_area = task_stack_vm_area(tsk);
447 		void *stack = task_stack_page(tsk);
455 void exit_task_stack_account(struct task_struct *tsk)
457 	account_kernel_stack(tsk, -1);
463 		vm_area = task_stack_vm_area(tsk);
469 static void release_task_stack(struct task_struct *tsk)
471 	if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
474 	free_thread_stack(tsk);
478 void put_task_stack(struct task_struct *tsk)
480 	if (refcount_dec_and_test(&tsk->stack_refcount))
481 		release_task_stack(tsk);
485 void free_task(struct task_struct *tsk)
488 	WARN_ON_ONCE(tsk->seccomp.filter);
490 	release_user_cpus_ptr(tsk);
491 	scs_release(tsk);
498 	release_task_stack(tsk);
504 	WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
506 	rt_mutex_debug_task_free(tsk);
507 	ftrace_graph_exit_task(tsk);
508 	arch_release_task_struct(tsk);
509 	if (tsk->flags & PF_KTHREAD)
510 		free_kthread_struct(tsk);
511 	bpf_task_storage_free(tsk);
512 	free_task_struct(tsk);
733 void __put_task_struct(struct task_struct *tsk)
735 	WARN_ON(!tsk->exit_state);
736 	WARN_ON(refcount_read(&tsk->usage));
737 	WARN_ON(tsk == current);
739 	unwind_task_free(tsk);
740 	sched_ext_free(tsk);
741 	io_uring_free(tsk);
742 	cgroup_free(tsk);
743 	task_numa_free(tsk, true);
744 	security_task_free(tsk);
745 	exit_creds(tsk);
746 	delayacct_tsk_free(tsk);
747 	put_signal_struct(tsk->signal);
748 	sched_core_free(tsk);
749 	free_task(tsk);
859 void set_task_stack_end_magic(struct task_struct *tsk)
863 	stackend = end_of_stack(tsk);
869 	struct task_struct *tsk;
874 	tsk = alloc_task_struct_node(node);
875 	if (!tsk)
878 	err = arch_dup_task_struct(tsk, orig);
882 	err = alloc_thread_stack_node(tsk, node);
887 	refcount_set(&tsk->stack_refcount, 1);
889 	account_kernel_stack(tsk, 1);
891 	err = scs_prepare(tsk, node);
902 	tsk->seccomp.filter = NULL;
905 	setup_thread_stack(tsk, orig);
906 	clear_user_return_notifier(tsk);
907 	clear_tsk_need_resched(tsk);
908 	set_task_stack_end_magic(tsk);
909 	clear_syscall_work_syscall_user_dispatch(tsk);
912 	tsk->stack_canary = get_random_canary();
915 		tsk->cpus_ptr = &tsk->cpus_mask;
916 	dup_user_cpus_ptr(tsk, orig, node);
922 	refcount_set(&tsk->rcu_users, 2);
924 	refcount_set(&tsk->usage, 1);
926 	tsk->btrace_seq = 0;
928 	tsk->splice_pipe = NULL;
929 	tsk->task_frag.page = NULL;
930 	tsk->wake_q.next = NULL;
931 	tsk->worker_private = NULL;
933 	kcov_task_init(tsk);
934 	kmsan_task_create(tsk);
935 	kmap_local_fork(tsk);
938 	tsk->fail_nth = 0;
942 	tsk->throttle_disk = NULL;
943 	tsk->use_memdelay = 0;
947 	tsk->pasid_activated = 0;
951 	tsk->active_memcg = NULL;
955 	tsk->reported_split_lock = 0;
959 	tsk->mm_cid = -1;
960 	tsk->last_mm_cid = -1;
961 	tsk->mm_cid_active = 0;
962 	tsk->migrate_from_cpu = -1;
964 	return tsk;
967 	exit_task_stack_account(tsk);
968 	free_thread_stack(tsk);
970 	free_task_struct(tsk);
1368 static void complete_vfork_done(struct task_struct *tsk)
1372 	task_lock(tsk);
1373 	vfork = tsk->vfork_done;
1375 		tsk->vfork_done = NULL;
1378 	task_unlock(tsk);
1414 static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
1416 	uprobe_free_utask(tsk);
1419 	deactivate_mm(tsk, mm);
1426 	if (tsk->clear_child_tid) {
1432 			put_user(0, tsk->clear_child_tid);
1433 			do_futex(tsk->clear_child_tid, FUTEX_WAKE,
1436 		tsk->clear_child_tid = NULL;
1443 	if (tsk->vfork_done)
1444 		complete_vfork_done(tsk);
1447 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1449 	futex_exit_release(tsk);
1450 	mm_release(tsk, mm);
1453 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
1455 	futex_exec_release(tsk);
1456 	mm_release(tsk, mm);
1461  * @tsk: the task_struct with which the new mm will be associated.
1469 static struct mm_struct *dup_mm(struct task_struct *tsk,
1481 	if (!mm_init(mm, tsk, mm->user_ns))
1510 static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
1514 	tsk->min_flt = tsk->maj_flt = 0;
1515 	tsk->nvcsw = tsk->nivcsw = 0;
1517 	tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
1518 	tsk->last_switch_time = 0;
1521 	tsk->mm = NULL;
1522 	tsk->active_mm = NULL;
1537 		mm = dup_mm(tsk, current->mm);
1542 	tsk->mm = mm;
1543 	tsk->active_mm = mm;
1544 	sched_mm_cid_fork(tsk);
1548 static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
1552 		/* tsk->fs is already what we want */
1563 	tsk->fs = copy_fs_struct(fs);
1564 	if (!tsk->fs)
1569 static int copy_files(unsigned long clone_flags, struct task_struct *tsk,
1582 		tsk->files = NULL;
1595 	tsk->files = newf;
1599 static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk)
1608 	RCU_INIT_POINTER(tsk->sighand, sig);
1619 		flush_signal_handlers(tsk, 0);
1648 static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
1656 	tsk->signal = sig;
1666 	sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
1667 	tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);
1670 	sig->curr_target = tsk;
1862 	struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
1864 	free_task(tsk);
1867 static __always_inline void delayed_free_task(struct task_struct *tsk)
1870 		call_rcu(&tsk->rcu, __delayed_free_task);
1872 		free_task(tsk);
1875 static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
1878 	if (!tsk->mm)
1887 	set_bit(MMF_MULTIPROCESS, &tsk->mm->flags);
1889 	tsk->signal->oom_score_adj = current->signal->oom_score_adj;
1890 	tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;