1 // SPDX-License-Identifier: GPL-2.0-only
9  * Documentation/dev-tools/kmemleak.rst.
12  * ----------------
16  * - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
21  *   red black trees used to look-up metadata based on a pointer to the
26  * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
33  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
46  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
131 #define KMEMLEAK_BLACK	-1
136  * object->lock. Insertions or deletions from object_list, gray_list or
138  * the notes on locking above). These objects are reference-counted
155 	/* minimum number of a pointers found before it is considered leak */
165 	pid_t pid;			/* pid of the current task */
179 /* flag set for per-CPU pointers */
199 /* the list of gray-colored objects (see color_gray comment below) */
295  * with the object->lock held.
300 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
303 	if (WARN_ON_ONCE(object->flags & (OBJECT_PHYS | OBJECT_PERCPU)))  in hex_dump_object()
307 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
318  * - white - orphan object, not enough references to it (count < min_count)
319  * - gray  - not orphan, not marked as false positive (min_count == 0) or
321  * - black - ignore, it doesn't contain references (e.g. text section)
322  *		(min_count == -1). No function defined for this color.
323  * Newly created objects don't have any color assigned (object->count == -1)
328 	return object->count != KMEMLEAK_BLACK &&  in color_white()
329 		object->count < object->min_count;  in color_white()
334 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
335 		object->count >= object->min_count;  in color_gray()
345 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
346 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
352  * print_unreferenced function must be called with the object->lock held.
357 	int i;  in print_unreferenced()  local
361 	nr_entries = stack_depot_fetch(object->trace_handle, &entries);  in print_unreferenced()
363 			  object->pointer, object->size);  in print_unreferenced()
365 			   object->comm, object->pid, object->jiffies);  in print_unreferenced()
367 	warn_or_seq_printf(seq, "  backtrace (crc %x):\n", object->checksum);  in print_unreferenced()
369 	for (i = 0; i < nr_entries; i++) {  in print_unreferenced()
370 		void *ptr = (void *)entries[i];  in print_unreferenced()
378  * the object->lock held.
383 			object->pointer, object->size);  in dump_object_info()
385 			object->comm, object->pid, object->jiffies);  in dump_object_info()
386 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
387 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
388 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
389 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
391 	if (object->trace_handle)  in dump_object_info()
392 		stack_depot_print(object->trace_handle);  in dump_object_info()
405  * Look-up a memory block metadata (kmemleak_object) in the object search
413 	struct rb_node *rb = object_tree(objflags)->rb_node;  in __lookup_object()
421 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
424 			rb = object->rb_node.rb_left;  in __lookup_object()
425 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
426 			rb = object->rb_node.rb_right;  in __lookup_object()
439 /* Look-up a kmemleak object which allocated with virtual address. */
453 	return atomic_inc_not_zero(&object->use_count);  in get_object()
476 		list_del(&object->object_list);  in mem_pool_alloc()
478 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
500 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
518 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
519 		hlist_del(&area->node);  in free_object_rcu()
527  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
529  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
534 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
538 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
546 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
548 		free_object_rcu(&object->rcu);  in put_object()
585 	rb_erase(&object->rb_node, object_tree(object->flags));  in __remove_object()
586 	if (!(object->del_state & DELSTATE_NO_DELETE))  in __remove_object()
587 		list_del_rcu(&object->object_list);  in __remove_object()
588 	object->del_state |= DELSTATE_REMOVED;  in __remove_object()
652 	INIT_LIST_HEAD(&object->object_list);  in __alloc_object()
653 	INIT_LIST_HEAD(&object->gray_list);  in __alloc_object()
654 	INIT_HLIST_HEAD(&object->area_list);  in __alloc_object()
655 	raw_spin_lock_init(&object->lock);  in __alloc_object()
656 	atomic_set(&object->use_count, 1);  in __alloc_object()
657 	object->excess_ref = 0;  in __alloc_object()
658 	object->count = 0;			/* white color initially */  in __alloc_object()
659 	object->checksum = 0;  in __alloc_object()
660 	object->del_state = 0;  in __alloc_object()
664 		object->pid = 0;  in __alloc_object()
665 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in __alloc_object()
667 		object->pid = 0;  in __alloc_object()
668 		strncpy(object->comm, "softirq", sizeof(object->comm));  in __alloc_object()
670 		object->pid = current->pid;  in __alloc_object()
674 		 * dependency issues with current->alloc_lock. In the worst  in __alloc_object()
677 		strncpy(object->comm, current->comm, sizeof(object->comm));  in __alloc_object()
681 	object->trace_handle = set_track_prepare();  in __alloc_object()
695 	object->flags = OBJECT_ALLOCATED | objflags;  in __link_object()
696 	object->pointer = ptr;  in __link_object()
697 	object->size = kfence_ksize((void *)ptr) ?: size;  in __link_object()
698 	object->min_count = min_count;  in __link_object()
699 	object->jiffies = jiffies;  in __link_object()
710 	link = &object_tree(objflags)->rb_node;  in __link_object()
715 		untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer);  in __link_object()
717 			link = &parent->rb_node.rb_left;  in __link_object()
718 		else if (untagged_objp + parent->size <= untagged_ptr)  in __link_object()
719 			link = &parent->rb_node.rb_right;  in __link_object()
724 			 * No need for parent->lock here since "parent" cannot  in __link_object()
728 			return -EEXIST;  in __link_object()
731 	rb_link_node(&object->rb_node, rb_parent, link);  in __link_object()
732 	rb_insert_color(&object->rb_node, object_tree(objflags));  in __link_object()
733 	list_add_tail_rcu(&object->object_list, &object_list);  in __link_object()
774 /* Create kmemleak object corresponding to a per-CPU allocation. */
788 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
789 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
795 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
796 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
797 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
854 	start = object->pointer;  in delete_object_part()
855 	end = object->pointer + object->size;  in delete_object_part()
857 	    !__link_object(object_l, start, ptr - start,  in delete_object_part()
858 			   object->min_count, objflags))  in delete_object_part()
861 	    !__link_object(object_r, ptr + size, end - ptr - size,  in delete_object_part()
862 			   object->min_count, objflags))  in delete_object_part()
879 	object->min_count = color;  in __paint_it()
881 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
888 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
890 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
910  * Mark an object permanently as gray-colored so that it can no longer be
911  * reported as a leak. This is used in general to mark a false positive.
919  * Mark the object as black-colored so that it is ignored from scans and
947 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
952 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
956 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
960 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
961 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
968 	INIT_HLIST_NODE(&area->node);  in add_scan_area()
969 	area->start = ptr;  in add_scan_area()
970 	area->size = size;  in add_scan_area()
972 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
974 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
996 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
997 	object->excess_ref = excess_ref;  in object_set_excess_ref()
998 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
1018 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
1019 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
1020 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
1025  * kmemleak_alloc - register a newly allocated object
1030  *		the object is reported as a memory leak. If @min_count is 0,
1031  *		the object is never reported as a leak. If @min_count is -1,
1032  *		the object is ignored (not scanned and not reported as a leak)
1049  * kmemleak_alloc_percpu - register a newly allocated __percpu object
1072  * kmemleak_vmalloc - register a newly vmalloc'ed object
1089 		create_object((unsigned long)area->addr, size, 2, gfp);  in kmemleak_vmalloc()
1091 				      (unsigned long)area->addr);  in kmemleak_vmalloc()
1097  * kmemleak_free - unregister a previously registered object
1113  * kmemleak_free_part - partially unregister a previously registered object
1131  * kmemleak_free_percpu - unregister a previously registered __percpu object
1147  * kmemleak_update_trace - update object allocation stack trace
1174 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1175 	object->trace_handle = trace_handle;  in kmemleak_update_trace()
1176 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1183  * kmemleak_not_leak - mark an allocated object as false positive
1187  * be reported as leak and always be scanned.
1199  * kmemleak_ignore - ignore an allocated object
1203  * ignored (not scanned and not reported as a leak). This is usually done when
1204  * it is known that the corresponding block is not a leak and does not contain
1217  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1237  * kmemleak_no_scan - do not scan an allocated object
1255  * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1275  * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1291  * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1309 	u32 old_csum = object->checksum;  in update_checksum()
1311 	if (WARN_ON_ONCE(object->flags & (OBJECT_PHYS | OBJECT_PERCPU)))  in update_checksum()
1316 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1320 	return object->checksum != old_csum;  in update_checksum()
1324  * Update an object's references. object->lock must be held by the caller.
1329 		/* non-orphan, ignored or new */  in update_refs()
1339 	object->count++;  in update_refs()
1343 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1360 	if (current->mm)  in scan_should_stop()
1361 		return signal_pending(current);  in scan_should_stop()
1377 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);  in scan_block()
1400 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1412 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1416 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1419 			excess_ref = object->excess_ref;  in scan_block()
1425 		raw_spin_unlock(&object->lock);  in scan_block()
1434 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1436 			raw_spin_unlock(&object->lock);  in scan_block()
1461  * that object->use_count >= 1.
1469 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1472 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1473 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1475 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1479 	if (object->flags & OBJECT_PERCPU) {  in scan_object()
1483 			void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu);  in scan_object()
1484 			void *end = start + object->size;  in scan_object()
1488 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1490 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1491 			if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1494 	} else if (hlist_empty(&object->area_list) ||  in scan_object()
1495 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1496 		void *start = object->flags & OBJECT_PHYS ?  in scan_object()
1497 				__va((phys_addr_t)object->pointer) :  in scan_object()
1498 				(void *)object->pointer;  in scan_object()
1499 		void *end = start + object->size;  in scan_object()
1510 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1512 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1513 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1515 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1516 			scan_block((void *)area->start,  in scan_object()
1517 				   (void *)(area->start + area->size),  in scan_object()
1521 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1538 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1545 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1549 		list_del(&object->gray_list);  in scan_gray_list()
1568 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1570 	object->del_state |= DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1578 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1579 		list_del_rcu(&object->object_list);  in kmemleak_cond_resched()
1580 	object->del_state &= ~DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1595 	int __maybe_unused i;  in kmemleak_scan()  local
1603 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1609 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1610 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1611 				 atomic_read(&object->use_count));  in kmemleak_scan()
1617 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1618 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1619 			unsigned long phys = object->pointer;  in kmemleak_scan()
1622 			    PHYS_PFN(phys + object->size) >= max_low_pfn)  in kmemleak_scan()
1627 		object->count = 0;  in kmemleak_scan()
1629 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1631 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1639 	/* per-cpu sections scanning */  in kmemleak_scan()
1640 	for_each_possible_cpu(i)  in kmemleak_scan()
1641 		scan_large_block(__per_cpu_start + per_cpu_offset(i),  in kmemleak_scan()
1642 				 __per_cpu_end + per_cpu_offset(i));  in kmemleak_scan()
1650 		unsigned long start_pfn = zone->zone_start_pfn;  in kmemleak_scan()
1713 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1714 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1717 			object->count = object->min_count;  in kmemleak_scan()
1718 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1720 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1725 	 * Re-scan the gray list for modified unreferenced objects.  in kmemleak_scan()
1750 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1752 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1753 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1760 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1782 	set_user_nice(current, 10);  in kmemleak_scan_thread()
1854 		if (n-- > 0)  in kmemleak_seq_start()
1912 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1913 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1915 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1938 		return -EINVAL;  in dump_str_object_info()
1942 		return -EINVAL;  in dump_str_object_info()
1945 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1947 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1965 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1966 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1969 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
1979  * File write operation to configure kmemleak at run-time. The following
1981  *   off	- disable kmemleak (irreversible)
1982  *   stack=on	- enable the task stacks scanning
1983  *   stack=off	- disable the tasks stacks scanning
1984  *   scan=on	- start the automatic memory scanning thread
1985  *   scan=off	- stop the automatic memory scanning thread
1986  *   scan=...	- set the automatic memory scanning period in seconds (0 to
1988  *   scan	- trigger a memory scan
1989  *   clear	- mark all current reported unreferenced kmemleak objects as
1992  *   dump=...	- dump information about the object found at the given address
2001 	buf_size = min(size, (sizeof(buf) - 1));  in kmemleak_write()
2003 		return -EFAULT;  in kmemleak_write()
2019 		ret = -EPERM;  in kmemleak_write()
2055 		ret = -EINVAL;  in kmemleak_write()
2136 	pr_info("Kernel memory leak detector disabled\n");  in kmemleak_disable()
2140  * Allow boot-time kmemleak disabling (enabled by default).
2145 		return -EINVAL;  in kmemleak_boot_config()
2153 		return -EINVAL;  in kmemleak_boot_config()
2180 	create_object((unsigned long)_sdata, _edata - _sdata,  in kmemleak_init()
2182 	create_object((unsigned long)__bss_start, __bss_stop - __bss_start,  in kmemleak_init()
2187 			      __end_ro_after_init - __start_ro_after_init,  in kmemleak_init()
2205 		 * two clean-up threads but serialized by scan_mutex.  in kmemleak_late_init()
2208 		return -ENOMEM;  in kmemleak_late_init()
2217 	pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",  in kmemleak_late_init()