Lines Matching +full:i +full:- +full:leak +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0-only
9 * Documentation/dev-tools/kmemleak.rst.
12 * ----------------
16 * - kmemleak_lock (raw_spinlock_t): protects the object_list as well as
21 * red black trees used to look-up metadata based on a pointer to the
26 * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
33 * - scan_mutex (mutex): ensures that only one thread may scan the memory for
46 * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48 * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
125 #define KMEMLEAK_BLACK -1
130 * object->lock. Insertions or deletions from object_list, gray_list or
132 * the notes on locking above). These objects are reference-counted
149 /* minimum number of a pointers found before it is considered leak */
159 pid_t pid; /* pid of the current task */
173 /* flag set for per-CPU pointers */
193 /* the list of gray-colored objects (see color_gray comment below) */
227 /* minimum and maximum address that may be valid per-CPU pointers */
293 * with the object->lock held.
298 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
301 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
304 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
305 ptr = (const u8 *)this_cpu_ptr((void __percpu *)object->pointer); in hex_dump_object()
308 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
310 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
323 * - white - orphan object, not enough references to it (count < min_count)
324 * - gray - not orphan, not marked as false positive (min_count == 0) or
326 * - black - ignore, it doesn't contain references (e.g. text section)
327 * (min_count == -1). No function defined for this color.
328 * Newly created objects don't have any color assigned (object->count == -1)
333 return object->count != KMEMLEAK_BLACK && in color_white()
334 object->count < object->min_count; in color_white()
339 return object->min_count != KMEMLEAK_BLACK && in color_gray()
340 object->count >= object->min_count; in color_gray()
350 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
351 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
357 if (object->flags & OBJECT_PHYS) in __object_type_str()
359 if (object->flags & OBJECT_PERCPU) in __object_type_str()
366 * print_unreferenced function must be called with the object->lock held.
371 int i; in print_unreferenced() local
375 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
378 object->pointer, object->size); in print_unreferenced()
380 object->comm, object->pid, object->jiffies); in print_unreferenced()
382 warn_or_seq_printf(seq, " backtrace (crc %x):\n", object->checksum); in print_unreferenced()
384 for (i = 0; i < nr_entries; i++) { in print_unreferenced()
385 void *ptr = (void *)entries[i]; in print_unreferenced()
393 * the object->lock held.
398 __object_type_str(object), object->pointer, object->size); in dump_object_info()
400 object->comm, object->pid, object->jiffies); in dump_object_info()
401 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
402 pr_notice(" count = %d\n", object->count); in dump_object_info()
403 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
404 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
406 if (object->trace_handle) in dump_object_info()
407 stack_depot_print(object->trace_handle); in dump_object_info()
420 * Look-up a memory block metadata (kmemleak_object) in the object search
428 struct rb_node *rb = object_tree(objflags)->rb_node; in __lookup_object()
436 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
439 rb = object->rb_node.rb_left; in __lookup_object()
440 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
441 rb = object->rb_node.rb_right; in __lookup_object()
454 /* Look-up a kmemleak object which allocated with virtual address. */
468 return atomic_inc_not_zero(&object->use_count); in get_object()
492 list_del(&object->object_list); in mem_pool_alloc()
494 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
516 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
534 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
535 hlist_del(&area->node); in free_object_rcu()
543 * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
545 * recursive call to the kernel allocator. Lock-less RCU object_list traversal
550 if (!atomic_dec_and_test(&object->use_count)) in put_object()
554 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
562 call_rcu(&object->rcu, free_object_rcu); in put_object()
564 free_object_rcu(&object->rcu); in put_object()
601 rb_erase(&object->rb_node, object_tree(object->flags)); in __remove_object()
602 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
603 list_del_rcu(&object->object_list); in __remove_object()
604 object->del_state |= DELSTATE_REMOVED; in __remove_object()
668 INIT_LIST_HEAD(&object->object_list); in __alloc_object()
669 INIT_LIST_HEAD(&object->gray_list); in __alloc_object()
670 INIT_HLIST_HEAD(&object->area_list); in __alloc_object()
671 raw_spin_lock_init(&object->lock); in __alloc_object()
672 atomic_set(&object->use_count, 1); in __alloc_object()
673 object->excess_ref = 0; in __alloc_object()
674 object->count = 0; /* white color initially */ in __alloc_object()
675 object->checksum = 0; in __alloc_object()
676 object->del_state = 0; in __alloc_object()
680 object->pid = 0; in __alloc_object()
681 strscpy(object->comm, "hardirq"); in __alloc_object()
683 object->pid = 0; in __alloc_object()
684 strscpy(object->comm, "softirq"); in __alloc_object()
686 object->pid = current->pid; in __alloc_object()
690 * dependency issues with current->alloc_lock. In the worst in __alloc_object()
693 strscpy(object->comm, current->comm); in __alloc_object()
697 object->trace_handle = set_track_prepare(); in __alloc_object()
711 object->flags = OBJECT_ALLOCATED | objflags; in __link_object()
712 object->pointer = ptr; in __link_object()
713 object->size = kfence_ksize((void *)ptr) ?: size; in __link_object()
714 object->min_count = min_count; in __link_object()
715 object->jiffies = jiffies; in __link_object()
720 * address. And update min_percpu_addr max_percpu_addr for per-CPU in __link_object()
730 link = &object_tree(objflags)->rb_node; in __link_object()
735 untagged_objp = (unsigned long)kasan_reset_tag((void *)parent->pointer); in __link_object()
737 link = &parent->rb_node.rb_left; in __link_object()
738 else if (untagged_objp + parent->size <= untagged_ptr) in __link_object()
739 link = &parent->rb_node.rb_right; in __link_object()
744 * No need for parent->lock here since "parent" cannot in __link_object()
748 return -EEXIST; in __link_object()
751 rb_link_node(&object->rb_node, rb_parent, link); in __link_object()
752 rb_insert_color(&object->rb_node, object_tree(objflags)); in __link_object()
753 list_add_tail_rcu(&object->object_list, &object_list); in __link_object()
794 /* Create kmemleak object corresponding to a per-CPU allocation. */
808 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
809 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
815 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
816 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
817 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
874 start = object->pointer; in delete_object_part()
875 end = object->pointer + object->size; in delete_object_part()
877 !__link_object(object_l, start, ptr - start, in delete_object_part()
878 object->min_count, objflags)) in delete_object_part()
881 !__link_object(object_r, ptr + size, end - ptr - size, in delete_object_part()
882 object->min_count, objflags)) in delete_object_part()
899 object->min_count = color; in __paint_it()
901 object->flags |= OBJECT_NO_SCAN; in __paint_it()
908 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
910 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
930 * Mark an object permanently as gray-colored so that it can no longer be
931 * reported as a leak. This is used in general to mark a false positive.
939 * Mark the object as black-colored so that it is ignored from scans and
949 * be reported as a leak during the next scan until its checksum is updated.
963 raw_spin_lock_irqsave(&object->lock, flags); in reset_checksum()
964 object->checksum = 0; in reset_checksum()
965 raw_spin_unlock_irqrestore(&object->lock, flags); in reset_checksum()
989 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
995 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
999 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
1003 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
1004 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
1011 INIT_HLIST_NODE(&area->node); in add_scan_area()
1012 area->start = ptr; in add_scan_area()
1013 area->size = size; in add_scan_area()
1015 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
1017 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
1039 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
1040 object->excess_ref = excess_ref; in object_set_excess_ref()
1041 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
1061 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
1062 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
1063 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
1068 * kmemleak_alloc - register a newly allocated object
1073 * the object is reported as a memory leak. If @min_count is 0,
1074 * the object is never reported as a leak. If @min_count is -1,
1075 * the object is ignored (not scanned and not reported as a leak)
1092 * kmemleak_alloc_percpu - register a newly allocated __percpu object
1111 * kmemleak_vmalloc - register a newly vmalloc'ed object
1128 create_object((unsigned long)area->addr, size, 2, gfp); in kmemleak_vmalloc()
1130 (unsigned long)area->addr); in kmemleak_vmalloc()
1136 * kmemleak_free - unregister a previously registered object
1152 * kmemleak_free_part - partially unregister a previously registered object
1170 * kmemleak_free_percpu - unregister a previously registered __percpu object
1186 * kmemleak_update_trace - update object allocation stack trace
1213 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1214 object->trace_handle = trace_handle; in kmemleak_update_trace()
1215 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1222 * kmemleak_not_leak - mark an allocated object as false positive
1226 * be reported as leak and always be scanned.
1238 * kmemleak_transient_leak - mark an allocated object as transient false positive
1242 * reported as a leak temporarily. This may happen, for example, if the object
1243 * is part of a singly linked list and the ->next reference to it is changed.
1255 * kmemleak_ignore - ignore an allocated object
1259 * ignored (not scanned and not reported as a leak). This is usually done when
1260 * it is known that the corresponding block is not a leak and does not contain
1273 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1293 * kmemleak_no_scan - do not scan an allocated object
1311 * kmemleak_alloc_phys - similar to kmemleak_alloc but taking a physical
1331 * kmemleak_free_part_phys - similar to kmemleak_free_part but taking a
1347 * kmemleak_ignore_phys - similar to kmemleak_ignore but taking a physical
1365 u32 old_csum = object->checksum; in update_checksum()
1367 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1372 if (object->flags & OBJECT_PERCPU) { in update_checksum()
1375 object->checksum = 0; in update_checksum()
1377 void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); in update_checksum()
1379 object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); in update_checksum()
1382 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1387 return object->checksum != old_csum; in update_checksum()
1391 * Update an object's references. object->lock must be held by the caller.
1396 /* non-orphan, ignored or new */ in update_refs()
1406 object->count++; in update_refs()
1410 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1432 * object->use_count cannot be dropped to 0 while the object in pointer_update_refs()
1444 * Avoid the lockdep recursive warning on object->lock being in pointer_update_refs()
1448 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1451 excess_ref = object->excess_ref; in pointer_update_refs()
1457 raw_spin_unlock(&object->lock); in pointer_update_refs()
1466 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1468 raw_spin_unlock(&object->lock); in pointer_update_refs()
1485 if (current->mm) in scan_should_stop()
1486 return signal_pending(current); in scan_should_stop()
1502 unsigned long *end = _end - (BYTES_PER_POINTER - 1); in scan_block()
1541 * that object->use_count >= 1.
1549 * Once the object->lock is acquired, the corresponding memory block in scan_object()
1552 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1553 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1555 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1559 if (object->flags & OBJECT_PERCPU) { in scan_object()
1563 void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu); in scan_object()
1564 void *end = start + object->size; in scan_object()
1568 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1570 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1571 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1574 } else if (hlist_empty(&object->area_list) || in scan_object()
1575 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1576 void *start = object->flags & OBJECT_PHYS ? in scan_object()
1577 __va((phys_addr_t)object->pointer) : in scan_object()
1578 (void *)object->pointer; in scan_object()
1579 void *end = start + object->size; in scan_object()
1590 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1592 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1593 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1595 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1596 scan_block((void *)area->start, in scan_object()
1597 (void *)(area->start + area->size), in scan_object()
1601 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1618 while (&object->gray_list != &gray_list) { in scan_gray_list()
1625 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1629 list_del(&object->gray_list); in scan_gray_list()
1648 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1650 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1658 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1659 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1660 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1675 int __maybe_unused i; in kmemleak_scan() local
1683 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1689 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1690 pr_debug("object->use_count = %d\n", in kmemleak_scan()
1691 atomic_read(&object->use_count)); in kmemleak_scan()
1697 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1698 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1699 unsigned long phys = object->pointer; in kmemleak_scan()
1702 PHYS_PFN(phys + object->size) > max_low_pfn) in kmemleak_scan()
1707 object->count = 0; in kmemleak_scan()
1709 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1711 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1719 /* per-cpu sections scanning */ in kmemleak_scan()
1720 for_each_possible_cpu(i) in kmemleak_scan()
1721 scan_large_block(__per_cpu_start + per_cpu_offset(i), in kmemleak_scan()
1722 __per_cpu_end + per_cpu_offset(i)); in kmemleak_scan()
1730 unsigned long start_pfn = zone->zone_start_pfn; in kmemleak_scan()
1793 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1794 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1797 object->count = object->min_count; in kmemleak_scan()
1798 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1800 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1805 * Re-scan the gray list for modified unreferenced objects. in kmemleak_scan()
1830 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1832 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1833 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1840 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1862 set_user_nice(current, 10); in kmemleak_scan_thread()
1934 if (n-- > 0) in kmemleak_seq_start()
1992 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
1993 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
1995 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
2020 raw_spin_lock_irqsave(&object->lock, flags); in __dump_str_object_info()
2022 raw_spin_unlock_irqrestore(&object->lock, flags); in __dump_str_object_info()
2035 return -EINVAL; in dump_str_object_info()
2043 return -EINVAL; in dump_str_object_info()
2061 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
2062 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
2065 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
2075 * File write operation to configure kmemleak at run-time. The following
2077 * off - disable kmemleak (irreversible)
2078 * stack=on - enable the task stacks scanning
2079 * stack=off - disable the tasks stacks scanning
2080 * scan=on - start the automatic memory scanning thread
2081 * scan=off - stop the automatic memory scanning thread
2082 * scan=... - set the automatic memory scanning period in seconds (0 to
2084 * scan - trigger a memory scan
2085 * clear - mark all current reported unreferenced kmemleak objects as
2088 * dump=... - dump information about the object found at the given address
2097 buf_size = min(size, (sizeof(buf) - 1)); in kmemleak_write()
2099 return -EFAULT; in kmemleak_write()
2115 ret = -EPERM; in kmemleak_write()
2151 ret = -EINVAL; in kmemleak_write()
2232 pr_info("Kernel memory leak detector disabled\n"); in kmemleak_disable()
2236 * Allow boot-time kmemleak disabling (enabled by default).
2241 return -EINVAL; in kmemleak_boot_config()
2249 return -EINVAL; in kmemleak_boot_config()
2276 create_object((unsigned long)_sdata, _edata - _sdata, in kmemleak_init()
2278 create_object((unsigned long)__bss_start, __bss_stop - __bss_start, in kmemleak_init()
2283 __end_ro_after_init - __start_ro_after_init, in kmemleak_init()
2301 * two clean-up threads but serialized by scan_mutex. in kmemleak_late_init()
2304 return -ENOMEM; in kmemleak_late_init()
2313 pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n", in kmemleak_late_init()