Lines Matching full:cpu
5 * Based on arch/x86/kernel/cpu/intel_cacheinfo.c
15 #include <linux/cpu.h>
24 /* pointer to per cpu cacheinfo */
26 #define ci_cacheinfo(cpu) (&per_cpu(ci_cpu_cacheinfo, cpu)) argument
27 #define cache_leaves(cpu) (ci_cacheinfo(cpu)->num_leaves) argument
28 #define per_cpu_cacheinfo(cpu) (ci_cacheinfo(cpu)->info_list) argument
29 #define per_cpu_cacheinfo_idx(cpu, idx) \ argument
30 (per_cpu_cacheinfo(cpu) + (idx))
35 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu) in get_cpu_cacheinfo() argument
37 return ci_cacheinfo(cpu); in get_cpu_cacheinfo()
58 bool last_level_cache_is_valid(unsigned int cpu) in last_level_cache_is_valid() argument
62 if (!cache_leaves(cpu) || !per_cpu_cacheinfo(cpu)) in last_level_cache_is_valid()
65 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); in last_level_cache_is_valid()
187 static bool match_cache_node(struct device_node *cpu, in match_cache_node() argument
190 struct device_node *prev, *cache = of_find_next_cache_node(cpu); in match_cache_node()
213 struct device_node *cpu; in cache_of_set_id() local
216 for_each_of_cpu_node(cpu) { in cache_of_set_id()
217 u64 id = of_get_cpu_hwid(cpu, 0); in cache_of_set_id()
221 of_node_put(cpu); in cache_of_set_id()
225 if (match_cache_node(cpu, cache_node)) in cache_of_set_id()
253 static int cache_setup_of_node(unsigned int cpu) in cache_setup_of_node() argument
258 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu); in cache_setup_of_node()
260 pr_err("Failed to find cpu%d device node\n", cpu); in cache_setup_of_node()
268 while (index < cache_leaves(cpu)) { in cache_setup_of_node()
269 this_leaf = per_cpu_cacheinfo_idx(cpu, index); in cache_setup_of_node()
281 if (index != cache_leaves(cpu)) /* not all OF nodes populated */ in cache_setup_of_node()
327 int init_of_cache_level(unsigned int cpu) in init_of_cache_level() argument
329 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); in init_of_cache_level()
330 struct device_node *np __free(device_node) = of_cpu_device_node_get(cpu); in init_of_cache_level()
365 static inline int cache_setup_of_node(unsigned int cpu) { return 0; } in cache_setup_of_node() argument
366 int init_of_cache_level(unsigned int cpu) { return 0; } in init_of_cache_level() argument
369 int __weak cache_setup_acpi(unsigned int cpu) in cache_setup_acpi() argument
376 static int cache_setup_properties(unsigned int cpu) in cache_setup_properties() argument
381 ret = cache_setup_of_node(cpu); in cache_setup_properties()
383 ret = cache_setup_acpi(cpu); in cache_setup_properties()
392 static int cache_shared_cpu_map_setup(unsigned int cpu) in cache_shared_cpu_map_setup() argument
394 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); in cache_shared_cpu_map_setup()
407 if (!last_level_cache_is_valid(cpu) && !use_arch_info) { in cache_shared_cpu_map_setup()
408 ret = cache_setup_properties(cpu); in cache_shared_cpu_map_setup()
413 for (index = 0; index < cache_leaves(cpu); index++) { in cache_shared_cpu_map_setup()
416 this_leaf = per_cpu_cacheinfo_idx(cpu, index); in cache_shared_cpu_map_setup()
418 cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); in cache_shared_cpu_map_setup()
420 if (i == cpu || !per_cpu_cacheinfo(i)) in cache_shared_cpu_map_setup()
435 cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); in cache_shared_cpu_map_setup()
446 /* shared_cpu_map is now populated for the cpu */ in cache_shared_cpu_map_setup()
451 static void cache_shared_cpu_map_remove(unsigned int cpu) in cache_shared_cpu_map_remove() argument
453 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); in cache_shared_cpu_map_remove()
457 for (index = 0; index < cache_leaves(cpu); index++) { in cache_shared_cpu_map_remove()
458 this_leaf = per_cpu_cacheinfo_idx(cpu, index); in cache_shared_cpu_map_remove()
460 if (sibling == cpu || !per_cpu_cacheinfo(sibling)) in cache_shared_cpu_map_remove()
476 cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map); in cache_shared_cpu_map_remove()
484 /* cpu is no longer populated in the shared map */ in cache_shared_cpu_map_remove()
488 static void free_cache_attributes(unsigned int cpu) in free_cache_attributes() argument
490 if (!per_cpu_cacheinfo(cpu)) in free_cache_attributes()
493 cache_shared_cpu_map_remove(cpu); in free_cache_attributes()
496 int __weak early_cache_level(unsigned int cpu) in early_cache_level() argument
501 int __weak init_cache_level(unsigned int cpu) in init_cache_level() argument
506 int __weak populate_cache_leaves(unsigned int cpu) in populate_cache_leaves() argument
511 static inline int allocate_cache_info(int cpu) in allocate_cache_info() argument
513 per_cpu_cacheinfo(cpu) = kzalloc_objs(struct cacheinfo, in allocate_cache_info()
514 cache_leaves(cpu), GFP_ATOMIC); in allocate_cache_info()
515 if (!per_cpu_cacheinfo(cpu)) { in allocate_cache_info()
516 cache_leaves(cpu) = 0; in allocate_cache_info()
523 int fetch_cache_info(unsigned int cpu) in fetch_cache_info() argument
525 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); in fetch_cache_info()
530 ret = init_of_cache_level(cpu); in fetch_cache_info()
532 ret = acpi_get_cache_info(cpu, &levels, &split_levels); in fetch_cache_info()
545 if (ret || !cache_leaves(cpu)) { in fetch_cache_info()
546 ret = early_cache_level(cpu); in fetch_cache_info()
550 if (!cache_leaves(cpu)) in fetch_cache_info()
556 return allocate_cache_info(cpu); in fetch_cache_info()
559 static inline int init_level_allocate_ci(unsigned int cpu) in init_level_allocate_ci() argument
561 unsigned int early_leaves = cache_leaves(cpu); in init_level_allocate_ci()
564 * via fetch_cache_info() and this also gets called as CPU hotplug in init_level_allocate_ci()
573 if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels) in init_level_allocate_ci()
576 if (init_cache_level(cpu) || !cache_leaves(cpu)) in init_level_allocate_ci()
582 * (e.g. as CPU hotplug callbacks). in init_level_allocate_ci()
584 ci_cacheinfo(cpu)->early_ci_levels = false; in init_level_allocate_ci()
590 if (cache_leaves(cpu) <= early_leaves && per_cpu_cacheinfo(cpu)) in init_level_allocate_ci()
593 kfree(per_cpu_cacheinfo(cpu)); in init_level_allocate_ci()
594 return allocate_cache_info(cpu); in init_level_allocate_ci()
597 int detect_cache_attributes(unsigned int cpu) in detect_cache_attributes() argument
601 ret = init_level_allocate_ci(cpu); in detect_cache_attributes()
607 * update the cpu map. in detect_cache_attributes()
609 if (!last_level_cache_is_valid(cpu)) { in detect_cache_attributes()
614 ret = populate_cache_leaves(cpu); in detect_cache_attributes()
624 ret = cache_shared_cpu_map_setup(cpu); in detect_cache_attributes()
626 pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu); in detect_cache_attributes()
633 free_cache_attributes(cpu); in detect_cache_attributes()
639 #define per_cpu_cache_dev(cpu) (per_cpu(ci_cache_dev, cpu)) argument
645 #define per_cpu_index_dev(cpu) (per_cpu(ci_index_dev, cpu)) argument
646 #define per_cache_index_dev(cpu, idx) ((per_cpu_index_dev(cpu))[idx]) argument
854 /* Add/Remove cache interface for CPU device */
855 static void cpu_cache_sysfs_exit(unsigned int cpu) in cpu_cache_sysfs_exit() argument
860 if (per_cpu_index_dev(cpu)) { in cpu_cache_sysfs_exit()
861 for (i = 0; i < cache_leaves(cpu); i++) { in cpu_cache_sysfs_exit()
862 ci_dev = per_cache_index_dev(cpu, i); in cpu_cache_sysfs_exit()
867 kfree(per_cpu_index_dev(cpu)); in cpu_cache_sysfs_exit()
868 per_cpu_index_dev(cpu) = NULL; in cpu_cache_sysfs_exit()
870 device_unregister(per_cpu_cache_dev(cpu)); in cpu_cache_sysfs_exit()
871 per_cpu_cache_dev(cpu) = NULL; in cpu_cache_sysfs_exit()
874 static int cpu_cache_sysfs_init(unsigned int cpu) in cpu_cache_sysfs_init() argument
876 struct device *dev = get_cpu_device(cpu); in cpu_cache_sysfs_init()
878 if (per_cpu_cacheinfo(cpu) == NULL) in cpu_cache_sysfs_init()
881 per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache"); in cpu_cache_sysfs_init()
882 if (IS_ERR(per_cpu_cache_dev(cpu))) in cpu_cache_sysfs_init()
883 return PTR_ERR(per_cpu_cache_dev(cpu)); in cpu_cache_sysfs_init()
886 per_cpu_index_dev(cpu) = kzalloc_objs(struct device *, in cpu_cache_sysfs_init()
887 cache_leaves(cpu)); in cpu_cache_sysfs_init()
888 if (unlikely(per_cpu_index_dev(cpu) == NULL)) in cpu_cache_sysfs_init()
894 cpu_cache_sysfs_exit(cpu); in cpu_cache_sysfs_init()
898 static int cache_add_dev(unsigned int cpu) in cache_add_dev() argument
906 rc = cpu_cache_sysfs_init(cpu); in cache_add_dev()
910 parent = per_cpu_cache_dev(cpu); in cache_add_dev()
911 for (i = 0; i < cache_leaves(cpu); i++) { in cache_add_dev()
912 this_leaf = per_cpu_cacheinfo_idx(cpu, i); in cache_add_dev()
924 per_cache_index_dev(cpu, i) = ci_dev; in cache_add_dev()
926 cpumask_set_cpu(cpu, &cache_dev_map); in cache_add_dev()
930 cpu_cache_sysfs_exit(cpu); in cache_add_dev()
934 static unsigned int cpu_map_shared_cache(bool online, unsigned int cpu, in cpu_map_shared_cache() argument
940 if (!last_level_cache_is_valid(cpu)) in cpu_map_shared_cache()
943 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); in cpu_map_shared_cache()
953 /* shared_cpu_map of offlined CPU will be cleared, so use sibling map */ in cpu_map_shared_cache()
955 if (sibling == cpu || !last_level_cache_is_valid(sibling)) in cpu_map_shared_cache()
966 * Calculate the size of the per-CPU data cache slice. This can be
968 * by one CPU under ideal circumstances. UNIFIED caches are counted
973 * available, we just use the size of the per-CPU slice of LLC to make
976 static void update_per_cpu_data_slice_size_cpu(unsigned int cpu) in update_per_cpu_data_slice_size_cpu() argument
982 if (!last_level_cache_is_valid(cpu)) in update_per_cpu_data_slice_size_cpu()
985 ci = ci_cacheinfo(cpu); in update_per_cpu_data_slice_size_cpu()
986 llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); in update_per_cpu_data_slice_size_cpu()
996 static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu, in update_per_cpu_data_slice_size() argument
1002 if (!cpu_online && icpu == cpu) in update_per_cpu_data_slice_size()
1009 static int cacheinfo_cpu_online(unsigned int cpu) in cacheinfo_cpu_online() argument
1011 int rc = detect_cache_attributes(cpu); in cacheinfo_cpu_online()
1016 rc = cache_add_dev(cpu); in cacheinfo_cpu_online()
1019 if (cpu_map_shared_cache(true, cpu, &cpu_map)) in cacheinfo_cpu_online()
1020 update_per_cpu_data_slice_size(true, cpu, cpu_map); in cacheinfo_cpu_online()
1023 free_cache_attributes(cpu); in cacheinfo_cpu_online()
1027 static int cacheinfo_cpu_pre_down(unsigned int cpu) in cacheinfo_cpu_pre_down() argument
1032 nr_shared = cpu_map_shared_cache(false, cpu, &cpu_map); in cacheinfo_cpu_pre_down()
1033 if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map)) in cacheinfo_cpu_pre_down()
1034 cpu_cache_sysfs_exit(cpu); in cacheinfo_cpu_pre_down()
1036 free_cache_attributes(cpu); in cacheinfo_cpu_pre_down()
1038 update_per_cpu_data_slice_size(false, cpu, cpu_map); in cacheinfo_cpu_pre_down()