1 // SPDX-License-Identifier: GPL-2.0-only
3  * arch/arm/mm/cache-l2x0.c - L210/L220/L310 cache controller support
20 #include <asm/hardware/cache-l2x0.h>
21 #include <asm/hardware/cache-aurora-l2.h>
22 #include "cache-tauros3.h"
54 static inline void l2c_wait_mask(void __iomem *reg, unsigned long mask)  in l2c_wait_mask()  argument
57 	while (readl_relaxed(reg) & mask)  in l2c_wait_mask()
62  * By default, we write directly to secure registers.  Platforms must
63  * override this if they are running non-secure.
65 static void l2c_write_sec(unsigned long val, void __iomem *base, unsigned reg)  in l2c_write_sec()  argument
67 	if (val == readl_relaxed(base + reg))  in l2c_write_sec()
70 		outer_cache.write_sec(val, reg);  in l2c_write_sec()
72 		writel_relaxed(val, base + reg);  in l2c_write_sec()
77  * register be written due to a work-around, as platforms running
78  * in non-secure mode may not be able to access this register.
85 static void __l2c_op_way(void __iomem *reg)  in __l2c_op_way()  argument
87 	writel_relaxed(l2x0_way_mask, reg);  in __l2c_op_way()
88 	l2c_wait_mask(reg, l2x0_way_mask);  in __l2c_op_way()
119 		l2x0_data->configure(base);  in l2c_enable()
121 	l2x0_data->unlock(base, num_lock);  in l2c_enable()
154 		l2c_enable(base, l2x0_data->num_lock);  in l2c_resume()
160  * L2C-210 specific code.
162  * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
171  * we use sync_reg_offset here so we can share some of this with L2C-310.
178 static void __l2c210_op_pa_range(void __iomem *reg, unsigned long start,  in __l2c210_op_pa_range()  argument
182 		writel_relaxed(start, reg);  in __l2c210_op_pa_range()
191 	if (start & (CACHE_LINE_SIZE - 1)) {  in l2c210_inv_range()
192 		start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_inv_range()
197 	if (end & (CACHE_LINE_SIZE - 1)) {  in l2c210_inv_range()
198 		end &= ~(CACHE_LINE_SIZE - 1);  in l2c210_inv_range()
210 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_clean_range()
219 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c210_flush_range()
240 	.type = "L2C-210",
259  * L2C-220 specific code.
263  * imprecise abort.)  Never uses sync_reg_offset, so we hard-code the
266  * However, we can re-use the l2c210_resume call.
274 static void l2c220_op_way(void __iomem *base, unsigned reg)  in l2c220_op_way()  argument
279 	__l2c_op_way(base + reg);  in l2c220_op_way()
284 static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start,  in l2c220_op_pa_range()  argument
290 		unsigned long blk_end = start + min(end - start, 4096UL);  in l2c220_op_pa_range()
293 			l2c_wait_mask(reg, 1);  in l2c220_op_pa_range()
294 			writel_relaxed(start, reg);  in l2c220_op_pa_range()
313 	if ((start | end) & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
314 		if (start & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
315 			start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_inv_range()
320 		if (end & (CACHE_LINE_SIZE - 1)) {  in l2c220_inv_range()
321 			end &= ~(CACHE_LINE_SIZE - 1);  in l2c220_inv_range()
339 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_clean_range()
340 	if ((end - start) >= l2x0_size) {  in l2c220_clean_range()
358 	start &= ~(CACHE_LINE_SIZE - 1);  in l2c220_flush_range()
359 	if ((end - start) >= l2x0_size) {  in l2c220_flush_range()
389 	 * Always enable non-secure access to the lockdown registers -  in l2c220_enable()
405 	.type = "L2C-220",
424  * L2C-310 specific code.
426  * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
439  * 588369: PL310 R0P0->R1P0, fixed R2P0.
447  * 727915: PL310 R2P0->R3P0, fixed R3P1.
453  * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
454  *	Affects: 8x64-bit (double fill) line fetches
463  * 769419: PL310 R0P0->R3P1, fixed R3P2.
471 	if ((start | end) & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
478 		if (start & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
479 			start &= ~(CACHE_LINE_SIZE - 1);  in l2c310_inv_range_erratum()
485 		if (end & (CACHE_LINE_SIZE - 1)) {  in l2c310_inv_range_erratum()
486 			end &= ~(CACHE_LINE_SIZE - 1);  in l2c310_inv_range_erratum()
507 		unsigned long blk_end = start + min(end - start, 4096UL);  in l2c310_flush_range_erratum()
616 			pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");  in l2c310_enable()
618 			pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");  in l2c310_enable()
627 		pr_debug("Cortex-A9 ACR=0x%08x\n", acr);  in l2c310_enable()
630 			pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");  in l2c310_enable()
633 			pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");  in l2c310_enable()
637 			pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");  in l2c310_enable()
640 		pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");  in l2c310_enable()
645 	 * Always enable non-secure access to the lockdown registers -  in l2c310_enable()
659 		pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",  in l2c310_enable()
670 		pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n",  in l2c310_enable()
691 	    fns->inv_range == l2c210_inv_range) {  in l2c310_fixup()
692 		fns->inv_range = l2c310_inv_range_erratum;  in l2c310_fixup()
693 		fns->flush_range = l2c310_flush_range_erratum;  in l2c310_fixup()
700 		fns->flush_all = l2c310_flush_all_erratum;  in l2c310_fixup()
726 		pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");  in l2c310_fixup()
736 	 * If full-line-of-zeros is enabled, we must first disable it in the  in l2c310_disable()
737 	 * Cortex-A9 auxiliary control register before disabling the L2 cache.  in l2c310_disable()
749 	/* Re-enable full-line-of-zeros for Cortex-A9 */  in l2c310_resume()
761 	.type = "L2C-310",
789 	 * context from callers can access the structure.  in __l2c_init()
793 		return -ENOMEM;  in __l2c_init()
808 		pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",  in __l2c_init()
838 	l2x0_way_mask = (1 << ways) - 1;  in __l2c_init()
850 	l2x0_size = ways * (data->way_size_0 << way_size_bits);  in __l2c_init()
852 	fns = data->outer_cache;  in __l2c_init()
855 	if (data->fixup)  in __l2c_init()
856 		data->fixup(l2x0_base, cache_id, &fns);  in __l2c_init()
864 	 * in non-secure mode accessing the below registers will fault.  in __l2c_init()
869 		data->enable(l2x0_base, data->num_lock);  in __l2c_init()
878 	if (data->save)  in __l2c_init()
879 		data->save(l2x0_base);  in __l2c_init()
881 	/* Re-read it in case some bits are reserved. */  in __l2c_init()
885 		data->type, ways, l2x0_size >> 10);  in __l2c_init()
887 		data->type, cache_id, aux);  in __l2c_init()
919 	if (data->save)  in l2x0_init()
920 		data->save(l2x0_base);  in l2x0_init()
933  * l2x0_cache_size_of_parse() - read cache size parameters from DT
935  * @aux_val: pointer to machine-supplied auxilary register value, to
937  * @aux_mask: pointer to machine-supplied auxilary register mask, to
954 	of_property_read_u32(np, "cache-size", &cache_size);  in l2x0_cache_size_of_parse()
955 	of_property_read_u32(np, "cache-sets", &sets);  in l2x0_cache_size_of_parse()
956 	of_property_read_u32(np, "cache-block-size", &block_size);  in l2x0_cache_size_of_parse()
957 	of_property_read_u32(np, "cache-line-size", &line_size);  in l2x0_cache_size_of_parse()
960 		return -ENODEV;  in l2x0_cache_size_of_parse()
995 		return -EINVAL;  in l2x0_cache_size_of_parse()
1007 	 * 512KB -> 6, 256KB -> 5, ... 16KB -> 1  in l2x0_cache_size_of_parse()
1009 	way_size_bits = ilog2(way_size >> 10) - 3;  in l2x0_cache_size_of_parse()
1013 		return -EINVAL;  in l2x0_cache_size_of_parse()
1036 	of_property_read_u32(np, "arm,tag-latency", &tag);  in l2x0_of_parse()
1039 		val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT;  in l2x0_of_parse()
1042 	of_property_read_u32_array(np, "arm,data-latency",  in l2x0_of_parse()
1047 		val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) |  in l2x0_of_parse()
1048 		       ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT);  in l2x0_of_parse()
1051 	of_property_read_u32(np, "arm,dirty-latency", &dirty);  in l2x0_of_parse()
1054 		val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT;  in l2x0_of_parse()
1057 	if (of_property_read_bool(np, "arm,parity-enable")) {  in l2x0_of_parse()
1060 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in l2x0_of_parse()
1064 	if (of_property_read_bool(np, "arm,shared-override")) {  in l2x0_of_parse()
1087 	.type = "L2C-210",
1107 	.type = "L2C-220",
1138 	of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag));  in l2c310_of_parse()
1141 			L310_LATENCY_CTRL_RD(tag[0] - 1) |  in l2c310_of_parse()
1142 			L310_LATENCY_CTRL_WR(tag[1] - 1) |  in l2c310_of_parse()
1143 			L310_LATENCY_CTRL_SETUP(tag[2] - 1);  in l2c310_of_parse()
1145 	of_property_read_u32_array(np, "arm,data-latency",  in l2c310_of_parse()
1149 			L310_LATENCY_CTRL_RD(data[0] - 1) |  in l2c310_of_parse()
1150 			L310_LATENCY_CTRL_WR(data[1] - 1) |  in l2c310_of_parse()
1151 			L310_LATENCY_CTRL_SETUP(data[2] - 1);  in l2c310_of_parse()
1153 	of_property_read_u32_array(np, "arm,filter-ranges",  in l2c310_of_parse()
1158 		l2x0_saved_regs.filter_start = (filter[0] & ~(SZ_1M - 1))  in l2c310_of_parse()
1175 			pr_err("L2C-310 OF cache associativity %d invalid, only 8 or 16 permitted\n",  in l2c310_of_parse()
1181 	if (of_property_read_bool(np, "arm,shared-override")) {  in l2c310_of_parse()
1186 	if (of_property_read_bool(np, "arm,parity-enable")) {  in l2c310_of_parse()
1189 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in l2c310_of_parse()
1194 	if (of_property_read_bool(np, "arm,early-bresp-disable"))  in l2c310_of_parse()
1197 	if (of_property_read_bool(np, "arm,full-line-zero-disable"))  in l2c310_of_parse()
1202 	ret = of_property_read_u32(np, "arm,double-linefill", &val);  in l2c310_of_parse()
1208 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1209 		pr_err("L2C-310 OF arm,double-linefill property value is missing\n");  in l2c310_of_parse()
1212 	ret = of_property_read_u32(np, "arm,double-linefill-incr", &val);  in l2c310_of_parse()
1218 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1219 		pr_err("L2C-310 OF arm,double-linefill-incr property value is missing\n");  in l2c310_of_parse()
1222 	ret = of_property_read_u32(np, "arm,double-linefill-wrap", &val);  in l2c310_of_parse()
1228 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1229 		pr_err("L2C-310 OF arm,double-linefill-wrap property value is missing\n");  in l2c310_of_parse()
1232 	ret = of_property_read_u32(np, "arm,prefetch-drop", &val);  in l2c310_of_parse()
1238 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1239 		pr_err("L2C-310 OF arm,prefetch-drop property value is missing\n");  in l2c310_of_parse()
1242 	ret = of_property_read_u32(np, "arm,prefetch-offset", &val);  in l2c310_of_parse()
1246 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1247 		pr_err("L2C-310 OF arm,prefetch-offset property value is missing\n");  in l2c310_of_parse()
1250 	ret = of_property_read_u32(np, "prefetch-data", &val);  in l2c310_of_parse()
1260 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1261 		pr_err("L2C-310 OF prefetch-data property value is missing\n");  in l2c310_of_parse()
1264 	ret = of_property_read_u32(np, "prefetch-instr", &val);  in l2c310_of_parse()
1274 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1275 		pr_err("L2C-310 OF prefetch-instr property value is missing\n");  in l2c310_of_parse()
1283 	ret = of_property_read_u32(np, "arm,dynamic-clock-gating", &val);  in l2c310_of_parse()
1287 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1288 		pr_err("L2C-310 OF dynamic-clock-gating property value is missing or invalid\n");  in l2c310_of_parse()
1290 	ret = of_property_read_u32(np, "arm,standby-mode", &val);  in l2c310_of_parse()
1294 	} else if (ret != -EINVAL) {  in l2c310_of_parse()
1295 		pr_err("L2C-310 OF standby-mode property value is missing or invalid\n");  in l2c310_of_parse()
1302 	.type = "L2C-310",
1332 	.type = "L2C-310 Coherent",
1385 	start &= ~(CACHE_LINE_SIZE - 1);  in aurora_pa_range()
1396 		writel_relaxed(range_end - CACHE_LINE_SIZE, base + offset);  in aurora_pa_range()
1493 	of_property_read_u32(np, "cache-id-part",  in aurora_of_parse()
1497 	l2_wt_override = of_property_read_bool(np, "wt-override");  in aurora_of_parse()
1504 	if (of_property_read_bool(np, "marvell,ecc-enable")) {  in aurora_of_parse()
1509 	if (of_property_read_bool(np, "arm,parity-enable")) {  in aurora_of_parse()
1512 	} else if (of_property_read_bool(np, "arm,parity-disable")) {  in aurora_of_parse()
1563  *   1     0x00000000 - 0x3FFFFFFF    0x80000000    VC
1564  *   2     0x40000000 - 0xBFFFFFFF    0x40000000    SYS
1565  *   3     0xC0000000 - 0xFFFFFFFF    0x80000000    VC
1570  * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2)
1571  * 0xC0000000 - 0xC0001000
1629 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_inv_range()
1656 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_clean_range()
1670 	if ((end - start) >= l2x0_size) {  in bcm_flush_range()
1688 		bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR-1));  in bcm_flush_range()
1693 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
1695 	.type = "BCM-L2C-310",
1749 	L2C_ID("arm,l210-cache", of_l2c210_data),
1750 	L2C_ID("arm,l220-cache", of_l2c220_data),
1751 	L2C_ID("arm,pl310-cache", of_l2c310_data),
1752 	L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1753 	L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data),
1754 	L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data),
1755 	L2C_ID("marvell,tauros3-cache", of_tauros3_data),
1757 	L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data),
1772 		return -ENODEV;  in l2x0_of_init()
1775 		return -ENODEV;  in l2x0_of_init()
1779 		return -ENOMEM;  in l2x0_of_init()
1783 	data = of_match_node(l2x0_ids, np)->data;  in l2x0_of_init()
1785 	if (of_device_is_compatible(np, "arm,pl310-cache") &&  in l2x0_of_init()
1786 	    of_property_read_bool(np, "arm,io-coherent"))  in l2x0_of_init()
1791 		pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",  in l2x0_of_init()
1798 	if (!of_property_read_bool(np, "cache-unified"))  in l2x0_of_init()
1801 	if (of_property_read_u32(np, "cache-level", &cache_level))  in l2x0_of_init()
1802 		pr_err("L2C: device tree omits to specify cache-level\n");  in l2x0_of_init()
1807 	nosync = of_property_read_bool(np, "arm,outer-sync-disable");  in l2x0_of_init()
1810 	if (data->save)  in l2x0_of_init()
1811 		data->save(l2x0_base);  in l2x0_of_init()
1815 		if (data->of_parse)  in l2x0_of_init()
1816 			data->of_parse(np, &aux_val, &aux_mask);  in l2x0_of_init()