1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2013 Imagination Technologies
4 * Author: Paul Burton <paul.burton@mips.com>
5 */
6
7#include <linux/init.h>
8#include <asm/addrspace.h>
9#include <asm/asm.h>
10#include <asm/asm-offsets.h>
11#include <asm/asmmacro.h>
12#include <asm/cacheops.h>
13#include <asm/eva.h>
14#include <asm/mipsregs.h>
15#include <asm/mipsmtregs.h>
16#include <asm/pm.h>
17#include <asm/smp-cps.h>
18
19#define GCR_CPC_BASE_OFS	0x0088
20#define GCR_CL_COHERENCE_OFS	0x2008
21#define GCR_CL_ID_OFS		0x2028
22#define  CM3_GCR_Cx_ID_CLUSTER_SHF	8
23#define  CM3_GCR_Cx_ID_CLUSTER_MSK	(0xff << 8)
24#define  CM3_GCR_Cx_ID_CORENUM_SHF	0
25#define  CM3_GCR_Cx_ID_CORENUM_MSK	(0xff << 0)
26
27#define CPC_CL_VC_STOP_OFS	0x2020
28#define CPC_CL_VC_RUN_OFS	0x2028
29
30.extern mips_cm_base
31
32.set noreorder
33
34#ifdef CONFIG_64BIT
35# define STATUS_BITDEPS		ST0_KX
36#else
37# define STATUS_BITDEPS		0
38#endif
39
40#ifdef CONFIG_MIPS_CPS_NS16550
41
42#define DUMP_EXCEP(name)		\
43	PTR_LA	a0, 8f;			\
44	jal	mips_cps_bev_dump;	\
45	 nop;				\
46	TEXT(name)
47
48#else /* !CONFIG_MIPS_CPS_NS16550 */
49
50#define DUMP_EXCEP(name)
51
52#endif /* !CONFIG_MIPS_CPS_NS16550 */
53
54	/*
55	 * Set dest to non-zero if the core supports the MT ASE, else zero. If
56	 * MT is not supported then branch to nomt.
57	 */
58	.macro	has_mt	dest, nomt
59	mfc0	\dest, CP0_CONFIG, 1
60	bgez	\dest, \nomt
61	 mfc0	\dest, CP0_CONFIG, 2
62	bgez	\dest, \nomt
63	 mfc0	\dest, CP0_CONFIG, 3
64	andi	\dest, \dest, MIPS_CONF3_MT
65	beqz	\dest, \nomt
66	 nop
67	.endm
68
69	/*
70	 * Set dest to non-zero if the core supports MIPSr6 multithreading
71	 * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then
72	 * branch to nomt.
73	 */
74	.macro	has_vp	dest, nomt
75	mfc0	\dest, CP0_CONFIG, 1
76	bgez	\dest, \nomt
77	 mfc0	\dest, CP0_CONFIG, 2
78	bgez	\dest, \nomt
79	 mfc0	\dest, CP0_CONFIG, 3
80	bgez	\dest, \nomt
81	 mfc0	\dest, CP0_CONFIG, 4
82	bgez	\dest, \nomt
83	 mfc0	\dest, CP0_CONFIG, 5
84	andi	\dest, \dest, MIPS_CONF5_VP
85	beqz	\dest, \nomt
86	 nop
87	.endm
88
89
90LEAF(mips_cps_core_boot)
91	/* Save  CCA and GCR base */
92	move   s0, a0
93	move   s1, a1
94
95	/* We don't know how to do coherence setup on earlier ISA */
96#if MIPS_ISA_REV > 0
97	/* Skip cache & coherence setup if we're already coherent */
98	lw	s7, GCR_CL_COHERENCE_OFS(s1)
99	bnez	s7, 1f
100	 nop
101
102	/* Initialize the L1 caches */
103	jal	mips_cps_cache_init
104	 nop
105
106	/* Enter the coherent domain */
107	li	t0, 0xff
108	sw	t0, GCR_CL_COHERENCE_OFS(s1)
109	ehb
110#endif /* MIPS_ISA_REV > 0 */
111
112	/* Set Kseg0 CCA to that in s0 */
1131:	mfc0	t0, CP0_CONFIG
114	ori	t0, 0x7
115	xori	t0, 0x7
116	or	t0, t0, s0
117	mtc0	t0, CP0_CONFIG
118	ehb
119
120	/* Jump to kseg0 */
121	PTR_LA	t0, 1f
122	jr	t0
123	 nop
124
125	/*
126	 * We're up, cached & coherent. Perform any EVA initialization necessary
127	 * before we access memory.
128	 */
1291:	eva_init
130
131	/* Retrieve boot configuration pointers */
132	jal	mips_cps_get_bootcfg
133	 nop
134
135	/* Skip core-level init if we started up coherent */
136	bnez	s7, 1f
137	 nop
138
139	/* Perform any further required core-level initialisation */
140	jal	mips_cps_core_init
141	 nop
142
143	/*
144	 * Boot any other VPEs within this core that should be online, and
145	 * deactivate this VPE if it should be offline.
146	 */
147	move	a1, t9
148	jal	mips_cps_boot_vpes
149	 move	a0, v0
150
151	/* Off we go! */
1521:	PTR_L	t1, VPEBOOTCFG_PC(v1)
153	PTR_L	gp, VPEBOOTCFG_GP(v1)
154	PTR_L	sp, VPEBOOTCFG_SP(v1)
155	jr	t1
156	 nop
157	END(mips_cps_core_boot)
158
159	__INIT
160LEAF(excep_tlbfill)
161	DUMP_EXCEP("TLB Fill")
162	b	.
163	 nop
164	END(excep_tlbfill)
165
166LEAF(excep_xtlbfill)
167	DUMP_EXCEP("XTLB Fill")
168	b	.
169	 nop
170	END(excep_xtlbfill)
171
172LEAF(excep_cache)
173	DUMP_EXCEP("Cache")
174	b	.
175	 nop
176	END(excep_cache)
177
178LEAF(excep_genex)
179	DUMP_EXCEP("General")
180	b	.
181	 nop
182	END(excep_genex)
183
184LEAF(excep_intex)
185	DUMP_EXCEP("Interrupt")
186	b	.
187	 nop
188	END(excep_intex)
189
190LEAF(excep_ejtag)
191	PTR_LA	k0, ejtag_debug_handler
192	jr	k0
193	 nop
194	END(excep_ejtag)
195	__FINIT
196
197LEAF(mips_cps_core_init)
198#ifdef CONFIG_MIPS_MT_SMP
199	/* Check that the core implements the MT ASE */
200	has_mt	t0, 3f
201
202	.set	push
203	.set	MIPS_ISA_LEVEL_RAW
204	.set	mt
205
206	/* Only allow 1 TC per VPE to execute... */
207	dmt
208
209	/* ...and for the moment only 1 VPE */
210	dvpe
211	PTR_LA	t1, 1f
212	jr.hb	t1
213	 nop
214
215	/* Enter VPE configuration state */
2161:	mfc0	t0, CP0_MVPCONTROL
217	ori	t0, t0, MVPCONTROL_VPC
218	mtc0	t0, CP0_MVPCONTROL
219
220	/* Retrieve the number of VPEs within the core */
221	mfc0	t0, CP0_MVPCONF0
222	srl	t0, t0, MVPCONF0_PVPE_SHIFT
223	andi	t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT)
224	addiu	ta3, t0, 1
225
226	/* If there's only 1, we're done */
227	beqz	t0, 2f
228	 nop
229
230	/* Loop through each VPE within this core */
231	li	ta1, 1
232
2331:	/* Operate on the appropriate TC */
234	mtc0	ta1, CP0_VPECONTROL
235	ehb
236
237	/* Bind TC to VPE (1:1 TC:VPE mapping) */
238	mttc0	ta1, CP0_TCBIND
239
240	/* Set exclusive TC, non-active, master */
241	li	t0, VPECONF0_MVP
242	sll	t1, ta1, VPECONF0_XTC_SHIFT
243	or	t0, t0, t1
244	mttc0	t0, CP0_VPECONF0
245
246	/* Set TC non-active, non-allocatable */
247	mttc0	zero, CP0_TCSTATUS
248
249	/* Set TC halted */
250	li	t0, TCHALT_H
251	mttc0	t0, CP0_TCHALT
252
253	/* Next VPE */
254	addiu	ta1, ta1, 1
255	slt	t0, ta1, ta3
256	bnez	t0, 1b
257	 nop
258
259	/* Leave VPE configuration state */
2602:	mfc0	t0, CP0_MVPCONTROL
261	xori	t0, t0, MVPCONTROL_VPC
262	mtc0	t0, CP0_MVPCONTROL
263
2643:	.set	pop
265#endif
266	jr	ra
267	 nop
268	END(mips_cps_core_init)
269
270/**
271 * mips_cps_get_bootcfg() - retrieve boot configuration pointers
272 *
273 * Returns: pointer to struct core_boot_config in v0, pointer to
274 *          struct vpe_boot_config in v1, VPE ID in t9
275 */
276LEAF(mips_cps_get_bootcfg)
277	/* Calculate a pointer to this cores struct core_boot_config */
278	PTR_LA	v0, mips_cps_cluster_bootcfg
279	PTR_L	v0, 0(v0)
280	lw	t0, GCR_CL_ID_OFS(s1)
281#ifdef CONFIG_CPU_MIPSR6
282	ext	t1, t0, CM3_GCR_Cx_ID_CLUSTER_SHF, 8
283	li	t2, CLUSTERBOOTCFG_SIZE
284	mul	t1, t1, t2
285	PTR_ADDU \
286		v0, v0, t1
287#endif
288	PTR_L	v0, CLUSTERBOOTCFG_CORECONFIG(v0)
289	andi	t0, t0, CM3_GCR_Cx_ID_CORENUM_MSK
290	li	t1, COREBOOTCFG_SIZE
291	mul	t0, t0, t1
292	PTR_ADDU v0, v0, t0
293
294	/* Calculate this VPEs ID. If the core doesn't support MT use 0 */
295	li	t9, 0
296#if defined(CONFIG_CPU_MIPSR6)
297	has_vp	ta2, 1f
298
299	/*
300	 * Assume non-contiguous numbering. Perhaps some day we'll need
301	 * to handle contiguous VP numbering, but no such systems yet
302	 * exist.
303	 */
304	mfc0	t9, CP0_GLOBALNUMBER
305	andi	t9, t9, MIPS_GLOBALNUMBER_VP
306#elif defined(CONFIG_MIPS_MT_SMP)
307	has_mt	ta2, 1f
308
309	/* Find the number of VPEs present in the core */
310	mfc0	t1, CP0_MVPCONF0
311	srl	t1, t1, MVPCONF0_PVPE_SHIFT
312	andi	t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT
313	addiu	t1, t1, 1
314
315	/* Calculate a mask for the VPE ID from EBase.CPUNum */
316	clz	t1, t1
317	li	t2, 31
318	subu	t1, t2, t1
319	li	t2, 1
320	sll	t1, t2, t1
321	addiu	t1, t1, -1
322
323	/* Retrieve the VPE ID from EBase.CPUNum */
324	mfc0	t9, $15, 1
325	and	t9, t9, t1
326#endif
327
3281:	/* Calculate a pointer to this VPEs struct vpe_boot_config */
329	li	t1, VPEBOOTCFG_SIZE
330	mul	v1, t9, t1
331	PTR_L	ta3, COREBOOTCFG_VPECONFIG(v0)
332	PTR_ADDU v1, v1, ta3
333
334	jr	ra
335	 nop
336	END(mips_cps_get_bootcfg)
337
338LEAF(mips_cps_boot_vpes)
339	lw	ta2, COREBOOTCFG_VPEMASK(a0)
340	PTR_L	ta3, COREBOOTCFG_VPECONFIG(a0)
341
342#if defined(CONFIG_CPU_MIPSR6)
343
344	has_vp	t0, 5f
345
346	/* Find base address of CPC */
347	PTR_LA	t1, mips_gcr_base
348	PTR_L	t1, 0(t1)
349	PTR_L	t1, GCR_CPC_BASE_OFS(t1)
350	PTR_LI	t2, ~0x7fff
351	and	t1, t1, t2
352	PTR_LI	t2, UNCAC_BASE
353	PTR_ADD	t1, t1, t2
354
355	/* Start any other VPs that ought to be running */
356	PTR_S	ta2, CPC_CL_VC_RUN_OFS(t1)
357
358	/* Ensure this VP stops running if it shouldn't be */
359	not	ta2
360	PTR_S	ta2, CPC_CL_VC_STOP_OFS(t1)
361	ehb
362
363#elif defined(CONFIG_MIPS_MT)
364
365	/* If the core doesn't support MT then return */
366	has_mt	t0, 5f
367
368	/* Enter VPE configuration state */
369	.set	push
370	.set	MIPS_ISA_LEVEL_RAW
371	.set	mt
372	dvpe
373	.set	pop
374
375	PTR_LA	t1, 1f
376	jr.hb	t1
377	 nop
3781:	mfc0	t1, CP0_MVPCONTROL
379	ori	t1, t1, MVPCONTROL_VPC
380	mtc0	t1, CP0_MVPCONTROL
381	ehb
382
383	/* Loop through each VPE */
384	move	t8, ta2
385	li	ta1, 0
386
387	/* Check whether the VPE should be running. If not, skip it */
3881:	andi	t0, ta2, 1
389	beqz	t0, 2f
390	 nop
391
392	/* Operate on the appropriate TC */
393	mfc0	t0, CP0_VPECONTROL
394	ori	t0, t0, VPECONTROL_TARGTC
395	xori	t0, t0, VPECONTROL_TARGTC
396	or	t0, t0, ta1
397	mtc0	t0, CP0_VPECONTROL
398	ehb
399
400	.set	push
401	.set	MIPS_ISA_LEVEL_RAW
402	.set	mt
403
404	/* Skip the VPE if its TC is not halted */
405	mftc0	t0, CP0_TCHALT
406	beqz	t0, 2f
407	 nop
408
409	/* Calculate a pointer to the VPEs struct vpe_boot_config */
410	li	t0, VPEBOOTCFG_SIZE
411	mul	t0, t0, ta1
412	PTR_ADDU t0, t0, ta3
413
414	/* Set the TC restart PC */
415	lw	t1, VPEBOOTCFG_PC(t0)
416	mttc0	t1, CP0_TCRESTART
417
418	/* Set the TC stack pointer */
419	lw	t1, VPEBOOTCFG_SP(t0)
420	mttgpr	t1, sp
421
422	/* Set the TC global pointer */
423	lw	t1, VPEBOOTCFG_GP(t0)
424	mttgpr	t1, gp
425
426	/* Copy config from this VPE */
427	mfc0	t0, CP0_CONFIG
428	mttc0	t0, CP0_CONFIG
429
430	/*
431	 * Copy the EVA config from this VPE if the CPU supports it.
432	 * CONFIG3 must exist to be running MT startup - just read it.
433	 */
434	mfc0	t0, CP0_CONFIG, 3
435	and	t0, t0, MIPS_CONF3_SC
436	beqz	t0, 3f
437	 nop
438	mfc0    t0, CP0_SEGCTL0
439	mttc0	t0, CP0_SEGCTL0
440	mfc0    t0, CP0_SEGCTL1
441	mttc0	t0, CP0_SEGCTL1
442	mfc0    t0, CP0_SEGCTL2
443	mttc0	t0, CP0_SEGCTL2
4443:
445	/* Ensure no software interrupts are pending */
446	mttc0	zero, CP0_CAUSE
447	mttc0	zero, CP0_STATUS
448
449	/* Set TC active, not interrupt exempt */
450	mftc0	t0, CP0_TCSTATUS
451	li	t1, ~TCSTATUS_IXMT
452	and	t0, t0, t1
453	ori	t0, t0, TCSTATUS_A
454	mttc0	t0, CP0_TCSTATUS
455
456	/* Clear the TC halt bit */
457	mttc0	zero, CP0_TCHALT
458
459	/* Set VPE active */
460	mftc0	t0, CP0_VPECONF0
461	ori	t0, t0, VPECONF0_VPA
462	mttc0	t0, CP0_VPECONF0
463
464	/* Next VPE */
4652:	srl	ta2, ta2, 1
466	addiu	ta1, ta1, 1
467	bnez	ta2, 1b
468	 nop
469
470	/* Leave VPE configuration state */
471	mfc0	t1, CP0_MVPCONTROL
472	xori	t1, t1, MVPCONTROL_VPC
473	mtc0	t1, CP0_MVPCONTROL
474	ehb
475	evpe
476
477	.set	pop
478
479	/* Check whether this VPE is meant to be running */
480	li	t0, 1
481	sll	t0, t0, a1
482	and	t0, t0, t8
483	bnez	t0, 2f
484	 nop
485
486	/* This VPE should be offline, halt the TC */
487	li	t0, TCHALT_H
488	mtc0	t0, CP0_TCHALT
489	PTR_LA	t0, 1f
4901:	jr.hb	t0
491	 nop
492
4932:
494
495#endif /* CONFIG_MIPS_MT_SMP */
496
497	/* Return */
4985:	jr	ra
499	 nop
500	END(mips_cps_boot_vpes)
501
502#if MIPS_ISA_REV > 0
503LEAF(mips_cps_cache_init)
504	/*
505	 * Clear the bits used to index the caches. Note that the architecture
506	 * dictates that writing to any of TagLo or TagHi selects 0 or 2 should
507	 * be valid for all MIPS32 CPUs, even those for which said writes are
508	 * unnecessary.
509	 */
510	mtc0	zero, CP0_TAGLO, 0
511	mtc0	zero, CP0_TAGHI, 0
512	mtc0	zero, CP0_TAGLO, 2
513	mtc0	zero, CP0_TAGHI, 2
514	ehb
515
516	/* Primary cache configuration is indicated by Config1 */
517	mfc0	v0, CP0_CONFIG, 1
518
519	/* Detect I-cache line size */
520	_EXT	t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ
521	beqz	t0, icache_done
522	 li	t1, 2
523	sllv	t0, t1, t0
524
525	/* Detect I-cache size */
526	_EXT	t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ
527	xori	t2, t1, 0x7
528	beqz	t2, 1f
529	 li	t3, 32
530	addiu	t1, t1, 1
531	sllv	t1, t3, t1
5321:	/* At this point t1 == I-cache sets per way */
533	_EXT	t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ
534	addiu	t2, t2, 1
535	mul	t1, t1, t0
536	mul	t1, t1, t2
537
538	li	a0, CKSEG0
539	PTR_ADD	a1, a0, t1
5401:	cache	Index_Store_Tag_I, 0(a0)
541	PTR_ADD	a0, a0, t0
542	bne	a0, a1, 1b
543	 nop
544icache_done:
545
546	/* Detect D-cache line size */
547	_EXT	t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ
548	beqz	t0, dcache_done
549	 li	t1, 2
550	sllv	t0, t1, t0
551
552	/* Detect D-cache size */
553	_EXT	t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ
554	xori	t2, t1, 0x7
555	beqz	t2, 1f
556	 li	t3, 32
557	addiu	t1, t1, 1
558	sllv	t1, t3, t1
5591:	/* At this point t1 == D-cache sets per way */
560	_EXT	t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ
561	addiu	t2, t2, 1
562	mul	t1, t1, t0
563	mul	t1, t1, t2
564
565	li	a0, CKSEG0
566	PTR_ADDU a1, a0, t1
567	PTR_SUBU a1, a1, t0
5681:	cache	Index_Store_Tag_D, 0(a0)
569	bne	a0, a1, 1b
570	 PTR_ADD a0, a0, t0
571dcache_done:
572
573	jr	ra
574	 nop
575	END(mips_cps_cache_init)
576#endif /* MIPS_ISA_REV > 0 */
577
578#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM)
579
580	/* Calculate a pointer to this CPUs struct mips_static_suspend_state */
581	.macro	psstate	dest
582	.set	push
583	.set	noat
584	lw	$1, TI_CPU(gp)
585	sll	$1, $1, LONGLOG
586	PTR_LA	\dest, __per_cpu_offset
587	PTR_ADDU $1, $1, \dest
588	lw	$1, 0($1)
589	PTR_LA	\dest, cps_cpu_state
590	PTR_ADDU \dest, \dest, $1
591	.set	pop
592	.endm
593
594LEAF(mips_cps_pm_save)
595	/* Save CPU state */
596	SUSPEND_SAVE_REGS
597	psstate	t1
598	SUSPEND_SAVE_STATIC
599	jr	v0
600	 nop
601	END(mips_cps_pm_save)
602
603LEAF(mips_cps_pm_restore)
604	/* Restore CPU state */
605	psstate	t1
606	RESUME_RESTORE_STATIC
607	RESUME_RESTORE_REGS_RETURN
608	END(mips_cps_pm_restore)
609
610#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */
611