1/*
2 *  linux/arch/arm/lib/uaccess.S
3 *
4 *  Copyright (C) 1995, 1996,1997,1998 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 *  Routines to block copy data to/from user memory
11 *   These are highly optimised both for the 4k page size
12 *   and for various alignments.
13 */
14#include <linux/linkage.h>
15#include <asm/assembler.h>
16#include <asm/errno.h>
17#include <asm/domain.h>
18
19		.text
20
21#define PAGE_SHIFT 12
22
23/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
24 * Purpose  : copy a block to user memory from kernel memory
25 * Params   : to   - user memory
26 *          : from - kernel memory
27 *          : n    - number of bytes to copy
28 * Returns  : Number of bytes NOT copied.
29 */
30
31.Lc2u_dest_not_aligned:
32		rsb	ip, ip, #4
33		cmp	ip, #2
34		ldrb	r3, [r1], #1
35USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
36		ldrgeb	r3, [r1], #1
37USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
38		ldrgtb	r3, [r1], #1
39USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
40		sub	r2, r2, ip
41		b	.Lc2u_dest_aligned
42
43ENTRY(__copy_to_user)
44		stmfd	sp!, {r2, r4 - r7, lr}
45		cmp	r2, #4
46		blt	.Lc2u_not_enough
47		ands	ip, r0, #3
48		bne	.Lc2u_dest_not_aligned
49.Lc2u_dest_aligned:
50
51		ands	ip, r1, #3
52		bne	.Lc2u_src_not_aligned
53/*
54 * Seeing as there has to be at least 8 bytes to copy, we can
55 * copy one word, and force a user-mode page fault...
56 */
57
58.Lc2u_0fupi:	subs	r2, r2, #4
59		addmi	ip, r2, #4
60		bmi	.Lc2u_0nowords
61		ldr	r3, [r1], #4
62USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
63		mov	ip, r0, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
64		rsb	ip, ip, #0
65		movs	ip, ip, lsr #32 - PAGE_SHIFT
66		beq	.Lc2u_0fupi
67/*
68 * ip = max no. of bytes to copy before needing another "strt" insn
69 */
70		cmp	r2, ip
71		movlt	ip, r2
72		sub	r2, r2, ip
73		subs	ip, ip, #32
74		blt	.Lc2u_0rem8lp
75
76.Lc2u_0cpy8lp:	ldmia	r1!, {r3 - r6}
77		stmia	r0!, {r3 - r6}			@ Shouldnt fault
78		ldmia	r1!, {r3 - r6}
79		subs	ip, ip, #32
80		stmia	r0!, {r3 - r6}			@ Shouldnt fault
81		bpl	.Lc2u_0cpy8lp
82
83.Lc2u_0rem8lp:	cmn	ip, #16
84		ldmgeia	r1!, {r3 - r6}
85		stmgeia	r0!, {r3 - r6}			@ Shouldnt fault
86		tst	ip, #8
87		ldmneia	r1!, {r3 - r4}
88		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
89		tst	ip, #4
90		ldrne	r3, [r1], #4
91	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
92		ands	ip, ip, #3
93		beq	.Lc2u_0fupi
94.Lc2u_0nowords:	teq	ip, #0
95		beq	.Lc2u_finished
96.Lc2u_nowords:	cmp	ip, #2
97		ldrb	r3, [r1], #1
98USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
99		ldrgeb	r3, [r1], #1
100USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
101		ldrgtb	r3, [r1], #1
102USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
103		b	.Lc2u_finished
104
105.Lc2u_not_enough:
106		movs	ip, r2
107		bne	.Lc2u_nowords
108.Lc2u_finished:	mov	r0, #0
109		ldmfd	sp!, {r2, r4 - r7, pc}
110
111.Lc2u_src_not_aligned:
112		bic	r1, r1, #3
113		ldr	r7, [r1], #4
114		cmp	ip, #2
115		bgt	.Lc2u_3fupi
116		beq	.Lc2u_2fupi
117.Lc2u_1fupi:	subs	r2, r2, #4
118		addmi	ip, r2, #4
119		bmi	.Lc2u_1nowords
120		mov	r3, r7, pull #8
121		ldr	r7, [r1], #4
122		orr	r3, r3, r7, push #24
123USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
124		mov	ip, r0, lsl #32 - PAGE_SHIFT
125		rsb	ip, ip, #0
126		movs	ip, ip, lsr #32 - PAGE_SHIFT
127		beq	.Lc2u_1fupi
128		cmp	r2, ip
129		movlt	ip, r2
130		sub	r2, r2, ip
131		subs	ip, ip, #16
132		blt	.Lc2u_1rem8lp
133
134.Lc2u_1cpy8lp:	mov	r3, r7, pull #8
135		ldmia	r1!, {r4 - r7}
136		subs	ip, ip, #16
137		orr	r3, r3, r4, push #24
138		mov	r4, r4, pull #8
139		orr	r4, r4, r5, push #24
140		mov	r5, r5, pull #8
141		orr	r5, r5, r6, push #24
142		mov	r6, r6, pull #8
143		orr	r6, r6, r7, push #24
144		stmia	r0!, {r3 - r6}			@ Shouldnt fault
145		bpl	.Lc2u_1cpy8lp
146
147.Lc2u_1rem8lp:	tst	ip, #8
148		movne	r3, r7, pull #8
149		ldmneia	r1!, {r4, r7}
150		orrne	r3, r3, r4, push #24
151		movne	r4, r4, pull #8
152		orrne	r4, r4, r7, push #24
153		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
154		tst	ip, #4
155		movne	r3, r7, pull #8
156		ldrne	r7, [r1], #4
157		orrne	r3, r3, r7, push #24
158	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
159		ands	ip, ip, #3
160		beq	.Lc2u_1fupi
161.Lc2u_1nowords:	mov	r3, r7, get_byte_1
162		teq	ip, #0
163		beq	.Lc2u_finished
164		cmp	ip, #2
165USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
166		movge	r3, r7, get_byte_2
167USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
168		movgt	r3, r7, get_byte_3
169USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
170		b	.Lc2u_finished
171
172.Lc2u_2fupi:	subs	r2, r2, #4
173		addmi	ip, r2, #4
174		bmi	.Lc2u_2nowords
175		mov	r3, r7, pull #16
176		ldr	r7, [r1], #4
177		orr	r3, r3, r7, push #16
178USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
179		mov	ip, r0, lsl #32 - PAGE_SHIFT
180		rsb	ip, ip, #0
181		movs	ip, ip, lsr #32 - PAGE_SHIFT
182		beq	.Lc2u_2fupi
183		cmp	r2, ip
184		movlt	ip, r2
185		sub	r2, r2, ip
186		subs	ip, ip, #16
187		blt	.Lc2u_2rem8lp
188
189.Lc2u_2cpy8lp:	mov	r3, r7, pull #16
190		ldmia	r1!, {r4 - r7}
191		subs	ip, ip, #16
192		orr	r3, r3, r4, push #16
193		mov	r4, r4, pull #16
194		orr	r4, r4, r5, push #16
195		mov	r5, r5, pull #16
196		orr	r5, r5, r6, push #16
197		mov	r6, r6, pull #16
198		orr	r6, r6, r7, push #16
199		stmia	r0!, {r3 - r6}			@ Shouldnt fault
200		bpl	.Lc2u_2cpy8lp
201
202.Lc2u_2rem8lp:	tst	ip, #8
203		movne	r3, r7, pull #16
204		ldmneia	r1!, {r4, r7}
205		orrne	r3, r3, r4, push #16
206		movne	r4, r4, pull #16
207		orrne	r4, r4, r7, push #16
208		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
209		tst	ip, #4
210		movne	r3, r7, pull #16
211		ldrne	r7, [r1], #4
212		orrne	r3, r3, r7, push #16
213	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
214		ands	ip, ip, #3
215		beq	.Lc2u_2fupi
216.Lc2u_2nowords:	mov	r3, r7, get_byte_2
217		teq	ip, #0
218		beq	.Lc2u_finished
219		cmp	ip, #2
220USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
221		movge	r3, r7, get_byte_3
222USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
223		ldrgtb	r3, [r1], #0
224USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
225		b	.Lc2u_finished
226
227.Lc2u_3fupi:	subs	r2, r2, #4
228		addmi	ip, r2, #4
229		bmi	.Lc2u_3nowords
230		mov	r3, r7, pull #24
231		ldr	r7, [r1], #4
232		orr	r3, r3, r7, push #8
233USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
234		mov	ip, r0, lsl #32 - PAGE_SHIFT
235		rsb	ip, ip, #0
236		movs	ip, ip, lsr #32 - PAGE_SHIFT
237		beq	.Lc2u_3fupi
238		cmp	r2, ip
239		movlt	ip, r2
240		sub	r2, r2, ip
241		subs	ip, ip, #16
242		blt	.Lc2u_3rem8lp
243
244.Lc2u_3cpy8lp:	mov	r3, r7, pull #24
245		ldmia	r1!, {r4 - r7}
246		subs	ip, ip, #16
247		orr	r3, r3, r4, push #8
248		mov	r4, r4, pull #24
249		orr	r4, r4, r5, push #8
250		mov	r5, r5, pull #24
251		orr	r5, r5, r6, push #8
252		mov	r6, r6, pull #24
253		orr	r6, r6, r7, push #8
254		stmia	r0!, {r3 - r6}			@ Shouldnt fault
255		bpl	.Lc2u_3cpy8lp
256
257.Lc2u_3rem8lp:	tst	ip, #8
258		movne	r3, r7, pull #24
259		ldmneia	r1!, {r4, r7}
260		orrne	r3, r3, r4, push #8
261		movne	r4, r4, pull #24
262		orrne	r4, r4, r7, push #8
263		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
264		tst	ip, #4
265		movne	r3, r7, pull #24
266		ldrne	r7, [r1], #4
267		orrne	r3, r3, r7, push #8
268	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
269		ands	ip, ip, #3
270		beq	.Lc2u_3fupi
271.Lc2u_3nowords:	mov	r3, r7, get_byte_3
272		teq	ip, #0
273		beq	.Lc2u_finished
274		cmp	ip, #2
275USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
276		ldrgeb	r3, [r1], #1
277USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
278		ldrgtb	r3, [r1], #0
279USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
280		b	.Lc2u_finished
281ENDPROC(__copy_to_user)
282
283		.pushsection .fixup,"ax"
284		.align	0
2859001:		ldmfd	sp!, {r0, r4 - r7, pc}
286		.popsection
287
288/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
289 * Purpose  : copy a block from user memory to kernel memory
290 * Params   : to   - kernel memory
291 *          : from - user memory
292 *          : n    - number of bytes to copy
293 * Returns  : Number of bytes NOT copied.
294 */
295.Lcfu_dest_not_aligned:
296		rsb	ip, ip, #4
297		cmp	ip, #2
298USER(	TUSER(	ldrb)	r3, [r1], #1)			@ May fault
299		strb	r3, [r0], #1
300USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
301		strgeb	r3, [r0], #1
302USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
303		strgtb	r3, [r0], #1
304		sub	r2, r2, ip
305		b	.Lcfu_dest_aligned
306
307ENTRY(__copy_from_user)
308		stmfd	sp!, {r0, r2, r4 - r7, lr}
309		cmp	r2, #4
310		blt	.Lcfu_not_enough
311		ands	ip, r0, #3
312		bne	.Lcfu_dest_not_aligned
313.Lcfu_dest_aligned:
314		ands	ip, r1, #3
315		bne	.Lcfu_src_not_aligned
316
317/*
318 * Seeing as there has to be at least 8 bytes to copy, we can
319 * copy one word, and force a user-mode page fault...
320 */
321
322.Lcfu_0fupi:	subs	r2, r2, #4
323		addmi	ip, r2, #4
324		bmi	.Lcfu_0nowords
325USER(	TUSER(	ldr)	r3, [r1], #4)
326		str	r3, [r0], #4
327		mov	ip, r1, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
328		rsb	ip, ip, #0
329		movs	ip, ip, lsr #32 - PAGE_SHIFT
330		beq	.Lcfu_0fupi
331/*
332 * ip = max no. of bytes to copy before needing another "strt" insn
333 */
334		cmp	r2, ip
335		movlt	ip, r2
336		sub	r2, r2, ip
337		subs	ip, ip, #32
338		blt	.Lcfu_0rem8lp
339
340.Lcfu_0cpy8lp:	ldmia	r1!, {r3 - r6}			@ Shouldnt fault
341		stmia	r0!, {r3 - r6}
342		ldmia	r1!, {r3 - r6}			@ Shouldnt fault
343		subs	ip, ip, #32
344		stmia	r0!, {r3 - r6}
345		bpl	.Lcfu_0cpy8lp
346
347.Lcfu_0rem8lp:	cmn	ip, #16
348		ldmgeia	r1!, {r3 - r6}			@ Shouldnt fault
349		stmgeia	r0!, {r3 - r6}
350		tst	ip, #8
351		ldmneia	r1!, {r3 - r4}			@ Shouldnt fault
352		stmneia	r0!, {r3 - r4}
353		tst	ip, #4
354	TUSER(	ldrne) r3, [r1], #4			@ Shouldnt fault
355		strne	r3, [r0], #4
356		ands	ip, ip, #3
357		beq	.Lcfu_0fupi
358.Lcfu_0nowords:	teq	ip, #0
359		beq	.Lcfu_finished
360.Lcfu_nowords:	cmp	ip, #2
361USER(	TUSER(	ldrb)	r3, [r1], #1)			@ May fault
362		strb	r3, [r0], #1
363USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
364		strgeb	r3, [r0], #1
365USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
366		strgtb	r3, [r0], #1
367		b	.Lcfu_finished
368
369.Lcfu_not_enough:
370		movs	ip, r2
371		bne	.Lcfu_nowords
372.Lcfu_finished:	mov	r0, #0
373		add	sp, sp, #8
374		ldmfd	sp!, {r4 - r7, pc}
375
376.Lcfu_src_not_aligned:
377		bic	r1, r1, #3
378USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
379		cmp	ip, #2
380		bgt	.Lcfu_3fupi
381		beq	.Lcfu_2fupi
382.Lcfu_1fupi:	subs	r2, r2, #4
383		addmi	ip, r2, #4
384		bmi	.Lcfu_1nowords
385		mov	r3, r7, pull #8
386USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
387		orr	r3, r3, r7, push #24
388		str	r3, [r0], #4
389		mov	ip, r1, lsl #32 - PAGE_SHIFT
390		rsb	ip, ip, #0
391		movs	ip, ip, lsr #32 - PAGE_SHIFT
392		beq	.Lcfu_1fupi
393		cmp	r2, ip
394		movlt	ip, r2
395		sub	r2, r2, ip
396		subs	ip, ip, #16
397		blt	.Lcfu_1rem8lp
398
399.Lcfu_1cpy8lp:	mov	r3, r7, pull #8
400		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
401		subs	ip, ip, #16
402		orr	r3, r3, r4, push #24
403		mov	r4, r4, pull #8
404		orr	r4, r4, r5, push #24
405		mov	r5, r5, pull #8
406		orr	r5, r5, r6, push #24
407		mov	r6, r6, pull #8
408		orr	r6, r6, r7, push #24
409		stmia	r0!, {r3 - r6}
410		bpl	.Lcfu_1cpy8lp
411
412.Lcfu_1rem8lp:	tst	ip, #8
413		movne	r3, r7, pull #8
414		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
415		orrne	r3, r3, r4, push #24
416		movne	r4, r4, pull #8
417		orrne	r4, r4, r7, push #24
418		stmneia	r0!, {r3 - r4}
419		tst	ip, #4
420		movne	r3, r7, pull #8
421USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
422		orrne	r3, r3, r7, push #24
423		strne	r3, [r0], #4
424		ands	ip, ip, #3
425		beq	.Lcfu_1fupi
426.Lcfu_1nowords:	mov	r3, r7, get_byte_1
427		teq	ip, #0
428		beq	.Lcfu_finished
429		cmp	ip, #2
430		strb	r3, [r0], #1
431		movge	r3, r7, get_byte_2
432		strgeb	r3, [r0], #1
433		movgt	r3, r7, get_byte_3
434		strgtb	r3, [r0], #1
435		b	.Lcfu_finished
436
437.Lcfu_2fupi:	subs	r2, r2, #4
438		addmi	ip, r2, #4
439		bmi	.Lcfu_2nowords
440		mov	r3, r7, pull #16
441USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
442		orr	r3, r3, r7, push #16
443		str	r3, [r0], #4
444		mov	ip, r1, lsl #32 - PAGE_SHIFT
445		rsb	ip, ip, #0
446		movs	ip, ip, lsr #32 - PAGE_SHIFT
447		beq	.Lcfu_2fupi
448		cmp	r2, ip
449		movlt	ip, r2
450		sub	r2, r2, ip
451		subs	ip, ip, #16
452		blt	.Lcfu_2rem8lp
453
454
455.Lcfu_2cpy8lp:	mov	r3, r7, pull #16
456		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
457		subs	ip, ip, #16
458		orr	r3, r3, r4, push #16
459		mov	r4, r4, pull #16
460		orr	r4, r4, r5, push #16
461		mov	r5, r5, pull #16
462		orr	r5, r5, r6, push #16
463		mov	r6, r6, pull #16
464		orr	r6, r6, r7, push #16
465		stmia	r0!, {r3 - r6}
466		bpl	.Lcfu_2cpy8lp
467
468.Lcfu_2rem8lp:	tst	ip, #8
469		movne	r3, r7, pull #16
470		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
471		orrne	r3, r3, r4, push #16
472		movne	r4, r4, pull #16
473		orrne	r4, r4, r7, push #16
474		stmneia	r0!, {r3 - r4}
475		tst	ip, #4
476		movne	r3, r7, pull #16
477USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
478		orrne	r3, r3, r7, push #16
479		strne	r3, [r0], #4
480		ands	ip, ip, #3
481		beq	.Lcfu_2fupi
482.Lcfu_2nowords:	mov	r3, r7, get_byte_2
483		teq	ip, #0
484		beq	.Lcfu_finished
485		cmp	ip, #2
486		strb	r3, [r0], #1
487		movge	r3, r7, get_byte_3
488		strgeb	r3, [r0], #1
489USER(	TUSER(	ldrgtb) r3, [r1], #0)			@ May fault
490		strgtb	r3, [r0], #1
491		b	.Lcfu_finished
492
493.Lcfu_3fupi:	subs	r2, r2, #4
494		addmi	ip, r2, #4
495		bmi	.Lcfu_3nowords
496		mov	r3, r7, pull #24
497USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
498		orr	r3, r3, r7, push #8
499		str	r3, [r0], #4
500		mov	ip, r1, lsl #32 - PAGE_SHIFT
501		rsb	ip, ip, #0
502		movs	ip, ip, lsr #32 - PAGE_SHIFT
503		beq	.Lcfu_3fupi
504		cmp	r2, ip
505		movlt	ip, r2
506		sub	r2, r2, ip
507		subs	ip, ip, #16
508		blt	.Lcfu_3rem8lp
509
510.Lcfu_3cpy8lp:	mov	r3, r7, pull #24
511		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
512		orr	r3, r3, r4, push #8
513		mov	r4, r4, pull #24
514		orr	r4, r4, r5, push #8
515		mov	r5, r5, pull #24
516		orr	r5, r5, r6, push #8
517		mov	r6, r6, pull #24
518		orr	r6, r6, r7, push #8
519		stmia	r0!, {r3 - r6}
520		subs	ip, ip, #16
521		bpl	.Lcfu_3cpy8lp
522
523.Lcfu_3rem8lp:	tst	ip, #8
524		movne	r3, r7, pull #24
525		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
526		orrne	r3, r3, r4, push #8
527		movne	r4, r4, pull #24
528		orrne	r4, r4, r7, push #8
529		stmneia	r0!, {r3 - r4}
530		tst	ip, #4
531		movne	r3, r7, pull #24
532USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
533		orrne	r3, r3, r7, push #8
534		strne	r3, [r0], #4
535		ands	ip, ip, #3
536		beq	.Lcfu_3fupi
537.Lcfu_3nowords:	mov	r3, r7, get_byte_3
538		teq	ip, #0
539		beq	.Lcfu_finished
540		cmp	ip, #2
541		strb	r3, [r0], #1
542USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
543		strgeb	r3, [r0], #1
544USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
545		strgtb	r3, [r0], #1
546		b	.Lcfu_finished
547ENDPROC(__copy_from_user)
548
549		.pushsection .fixup,"ax"
550		.align	0
551		/*
552		 * We took an exception.  r0 contains a pointer to
553		 * the byte not copied.
554		 */
5559001:		ldr	r2, [sp], #4			@ void *to
556		sub	r2, r0, r2			@ bytes copied
557		ldr	r1, [sp], #4			@ unsigned long count
558		subs	r4, r1, r2			@ bytes left to copy
559		movne	r1, r4
560		blne	__memzero
561		mov	r0, r4
562		ldmfd	sp!, {r4 - r7, pc}
563		.popsection
564
565