43 		"1: prefetch (%0)\n"		/* This set is 28 bytes */  in _mmx_memcpy()
44 		"   prefetch 64(%0)\n"  in _mmx_memcpy()
45 		"   prefetch 128(%0)\n"  in _mmx_memcpy()
46 		"   prefetch 192(%0)\n"  in _mmx_memcpy()
47 		"   prefetch 256(%0)\n"  in _mmx_memcpy()
48 		"2:  \n"  in _mmx_memcpy()
49 		".section .fixup, \"ax\"\n"  in _mmx_memcpy()
50 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in _mmx_memcpy()
51 		"   jmp 2b\n"  in _mmx_memcpy()
52 		".previous\n"  in _mmx_memcpy()
58 		"1:  prefetch 320(%0)\n"  in _mmx_memcpy()
59 		"2:  movq (%0), %%mm0\n"  in _mmx_memcpy()
60 		"  movq 8(%0), %%mm1\n"  in _mmx_memcpy()
61 		"  movq 16(%0), %%mm2\n"  in _mmx_memcpy()
62 		"  movq 24(%0), %%mm3\n"  in _mmx_memcpy()
63 		"  movq %%mm0, (%1)\n"  in _mmx_memcpy()
64 		"  movq %%mm1, 8(%1)\n"  in _mmx_memcpy()
65 		"  movq %%mm2, 16(%1)\n"  in _mmx_memcpy()
66 		"  movq %%mm3, 24(%1)\n"  in _mmx_memcpy()
67 		"  movq 32(%0), %%mm0\n"  in _mmx_memcpy()
68 		"  movq 40(%0), %%mm1\n"  in _mmx_memcpy()
69 		"  movq 48(%0), %%mm2\n"  in _mmx_memcpy()
70 		"  movq 56(%0), %%mm3\n"  in _mmx_memcpy()
71 		"  movq %%mm0, 32(%1)\n"  in _mmx_memcpy()
72 		"  movq %%mm1, 40(%1)\n"  in _mmx_memcpy()
73 		"  movq %%mm2, 48(%1)\n"  in _mmx_memcpy()
74 		"  movq %%mm3, 56(%1)\n"  in _mmx_memcpy()
75 		".section .fixup, \"ax\"\n"  in _mmx_memcpy()
76 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in _mmx_memcpy()
77 		"   jmp 2b\n"  in _mmx_memcpy()
78 		".previous\n"  in _mmx_memcpy()
88 		"  movq (%0), %%mm0\n"  in _mmx_memcpy()
89 		"  movq 8(%0), %%mm1\n"  in _mmx_memcpy()
90 		"  movq 16(%0), %%mm2\n"  in _mmx_memcpy()
91 		"  movq 24(%0), %%mm3\n"  in _mmx_memcpy()
92 		"  movq %%mm0, (%1)\n"  in _mmx_memcpy()
93 		"  movq %%mm1, 8(%1)\n"  in _mmx_memcpy()
94 		"  movq %%mm2, 16(%1)\n"  in _mmx_memcpy()
95 		"  movq %%mm3, 24(%1)\n"  in _mmx_memcpy()
96 		"  movq 32(%0), %%mm0\n"  in _mmx_memcpy()
97 		"  movq 40(%0), %%mm1\n"  in _mmx_memcpy()
98 		"  movq 48(%0), %%mm2\n"  in _mmx_memcpy()
99 		"  movq 56(%0), %%mm3\n"  in _mmx_memcpy()
100 		"  movq %%mm0, 32(%1)\n"  in _mmx_memcpy()
101 		"  movq %%mm1, 40(%1)\n"  in _mmx_memcpy()
102 		"  movq %%mm2, 48(%1)\n"  in _mmx_memcpy()
103 		"  movq %%mm3, 56(%1)\n"  in _mmx_memcpy()
133 		"  pxor %%mm0, %%mm0\n" : :  in fast_clear_page()
138 		"  movntq %%mm0, (%0)\n"  in fast_clear_page()
139 		"  movntq %%mm0, 8(%0)\n"  in fast_clear_page()
140 		"  movntq %%mm0, 16(%0)\n"  in fast_clear_page()
141 		"  movntq %%mm0, 24(%0)\n"  in fast_clear_page()
142 		"  movntq %%mm0, 32(%0)\n"  in fast_clear_page()
143 		"  movntq %%mm0, 40(%0)\n"  in fast_clear_page()
144 		"  movntq %%mm0, 48(%0)\n"  in fast_clear_page()
145 		"  movntq %%mm0, 56(%0)\n"  in fast_clear_page()
154 	__asm__ __volatile__("sfence\n"::);  in fast_clear_page()
170 		"1: prefetch (%0)\n"  in fast_copy_page()
171 		"   prefetch 64(%0)\n"  in fast_copy_page()
172 		"   prefetch 128(%0)\n"  in fast_copy_page()
173 		"   prefetch 192(%0)\n"  in fast_copy_page()
174 		"   prefetch 256(%0)\n"  in fast_copy_page()
175 		"2:  \n"  in fast_copy_page()
176 		".section .fixup, \"ax\"\n"  in fast_copy_page()
177 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in fast_copy_page()
178 		"   jmp 2b\n"  in fast_copy_page()
179 		".previous\n"  in fast_copy_page()
184 		"1: prefetch 320(%0)\n"  in fast_copy_page()
185 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
186 		"   movntq %%mm0, (%1)\n"  in fast_copy_page()
187 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
188 		"   movntq %%mm1, 8(%1)\n"  in fast_copy_page()
189 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
190 		"   movntq %%mm2, 16(%1)\n"  in fast_copy_page()
191 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
192 		"   movntq %%mm3, 24(%1)\n"  in fast_copy_page()
193 		"   movq 32(%0), %%mm4\n"  in fast_copy_page()
194 		"   movntq %%mm4, 32(%1)\n"  in fast_copy_page()
195 		"   movq 40(%0), %%mm5\n"  in fast_copy_page()
196 		"   movntq %%mm5, 40(%1)\n"  in fast_copy_page()
197 		"   movq 48(%0), %%mm6\n"  in fast_copy_page()
198 		"   movntq %%mm6, 48(%1)\n"  in fast_copy_page()
199 		"   movq 56(%0), %%mm7\n"  in fast_copy_page()
200 		"   movntq %%mm7, 56(%1)\n"  in fast_copy_page()
201 		".section .fixup, \"ax\"\n"  in fast_copy_page()
202 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in fast_copy_page()
203 		"   jmp 2b\n"  in fast_copy_page()
204 		".previous\n"  in fast_copy_page()
213 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
214 		"   movntq %%mm0, (%1)\n"  in fast_copy_page()
215 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
216 		"   movntq %%mm1, 8(%1)\n"  in fast_copy_page()
217 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
218 		"   movntq %%mm2, 16(%1)\n"  in fast_copy_page()
219 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
220 		"   movntq %%mm3, 24(%1)\n"  in fast_copy_page()
221 		"   movq 32(%0), %%mm4\n"  in fast_copy_page()
222 		"   movntq %%mm4, 32(%1)\n"  in fast_copy_page()
223 		"   movq 40(%0), %%mm5\n"  in fast_copy_page()
224 		"   movntq %%mm5, 40(%1)\n"  in fast_copy_page()
225 		"   movq 48(%0), %%mm6\n"  in fast_copy_page()
226 		"   movntq %%mm6, 48(%1)\n"  in fast_copy_page()
227 		"   movq 56(%0), %%mm7\n"  in fast_copy_page()
228 		"   movntq %%mm7, 56(%1)\n"  in fast_copy_page()
237 	__asm__ __volatile__("sfence \n"::);  in fast_copy_page()
253 		"  pxor %%mm0, %%mm0\n" : :  in fast_clear_page()
258 		"  movq %%mm0, (%0)\n"  in fast_clear_page()
259 		"  movq %%mm0, 8(%0)\n"  in fast_clear_page()
260 		"  movq %%mm0, 16(%0)\n"  in fast_clear_page()
261 		"  movq %%mm0, 24(%0)\n"  in fast_clear_page()
262 		"  movq %%mm0, 32(%0)\n"  in fast_clear_page()
263 		"  movq %%mm0, 40(%0)\n"  in fast_clear_page()
264 		"  movq %%mm0, 48(%0)\n"  in fast_clear_page()
265 		"  movq %%mm0, 56(%0)\n"  in fast_clear_page()
266 		"  movq %%mm0, 64(%0)\n"  in fast_clear_page()
267 		"  movq %%mm0, 72(%0)\n"  in fast_clear_page()
268 		"  movq %%mm0, 80(%0)\n"  in fast_clear_page()
269 		"  movq %%mm0, 88(%0)\n"  in fast_clear_page()
270 		"  movq %%mm0, 96(%0)\n"  in fast_clear_page()
271 		"  movq %%mm0, 104(%0)\n"  in fast_clear_page()
272 		"  movq %%mm0, 112(%0)\n"  in fast_clear_page()
273 		"  movq %%mm0, 120(%0)\n"  in fast_clear_page()
288 		"1: prefetch (%0)\n"  in fast_copy_page()
289 		"   prefetch 64(%0)\n"  in fast_copy_page()
290 		"   prefetch 128(%0)\n"  in fast_copy_page()
291 		"   prefetch 192(%0)\n"  in fast_copy_page()
292 		"   prefetch 256(%0)\n"  in fast_copy_page()
293 		"2:  \n"  in fast_copy_page()
294 		".section .fixup, \"ax\"\n"  in fast_copy_page()
295 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in fast_copy_page()
296 		"   jmp 2b\n"  in fast_copy_page()
297 		".previous\n"  in fast_copy_page()
302 		"1: prefetch 320(%0)\n"  in fast_copy_page()
303 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
304 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
305 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
306 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
307 		"   movq %%mm0, (%1)\n"  in fast_copy_page()
308 		"   movq %%mm1, 8(%1)\n"  in fast_copy_page()
309 		"   movq %%mm2, 16(%1)\n"  in fast_copy_page()
310 		"   movq %%mm3, 24(%1)\n"  in fast_copy_page()
311 		"   movq 32(%0), %%mm0\n"  in fast_copy_page()
312 		"   movq 40(%0), %%mm1\n"  in fast_copy_page()
313 		"   movq 48(%0), %%mm2\n"  in fast_copy_page()
314 		"   movq 56(%0), %%mm3\n"  in fast_copy_page()
315 		"   movq %%mm0, 32(%1)\n"  in fast_copy_page()
316 		"   movq %%mm1, 40(%1)\n"  in fast_copy_page()
317 		"   movq %%mm2, 48(%1)\n"  in fast_copy_page()
318 		"   movq %%mm3, 56(%1)\n"  in fast_copy_page()
319 		".section .fixup, \"ax\"\n"  in fast_copy_page()
320 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in fast_copy_page()
321 		"   jmp 2b\n"  in fast_copy_page()
322 		".previous\n"  in fast_copy_page()
342 		"cld\n\t"  in slow_zero_page()
364 		"cld\n\t"  in slow_copy_page()