1/* 2 * ppc64 MMU hashtable management routines 3 * 4 * (c) Copyright IBM Corp. 2003, 2005 5 * 6 * Maintained by: Benjamin Herrenschmidt 7 * <benh@kernel.crashing.org> 8 * 9 * This file is covered by the GNU Public Licence v2 as 10 * described in the kernel's COPYING file. 11 */ 12 13#include <asm/reg.h> 14#include <asm/pgtable.h> 15#include <asm/mmu.h> 16#include <asm/page.h> 17#include <asm/types.h> 18#include <asm/ppc_asm.h> 19#include <asm/asm-offsets.h> 20#include <asm/cputable.h> 21 22 .text 23 24/* 25 * Stackframe: 26 * 27 * +-> Back chain (SP + 256) 28 * | General register save area (SP + 112) 29 * | Parameter save area (SP + 48) 30 * | TOC save area (SP + 40) 31 * | link editor doubleword (SP + 32) 32 * | compiler doubleword (SP + 24) 33 * | LR save area (SP + 16) 34 * | CR save area (SP + 8) 35 * SP ---> +-- Back chain (SP + 0) 36 */ 37#define STACKFRAMESIZE 256 38 39/* Save parameters offsets */ 40#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) 41 42/* Save non-volatile offsets */ 43#define STK_REG(i) (112 + ((i)-14)*8) 44 45 46#ifndef CONFIG_PPC_64K_PAGES 47 48/***************************************************************************** 49 * * 50 * 4K SW & 4K HW pages implementation * 51 * * 52 *****************************************************************************/ 53 54 55/* 56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 57 * pte_t *ptep, unsigned long trap, int local, int ssize) 58 * 59 * Adds a 4K page to the hash table in a segment of 4K pages only 60 */ 61 62_GLOBAL(__hash_page_4K) 63 mflr r0 64 std r0,16(r1) 65 stdu r1,-STACKFRAMESIZE(r1) 66 /* Save all params that we need after a function call */ 67 std r6,STK_PARM(r6)(r1) 68 std r8,STK_PARM(r8)(r1) 69 std r9,STK_PARM(r9)(r1) 70 71 /* Save non-volatile registers. 72 * r31 will hold "old PTE" 73 * r30 is "new PTE" 74 * r29 is "va" 75 * r28 is a hash value 76 * r27 is hashtab mask (maybe dynamic patched instead ?) 77 */ 78 std r27,STK_REG(r27)(r1) 79 std r28,STK_REG(r28)(r1) 80 std r29,STK_REG(r29)(r1) 81 std r30,STK_REG(r30)(r1) 82 std r31,STK_REG(r31)(r1) 83 84 /* Step 1: 85 * 86 * Check permissions, atomically mark the linux PTE busy 87 * and hashed. 88 */ 891: 90 ldarx r31,0,r6 91 /* Check access rights (access & ~(pte_val(*ptep))) */ 92 andc. r0,r4,r31 93 bne- htab_wrong_access 94 /* Check if PTE is busy */ 95 andi. r0,r31,_PAGE_BUSY 96 /* If so, just bail out and refault if needed. Someone else 97 * is changing this PTE anyway and might hash it. 98 */ 99 bne- htab_bail_ok 100 101 /* Prepare new PTE value (turn access RW into DIRTY, then 102 * add BUSY,HASHPTE and ACCESSED) 103 */ 104 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 105 or r30,r30,r31 106 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 107 /* Write the linux PTE atomically (setting busy) */ 108 stdcx. r30,0,r6 109 bne- 1b 110 isync 111 112 /* Step 2: 113 * 114 * Insert/Update the HPTE in the hash table. At this point, 115 * r4 (access) is re-useable, we use it for the new HPTE flags 116 */ 117 118BEGIN_FTR_SECTION 119 cmpdi r9,0 /* check segment size */ 120 bne 3f 121END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 122 /* Calc va and put it in r29 */ 123 rldicr r29,r5,28,63-28 124 rldicl r3,r3,0,36 125 or r29,r3,r29 126 127 /* Calculate hash value for primary slot and store it in r28 */ 128 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 129 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 130 xor r28,r5,r0 131 b 4f 132 1333: /* Calc VA and hash in r29 and r28 for 1T segment */ 134 sldi r29,r5,40 /* vsid << 40 */ 135 clrldi r3,r3,24 /* ea & 0xffffffffff */ 136 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 137 clrldi r5,r5,40 /* vsid & 0xffffff */ 138 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ 139 xor r28,r28,r5 140 or r29,r3,r29 /* VA */ 141 xor r28,r28,r0 /* hash */ 142 143 /* Convert linux PTE bits into HW equivalents */ 1444: andi. r3,r30,0x1fe /* Get basic set of flags */ 145 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 146 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 147 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 148 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 149 andc r0,r30,r0 /* r0 = pte & ~r0 */ 150 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 151 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 152 153 /* We eventually do the icache sync here (maybe inline that 154 * code rather than call a C function...) 155 */ 156BEGIN_FTR_SECTION 157 mr r4,r30 158 mr r5,r7 159 bl .hash_page_do_lazy_icache 160END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 161 162 /* At this point, r3 contains new PP bits, save them in 163 * place of "access" in the param area (sic) 164 */ 165 std r3,STK_PARM(r4)(r1) 166 167 /* Get htab_hash_mask */ 168 ld r4,htab_hash_mask@got(2) 169 ld r27,0(r4) /* htab_hash_mask -> r27 */ 170 171 /* Check if we may already be in the hashtable, in this case, we 172 * go to out-of-line code to try to modify the HPTE 173 */ 174 andi. r0,r31,_PAGE_HASHPTE 175 bne htab_modify_pte 176 177htab_insert_pte: 178 /* Clear hpte bits in new pte (we also clear BUSY btw) and 179 * add _PAGE_HASHPTE 180 */ 181 lis r0,_PAGE_HPTEFLAGS@h 182 ori r0,r0,_PAGE_HPTEFLAGS@l 183 andc r30,r30,r0 184 ori r30,r30,_PAGE_HASHPTE 185 186 /* physical address r5 */ 187 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 188 sldi r5,r5,PAGE_SHIFT 189 190 /* Calculate primary group hash */ 191 and r0,r28,r27 192 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ 193 194 /* Call ppc_md.hpte_insert */ 195 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 196 mr r4,r29 /* Retrieve va */ 197 li r7,0 /* !bolted, !secondary */ 198 li r8,MMU_PAGE_4K /* page size */ 199 ld r9,STK_PARM(r9)(r1) /* segment size */ 200_GLOBAL(htab_call_hpte_insert1) 201 bl . /* Patched by htab_finish_init() */ 202 cmpdi 0,r3,0 203 bge htab_pte_insert_ok /* Insertion successful */ 204 cmpdi 0,r3,-2 /* Critical failure */ 205 beq- htab_pte_insert_failure 206 207 /* Now try secondary slot */ 208 209 /* physical address r5 */ 210 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 211 sldi r5,r5,PAGE_SHIFT 212 213 /* Calculate secondary group hash */ 214 andc r0,r27,r28 215 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 216 217 /* Call ppc_md.hpte_insert */ 218 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 219 mr r4,r29 /* Retrieve va */ 220 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 221 li r8,MMU_PAGE_4K /* page size */ 222 ld r9,STK_PARM(r9)(r1) /* segment size */ 223_GLOBAL(htab_call_hpte_insert2) 224 bl . /* Patched by htab_finish_init() */ 225 cmpdi 0,r3,0 226 bge+ htab_pte_insert_ok /* Insertion successful */ 227 cmpdi 0,r3,-2 /* Critical failure */ 228 beq- htab_pte_insert_failure 229 230 /* Both are full, we need to evict something */ 231 mftb r0 232 /* Pick a random group based on TB */ 233 andi. r0,r0,1 234 mr r5,r28 235 bne 2f 236 not r5,r5 2372: and r0,r5,r27 238 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 239 /* Call ppc_md.hpte_remove */ 240_GLOBAL(htab_call_hpte_remove) 241 bl . /* Patched by htab_finish_init() */ 242 243 /* Try all again */ 244 b htab_insert_pte 245 246htab_bail_ok: 247 li r3,0 248 b htab_bail 249 250htab_pte_insert_ok: 251 /* Insert slot number & secondary bit in PTE */ 252 rldimi r30,r3,12,63-15 253 254 /* Write out the PTE with a normal write 255 * (maybe add eieio may be good still ?) 256 */ 257htab_write_out_pte: 258 ld r6,STK_PARM(r6)(r1) 259 std r30,0(r6) 260 li r3, 0 261htab_bail: 262 ld r27,STK_REG(r27)(r1) 263 ld r28,STK_REG(r28)(r1) 264 ld r29,STK_REG(r29)(r1) 265 ld r30,STK_REG(r30)(r1) 266 ld r31,STK_REG(r31)(r1) 267 addi r1,r1,STACKFRAMESIZE 268 ld r0,16(r1) 269 mtlr r0 270 blr 271 272htab_modify_pte: 273 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 274 mr r4,r3 275 rlwinm r3,r31,32-12,29,31 276 277 /* Secondary group ? if yes, get a inverted hash value */ 278 mr r5,r28 279 andi. r0,r31,_PAGE_SECONDARY 280 beq 1f 281 not r5,r5 2821: 283 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 284 and r0,r5,r27 285 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 286 add r3,r0,r3 /* add slot idx */ 287 288 /* Call ppc_md.hpte_updatepp */ 289 mr r5,r29 /* va */ 290 li r6,MMU_PAGE_4K /* page size */ 291 ld r7,STK_PARM(r9)(r1) /* segment size */ 292 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 293_GLOBAL(htab_call_hpte_updatepp) 294 bl . /* Patched by htab_finish_init() */ 295 296 /* if we failed because typically the HPTE wasn't really here 297 * we try an insertion. 298 */ 299 cmpdi 0,r3,-1 300 beq- htab_insert_pte 301 302 /* Clear the BUSY bit and Write out the PTE */ 303 li r0,_PAGE_BUSY 304 andc r30,r30,r0 305 b htab_write_out_pte 306 307htab_wrong_access: 308 /* Bail out clearing reservation */ 309 stdcx. r31,0,r6 310 li r3,1 311 b htab_bail 312 313htab_pte_insert_failure: 314 /* Bail out restoring old PTE */ 315 ld r6,STK_PARM(r6)(r1) 316 std r31,0(r6) 317 li r3,-1 318 b htab_bail 319 320 321#else /* CONFIG_PPC_64K_PAGES */ 322 323 324/***************************************************************************** 325 * * 326 * 64K SW & 4K or 64K HW in a 4K segment pages implementation * 327 * * 328 *****************************************************************************/ 329 330/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 331 * pte_t *ptep, unsigned long trap, int local, int ssize, 332 * int subpg_prot) 333 */ 334 335/* 336 * For now, we do NOT implement Admixed pages 337 */ 338_GLOBAL(__hash_page_4K) 339 mflr r0 340 std r0,16(r1) 341 stdu r1,-STACKFRAMESIZE(r1) 342 /* Save all params that we need after a function call */ 343 std r6,STK_PARM(r6)(r1) 344 std r8,STK_PARM(r8)(r1) 345 std r9,STK_PARM(r9)(r1) 346 347 /* Save non-volatile registers. 348 * r31 will hold "old PTE" 349 * r30 is "new PTE" 350 * r29 is "va" 351 * r28 is a hash value 352 * r27 is hashtab mask (maybe dynamic patched instead ?) 353 * r26 is the hidx mask 354 * r25 is the index in combo page 355 */ 356 std r25,STK_REG(r25)(r1) 357 std r26,STK_REG(r26)(r1) 358 std r27,STK_REG(r27)(r1) 359 std r28,STK_REG(r28)(r1) 360 std r29,STK_REG(r29)(r1) 361 std r30,STK_REG(r30)(r1) 362 std r31,STK_REG(r31)(r1) 363 364 /* Step 1: 365 * 366 * Check permissions, atomically mark the linux PTE busy 367 * and hashed. 368 */ 3691: 370 ldarx r31,0,r6 371 /* Check access rights (access & ~(pte_val(*ptep))) */ 372 andc. r0,r4,r31 373 bne- htab_wrong_access 374 /* Check if PTE is busy */ 375 andi. r0,r31,_PAGE_BUSY 376 /* If so, just bail out and refault if needed. Someone else 377 * is changing this PTE anyway and might hash it. 378 */ 379 bne- htab_bail_ok 380 /* Prepare new PTE value (turn access RW into DIRTY, then 381 * add BUSY and ACCESSED) 382 */ 383 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 384 or r30,r30,r31 385 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 386 oris r30,r30,_PAGE_COMBO@h 387 /* Write the linux PTE atomically (setting busy) */ 388 stdcx. r30,0,r6 389 bne- 1b 390 isync 391 392 /* Step 2: 393 * 394 * Insert/Update the HPTE in the hash table. At this point, 395 * r4 (access) is re-useable, we use it for the new HPTE flags 396 */ 397 398 /* Load the hidx index */ 399 rldicl r25,r3,64-12,60 400 401BEGIN_FTR_SECTION 402 cmpdi r9,0 /* check segment size */ 403 bne 3f 404END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 405 /* Calc va and put it in r29 */ 406 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ 407 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ 408 or r29,r3,r29 /* r29 = va */ 409 410 /* Calculate hash value for primary slot and store it in r28 */ 411 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 412 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 413 xor r28,r5,r0 414 b 4f 415 4163: /* Calc VA and hash in r29 and r28 for 1T segment */ 417 sldi r29,r5,40 /* vsid << 40 */ 418 clrldi r3,r3,24 /* ea & 0xffffffffff */ 419 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 420 clrldi r5,r5,40 /* vsid & 0xffffff */ 421 rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ 422 xor r28,r28,r5 423 or r29,r3,r29 /* VA */ 424 xor r28,r28,r0 /* hash */ 425 426 /* Convert linux PTE bits into HW equivalents */ 4274: 428#ifdef CONFIG_PPC_SUBPAGE_PROT 429 andc r10,r30,r10 430 andi. r3,r10,0x1fe /* Get basic set of flags */ 431 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 432#else 433 andi. r3,r30,0x1fe /* Get basic set of flags */ 434 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 435#endif 436 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 437 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 438 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 439 andc r0,r3,r0 /* r0 = pte & ~r0 */ 440 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 441 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 442 443 /* We eventually do the icache sync here (maybe inline that 444 * code rather than call a C function...) 445 */ 446BEGIN_FTR_SECTION 447 mr r4,r30 448 mr r5,r7 449 bl .hash_page_do_lazy_icache 450END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 451 452 /* At this point, r3 contains new PP bits, save them in 453 * place of "access" in the param area (sic) 454 */ 455 std r3,STK_PARM(r4)(r1) 456 457 /* Get htab_hash_mask */ 458 ld r4,htab_hash_mask@got(2) 459 ld r27,0(r4) /* htab_hash_mask -> r27 */ 460 461 /* Check if we may already be in the hashtable, in this case, we 462 * go to out-of-line code to try to modify the HPTE. We look for 463 * the bit at (1 >> (index + 32)) 464 */ 465 rldicl. r0,r31,64-12,48 466 li r26,0 /* Default hidx */ 467 beq htab_insert_pte 468 469 /* 470 * Check if the pte was already inserted into the hash table 471 * as a 64k HW page, and invalidate the 64k HPTE if so. 472 */ 473 andis. r0,r31,_PAGE_COMBO@h 474 beq htab_inval_old_hpte 475 476 ld r6,STK_PARM(r6)(r1) 477 ori r26,r6,0x8000 /* Load the hidx mask */ 478 ld r26,0(r26) 479 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ 480 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ 481 bne htab_modify_pte 482 483htab_insert_pte: 484 /* real page number in r5, PTE RPN value + index */ 485 andis. r0,r31,_PAGE_4K_PFN@h 486 srdi r5,r31,PTE_RPN_SHIFT 487 bne- htab_special_pfn 488 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 489 add r5,r5,r25 490htab_special_pfn: 491 sldi r5,r5,HW_PAGE_SHIFT 492 493 /* Calculate primary group hash */ 494 and r0,r28,r27 495 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 496 497 /* Call ppc_md.hpte_insert */ 498 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 499 mr r4,r29 /* Retrieve va */ 500 li r7,0 /* !bolted, !secondary */ 501 li r8,MMU_PAGE_4K /* page size */ 502 ld r9,STK_PARM(r9)(r1) /* segment size */ 503_GLOBAL(htab_call_hpte_insert1) 504 bl . /* patched by htab_finish_init() */ 505 cmpdi 0,r3,0 506 bge htab_pte_insert_ok /* Insertion successful */ 507 cmpdi 0,r3,-2 /* Critical failure */ 508 beq- htab_pte_insert_failure 509 510 /* Now try secondary slot */ 511 512 /* real page number in r5, PTE RPN value + index */ 513 andis. r0,r31,_PAGE_4K_PFN@h 514 srdi r5,r31,PTE_RPN_SHIFT 515 bne- 3f 516 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 517 add r5,r5,r25 5183: sldi r5,r5,HW_PAGE_SHIFT 519 520 /* Calculate secondary group hash */ 521 andc r0,r27,r28 522 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 523 524 /* Call ppc_md.hpte_insert */ 525 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 526 mr r4,r29 /* Retrieve va */ 527 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 528 li r8,MMU_PAGE_4K /* page size */ 529 ld r9,STK_PARM(r9)(r1) /* segment size */ 530_GLOBAL(htab_call_hpte_insert2) 531 bl . /* patched by htab_finish_init() */ 532 cmpdi 0,r3,0 533 bge+ htab_pte_insert_ok /* Insertion successful */ 534 cmpdi 0,r3,-2 /* Critical failure */ 535 beq- htab_pte_insert_failure 536 537 /* Both are full, we need to evict something */ 538 mftb r0 539 /* Pick a random group based on TB */ 540 andi. r0,r0,1 541 mr r5,r28 542 bne 2f 543 not r5,r5 5442: and r0,r5,r27 545 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 546 /* Call ppc_md.hpte_remove */ 547_GLOBAL(htab_call_hpte_remove) 548 bl . /* patched by htab_finish_init() */ 549 550 /* Try all again */ 551 b htab_insert_pte 552 553 /* 554 * Call out to C code to invalidate an 64k HW HPTE that is 555 * useless now that the segment has been switched to 4k pages. 556 */ 557htab_inval_old_hpte: 558 mr r3,r29 /* virtual addr */ 559 mr r4,r31 /* PTE.pte */ 560 li r5,0 /* PTE.hidx */ 561 li r6,MMU_PAGE_64K /* psize */ 562 ld r7,STK_PARM(r9)(r1) /* ssize */ 563 ld r8,STK_PARM(r8)(r1) /* local */ 564 bl .flush_hash_page 565 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ 566 lis r0,_PAGE_HPTE_SUB@h 567 ori r0,r0,_PAGE_HPTE_SUB@l 568 andc r30,r30,r0 569 b htab_insert_pte 570 571htab_bail_ok: 572 li r3,0 573 b htab_bail 574 575htab_pte_insert_ok: 576 /* Insert slot number & secondary bit in PTE second half, 577 * clear _PAGE_BUSY and set approriate HPTE slot bit 578 */ 579 ld r6,STK_PARM(r6)(r1) 580 li r0,_PAGE_BUSY 581 andc r30,r30,r0 582 /* HPTE SUB bit */ 583 li r0,1 584 subfic r5,r25,27 /* Must match bit position in */ 585 sld r0,r0,r5 /* pgtable.h */ 586 or r30,r30,r0 587 /* hindx */ 588 sldi r5,r25,2 589 sld r3,r3,r5 590 li r4,0xf 591 sld r4,r4,r5 592 andc r26,r26,r4 593 or r26,r26,r3 594 ori r5,r6,0x8000 595 std r26,0(r5) 596 lwsync 597 std r30,0(r6) 598 li r3, 0 599htab_bail: 600 ld r25,STK_REG(r25)(r1) 601 ld r26,STK_REG(r26)(r1) 602 ld r27,STK_REG(r27)(r1) 603 ld r28,STK_REG(r28)(r1) 604 ld r29,STK_REG(r29)(r1) 605 ld r30,STK_REG(r30)(r1) 606 ld r31,STK_REG(r31)(r1) 607 addi r1,r1,STACKFRAMESIZE 608 ld r0,16(r1) 609 mtlr r0 610 blr 611 612htab_modify_pte: 613 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 614 mr r4,r3 615 sldi r5,r25,2 616 srd r3,r26,r5 617 618 /* Secondary group ? if yes, get a inverted hash value */ 619 mr r5,r28 620 andi. r0,r3,0x8 /* page secondary ? */ 621 beq 1f 622 not r5,r5 6231: andi. r3,r3,0x7 /* extract idx alone */ 624 625 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 626 and r0,r5,r27 627 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 628 add r3,r0,r3 /* add slot idx */ 629 630 /* Call ppc_md.hpte_updatepp */ 631 mr r5,r29 /* va */ 632 li r6,MMU_PAGE_4K /* page size */ 633 ld r7,STK_PARM(r9)(r1) /* segment size */ 634 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 635_GLOBAL(htab_call_hpte_updatepp) 636 bl . /* patched by htab_finish_init() */ 637 638 /* if we failed because typically the HPTE wasn't really here 639 * we try an insertion. 640 */ 641 cmpdi 0,r3,-1 642 beq- htab_insert_pte 643 644 /* Clear the BUSY bit and Write out the PTE */ 645 li r0,_PAGE_BUSY 646 andc r30,r30,r0 647 ld r6,STK_PARM(r6)(r1) 648 std r30,0(r6) 649 li r3,0 650 b htab_bail 651 652htab_wrong_access: 653 /* Bail out clearing reservation */ 654 stdcx. r31,0,r6 655 li r3,1 656 b htab_bail 657 658htab_pte_insert_failure: 659 /* Bail out restoring old PTE */ 660 ld r6,STK_PARM(r6)(r1) 661 std r31,0(r6) 662 li r3,-1 663 b htab_bail 664 665#endif /* CONFIG_PPC_64K_PAGES */ 666 667#ifdef CONFIG_PPC_HAS_HASH_64K 668 669/***************************************************************************** 670 * * 671 * 64K SW & 64K HW in a 64K segment pages implementation * 672 * * 673 *****************************************************************************/ 674 675_GLOBAL(__hash_page_64K) 676 mflr r0 677 std r0,16(r1) 678 stdu r1,-STACKFRAMESIZE(r1) 679 /* Save all params that we need after a function call */ 680 std r6,STK_PARM(r6)(r1) 681 std r8,STK_PARM(r8)(r1) 682 std r9,STK_PARM(r9)(r1) 683 684 /* Save non-volatile registers. 685 * r31 will hold "old PTE" 686 * r30 is "new PTE" 687 * r29 is "va" 688 * r28 is a hash value 689 * r27 is hashtab mask (maybe dynamic patched instead ?) 690 */ 691 std r27,STK_REG(r27)(r1) 692 std r28,STK_REG(r28)(r1) 693 std r29,STK_REG(r29)(r1) 694 std r30,STK_REG(r30)(r1) 695 std r31,STK_REG(r31)(r1) 696 697 /* Step 1: 698 * 699 * Check permissions, atomically mark the linux PTE busy 700 * and hashed. 701 */ 7021: 703 ldarx r31,0,r6 704 /* Check access rights (access & ~(pte_val(*ptep))) */ 705 andc. r0,r4,r31 706 bne- ht64_wrong_access 707 /* Check if PTE is busy */ 708 andi. r0,r31,_PAGE_BUSY 709 /* If so, just bail out and refault if needed. Someone else 710 * is changing this PTE anyway and might hash it. 711 */ 712 bne- ht64_bail_ok 713BEGIN_FTR_SECTION 714 /* Check if PTE has the cache-inhibit bit set */ 715 andi. r0,r31,_PAGE_NO_CACHE 716 /* If so, bail out and refault as a 4k page */ 717 bne- ht64_bail_ok 718END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE) 719 /* Prepare new PTE value (turn access RW into DIRTY, then 720 * add BUSY and ACCESSED) 721 */ 722 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 723 or r30,r30,r31 724 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 725 /* Write the linux PTE atomically (setting busy) */ 726 stdcx. r30,0,r6 727 bne- 1b 728 isync 729 730 /* Step 2: 731 * 732 * Insert/Update the HPTE in the hash table. At this point, 733 * r4 (access) is re-useable, we use it for the new HPTE flags 734 */ 735 736BEGIN_FTR_SECTION 737 cmpdi r9,0 /* check segment size */ 738 bne 3f 739END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 740 /* Calc va and put it in r29 */ 741 rldicr r29,r5,28,63-28 742 rldicl r3,r3,0,36 743 or r29,r3,r29 744 745 /* Calculate hash value for primary slot and store it in r28 */ 746 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 747 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ 748 xor r28,r5,r0 749 b 4f 750 7513: /* Calc VA and hash in r29 and r28 for 1T segment */ 752 sldi r29,r5,40 /* vsid << 40 */ 753 clrldi r3,r3,24 /* ea & 0xffffffffff */ 754 rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ 755 clrldi r5,r5,40 /* vsid & 0xffffff */ 756 rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ 757 xor r28,r28,r5 758 or r29,r3,r29 /* VA */ 759 xor r28,r28,r0 /* hash */ 760 761 /* Convert linux PTE bits into HW equivalents */ 7624: andi. r3,r30,0x1fe /* Get basic set of flags */ 763 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 764 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 765 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 766 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 767 andc r0,r30,r0 /* r0 = pte & ~r0 */ 768 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 769 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 770 771 /* We eventually do the icache sync here (maybe inline that 772 * code rather than call a C function...) 773 */ 774BEGIN_FTR_SECTION 775 mr r4,r30 776 mr r5,r7 777 bl .hash_page_do_lazy_icache 778END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 779 780 /* At this point, r3 contains new PP bits, save them in 781 * place of "access" in the param area (sic) 782 */ 783 std r3,STK_PARM(r4)(r1) 784 785 /* Get htab_hash_mask */ 786 ld r4,htab_hash_mask@got(2) 787 ld r27,0(r4) /* htab_hash_mask -> r27 */ 788 789 /* Check if we may already be in the hashtable, in this case, we 790 * go to out-of-line code to try to modify the HPTE 791 */ 792 rldicl. r0,r31,64-12,48 793 bne ht64_modify_pte 794 795ht64_insert_pte: 796 /* Clear hpte bits in new pte (we also clear BUSY btw) and 797 * add _PAGE_HPTE_SUB0 798 */ 799 lis r0,_PAGE_HPTEFLAGS@h 800 ori r0,r0,_PAGE_HPTEFLAGS@l 801 andc r30,r30,r0 802#ifdef CONFIG_PPC_64K_PAGES 803 oris r30,r30,_PAGE_HPTE_SUB0@h 804#else 805 ori r30,r30,_PAGE_HASHPTE 806#endif 807 /* Phyical address in r5 */ 808 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 809 sldi r5,r5,PAGE_SHIFT 810 811 /* Calculate primary group hash */ 812 and r0,r28,r27 813 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 814 815 /* Call ppc_md.hpte_insert */ 816 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 817 mr r4,r29 /* Retrieve va */ 818 li r7,0 /* !bolted, !secondary */ 819 li r8,MMU_PAGE_64K 820 ld r9,STK_PARM(r9)(r1) /* segment size */ 821_GLOBAL(ht64_call_hpte_insert1) 822 bl . /* patched by htab_finish_init() */ 823 cmpdi 0,r3,0 824 bge ht64_pte_insert_ok /* Insertion successful */ 825 cmpdi 0,r3,-2 /* Critical failure */ 826 beq- ht64_pte_insert_failure 827 828 /* Now try secondary slot */ 829 830 /* Phyical address in r5 */ 831 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 832 sldi r5,r5,PAGE_SHIFT 833 834 /* Calculate secondary group hash */ 835 andc r0,r27,r28 836 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 837 838 /* Call ppc_md.hpte_insert */ 839 ld r6,STK_PARM(r4)(r1) /* Retrieve new pp bits */ 840 mr r4,r29 /* Retrieve va */ 841 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 842 li r8,MMU_PAGE_64K 843 ld r9,STK_PARM(r9)(r1) /* segment size */ 844_GLOBAL(ht64_call_hpte_insert2) 845 bl . /* patched by htab_finish_init() */ 846 cmpdi 0,r3,0 847 bge+ ht64_pte_insert_ok /* Insertion successful */ 848 cmpdi 0,r3,-2 /* Critical failure */ 849 beq- ht64_pte_insert_failure 850 851 /* Both are full, we need to evict something */ 852 mftb r0 853 /* Pick a random group based on TB */ 854 andi. r0,r0,1 855 mr r5,r28 856 bne 2f 857 not r5,r5 8582: and r0,r5,r27 859 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 860 /* Call ppc_md.hpte_remove */ 861_GLOBAL(ht64_call_hpte_remove) 862 bl . /* patched by htab_finish_init() */ 863 864 /* Try all again */ 865 b ht64_insert_pte 866 867ht64_bail_ok: 868 li r3,0 869 b ht64_bail 870 871ht64_pte_insert_ok: 872 /* Insert slot number & secondary bit in PTE */ 873 rldimi r30,r3,12,63-15 874 875 /* Write out the PTE with a normal write 876 * (maybe add eieio may be good still ?) 877 */ 878ht64_write_out_pte: 879 ld r6,STK_PARM(r6)(r1) 880 std r30,0(r6) 881 li r3, 0 882ht64_bail: 883 ld r27,STK_REG(r27)(r1) 884 ld r28,STK_REG(r28)(r1) 885 ld r29,STK_REG(r29)(r1) 886 ld r30,STK_REG(r30)(r1) 887 ld r31,STK_REG(r31)(r1) 888 addi r1,r1,STACKFRAMESIZE 889 ld r0,16(r1) 890 mtlr r0 891 blr 892 893ht64_modify_pte: 894 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 895 mr r4,r3 896 rlwinm r3,r31,32-12,29,31 897 898 /* Secondary group ? if yes, get a inverted hash value */ 899 mr r5,r28 900 andi. r0,r31,_PAGE_F_SECOND 901 beq 1f 902 not r5,r5 9031: 904 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 905 and r0,r5,r27 906 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 907 add r3,r0,r3 /* add slot idx */ 908 909 /* Call ppc_md.hpte_updatepp */ 910 mr r5,r29 /* va */ 911 li r6,MMU_PAGE_64K 912 ld r7,STK_PARM(r9)(r1) /* segment size */ 913 ld r8,STK_PARM(r8)(r1) /* get "local" param */ 914_GLOBAL(ht64_call_hpte_updatepp) 915 bl . /* patched by htab_finish_init() */ 916 917 /* if we failed because typically the HPTE wasn't really here 918 * we try an insertion. 919 */ 920 cmpdi 0,r3,-1 921 beq- ht64_insert_pte 922 923 /* Clear the BUSY bit and Write out the PTE */ 924 li r0,_PAGE_BUSY 925 andc r30,r30,r0 926 b ht64_write_out_pte 927 928ht64_wrong_access: 929 /* Bail out clearing reservation */ 930 stdcx. r31,0,r6 931 li r3,1 932 b ht64_bail 933 934ht64_pte_insert_failure: 935 /* Bail out restoring old PTE */ 936 ld r6,STK_PARM(r6)(r1) 937 std r31,0(r6) 938 li r3,-1 939 b ht64_bail 940 941 942#endif /* CONFIG_PPC_HAS_HASH_64K */ 943 944 945/***************************************************************************** 946 * * 947 * Huge pages implementation is in hugetlbpage.c * 948 * * 949 *****************************************************************************/ 950