1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/translation-block.h" 31 #include "exec/plugin-gen.h" 32 #include "tcg-internal.h" 33 #include "tcg-has.h" 34 #include "tcg-target-mo.h" 35 36 static void check_max_alignment(unsigned a_bits) 37 { 38 /* 39 * The requested alignment cannot overlap the TLB flags. 40 * FIXME: Must keep the count up-to-date with "exec/cpu-all.h". 41 */ 42 if (tcg_use_softmmu) { 43 tcg_debug_assert(a_bits + 5 <= tcg_ctx->page_bits); 44 } 45 } 46 47 static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 48 { 49 unsigned a_bits = memop_alignment_bits(op); 50 51 check_max_alignment(a_bits); 52 53 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 54 if (a_bits == (op & MO_SIZE)) { 55 op = (op & ~MO_AMASK) | MO_ALIGN; 56 } 57 58 switch (op & MO_SIZE) { 59 case MO_8: 60 op &= ~MO_BSWAP; 61 break; 62 case MO_16: 63 break; 64 case MO_32: 65 if (!is64) { 66 op &= ~MO_SIGN; 67 } 68 break; 69 case MO_64: 70 if (is64) { 71 op &= ~MO_SIGN; 72 break; 73 } 74 /* fall through */ 75 default: 76 g_assert_not_reached(); 77 } 78 if (st) { 79 op &= ~MO_SIGN; 80 } 81 82 /* In serial mode, reduce atomicity. */ 83 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 84 op &= ~MO_ATOM_MASK; 85 op |= MO_ATOM_NONE; 86 } 87 88 return op; 89 } 90 91 static void gen_ldst(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh, 92 TCGTemp *addr, MemOpIdx oi) 93 { 94 if (vh) { 95 tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh), temp_arg(addr), oi); 96 } else { 97 tcg_gen_op3(opc, type, temp_arg(vl), temp_arg(addr), oi); 98 } 99 } 100 101 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) 102 { 103 if (TCG_TARGET_REG_BITS == 32) { 104 TCGTemp *vl = tcgv_i32_temp(TCGV_LOW(v)); 105 TCGTemp *vh = tcgv_i32_temp(TCGV_HIGH(v)); 106 gen_ldst(opc, TCG_TYPE_I64, vl, vh, addr, oi); 107 } else { 108 gen_ldst(opc, TCG_TYPE_I64, tcgv_i64_temp(v), NULL, addr, oi); 109 } 110 } 111 112 static void tcg_gen_req_mo(TCGBar type) 113 { 114 type &= tcg_ctx->guest_mo; 115 type &= ~TCG_TARGET_DEFAULT_MO; 116 if (type) { 117 tcg_gen_mb(type | TCG_BAR_SC); 118 } 119 } 120 121 /* Only required for loads, where value might overlap addr. */ 122 static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr) 123 { 124 #ifdef CONFIG_PLUGIN 125 if (tcg_ctx->plugin_insn != NULL) { 126 /* Save a copy of the vaddr for use after a load. */ 127 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 128 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 129 tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr)); 130 } else { 131 tcg_gen_mov_i64(temp, temp_tcgv_i64(addr)); 132 } 133 return temp; 134 } 135 #endif 136 return NULL; 137 } 138 139 #ifdef CONFIG_PLUGIN 140 static void 141 plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi, 142 enum qemu_plugin_mem_rw rw) 143 { 144 if (tcg_ctx->plugin_insn != NULL) { 145 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 146 147 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 148 if (!copy_addr) { 149 copy_addr = tcg_temp_ebb_new_i64(); 150 tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr)); 151 } 152 tcg_gen_plugin_mem_cb(copy_addr, info); 153 tcg_temp_free_i64(copy_addr); 154 } else { 155 if (copy_addr) { 156 tcg_gen_plugin_mem_cb(copy_addr, info); 157 tcg_temp_free_i64(copy_addr); 158 } else { 159 tcg_gen_plugin_mem_cb(temp_tcgv_i64(orig_addr), info); 160 } 161 } 162 } 163 } 164 #endif 165 166 static void 167 plugin_gen_mem_callbacks_i32(TCGv_i32 val, 168 TCGv_i64 copy_addr, TCGTemp *orig_addr, 169 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 170 { 171 #ifdef CONFIG_PLUGIN 172 if (tcg_ctx->plugin_insn != NULL) { 173 tcg_gen_st_i32(val, tcg_env, 174 offsetof(CPUState, neg.plugin_mem_value_low) - 175 sizeof(CPUState) + (HOST_BIG_ENDIAN * 4)); 176 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 177 } 178 #endif 179 } 180 181 static void 182 plugin_gen_mem_callbacks_i64(TCGv_i64 val, 183 TCGv_i64 copy_addr, TCGTemp *orig_addr, 184 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 185 { 186 #ifdef CONFIG_PLUGIN 187 if (tcg_ctx->plugin_insn != NULL) { 188 tcg_gen_st_i64(val, tcg_env, 189 offsetof(CPUState, neg.plugin_mem_value_low) - 190 sizeof(CPUState)); 191 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 192 } 193 #endif 194 } 195 196 static void 197 plugin_gen_mem_callbacks_i128(TCGv_i128 val, 198 TCGv_i64 copy_addr, TCGTemp *orig_addr, 199 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 200 { 201 #ifdef CONFIG_PLUGIN 202 if (tcg_ctx->plugin_insn != NULL) { 203 tcg_gen_st_i64(TCGV128_LOW(val), tcg_env, 204 offsetof(CPUState, neg.plugin_mem_value_low) - 205 sizeof(CPUState)); 206 tcg_gen_st_i64(TCGV128_HIGH(val), tcg_env, 207 offsetof(CPUState, neg.plugin_mem_value_high) - 208 sizeof(CPUState)); 209 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 210 } 211 #endif 212 } 213 214 static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, 215 TCGArg idx, MemOp memop) 216 { 217 MemOp orig_memop; 218 MemOpIdx orig_oi, oi; 219 TCGv_i64 copy_addr; 220 TCGOpcode opc; 221 222 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 223 orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0); 224 orig_oi = oi = make_memop_idx(memop, idx); 225 226 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 227 memop &= ~MO_BSWAP; 228 /* The bswap primitive benefits from zero-extended input. */ 229 if ((memop & MO_SSIZE) == MO_SW) { 230 memop &= ~MO_SIGN; 231 } 232 oi = make_memop_idx(memop, idx); 233 } 234 235 copy_addr = plugin_maybe_preserve_addr(addr); 236 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 237 opc = INDEX_op_qemu_ld_a32_i32; 238 } else { 239 opc = INDEX_op_qemu_ld_a64_i32; 240 } 241 gen_ldst(opc, TCG_TYPE_I32, tcgv_i32_temp(val), NULL, addr, oi); 242 plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi, 243 QEMU_PLUGIN_MEM_R); 244 245 if ((orig_memop ^ memop) & MO_BSWAP) { 246 switch (orig_memop & MO_SIZE) { 247 case MO_16: 248 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 249 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 250 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 251 break; 252 case MO_32: 253 tcg_gen_bswap32_i32(val, val); 254 break; 255 default: 256 g_assert_not_reached(); 257 } 258 } 259 } 260 261 void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 262 MemOp memop, TCGType addr_type) 263 { 264 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 265 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 266 tcg_gen_qemu_ld_i32_int(val, addr, idx, memop); 267 } 268 269 static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, 270 TCGArg idx, MemOp memop) 271 { 272 TCGv_i32 swap = NULL; 273 MemOpIdx orig_oi, oi; 274 TCGOpcode opc; 275 276 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 277 memop = tcg_canonicalize_memop(memop, 0, 1); 278 orig_oi = oi = make_memop_idx(memop, idx); 279 280 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 281 swap = tcg_temp_ebb_new_i32(); 282 switch (memop & MO_SIZE) { 283 case MO_16: 284 tcg_gen_bswap16_i32(swap, val, 0); 285 break; 286 case MO_32: 287 tcg_gen_bswap32_i32(swap, val); 288 break; 289 default: 290 g_assert_not_reached(); 291 } 292 val = swap; 293 memop &= ~MO_BSWAP; 294 oi = make_memop_idx(memop, idx); 295 } 296 297 if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) { 298 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 299 opc = INDEX_op_qemu_st8_a32_i32; 300 } else { 301 opc = INDEX_op_qemu_st8_a64_i32; 302 } 303 } else { 304 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 305 opc = INDEX_op_qemu_st_a32_i32; 306 } else { 307 opc = INDEX_op_qemu_st_a64_i32; 308 } 309 } 310 gen_ldst(opc, TCG_TYPE_I32, tcgv_i32_temp(val), NULL, addr, oi); 311 plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 312 313 if (swap) { 314 tcg_temp_free_i32(swap); 315 } 316 } 317 318 void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 319 MemOp memop, TCGType addr_type) 320 { 321 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 322 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 323 tcg_gen_qemu_st_i32_int(val, addr, idx, memop); 324 } 325 326 static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, 327 TCGArg idx, MemOp memop) 328 { 329 MemOp orig_memop; 330 MemOpIdx orig_oi, oi; 331 TCGv_i64 copy_addr; 332 TCGOpcode opc; 333 334 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 335 tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop); 336 if (memop & MO_SIGN) { 337 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 338 } else { 339 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 340 } 341 return; 342 } 343 344 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 345 orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0); 346 orig_oi = oi = make_memop_idx(memop, idx); 347 348 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 349 memop &= ~MO_BSWAP; 350 /* The bswap primitive benefits from zero-extended input. */ 351 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 352 memop &= ~MO_SIGN; 353 } 354 oi = make_memop_idx(memop, idx); 355 } 356 357 copy_addr = plugin_maybe_preserve_addr(addr); 358 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 359 opc = INDEX_op_qemu_ld_a32_i64; 360 } else { 361 opc = INDEX_op_qemu_ld_a64_i64; 362 } 363 gen_ldst_i64(opc, val, addr, oi); 364 plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi, 365 QEMU_PLUGIN_MEM_R); 366 367 if ((orig_memop ^ memop) & MO_BSWAP) { 368 int flags = (orig_memop & MO_SIGN 369 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 370 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 371 switch (orig_memop & MO_SIZE) { 372 case MO_16: 373 tcg_gen_bswap16_i64(val, val, flags); 374 break; 375 case MO_32: 376 tcg_gen_bswap32_i64(val, val, flags); 377 break; 378 case MO_64: 379 tcg_gen_bswap64_i64(val, val); 380 break; 381 default: 382 g_assert_not_reached(); 383 } 384 } 385 } 386 387 void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 388 MemOp memop, TCGType addr_type) 389 { 390 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 391 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 392 tcg_gen_qemu_ld_i64_int(val, addr, idx, memop); 393 } 394 395 static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, 396 TCGArg idx, MemOp memop) 397 { 398 TCGv_i64 swap = NULL; 399 MemOpIdx orig_oi, oi; 400 TCGOpcode opc; 401 402 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 403 tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop); 404 return; 405 } 406 407 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 408 memop = tcg_canonicalize_memop(memop, 1, 1); 409 orig_oi = oi = make_memop_idx(memop, idx); 410 411 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 412 swap = tcg_temp_ebb_new_i64(); 413 switch (memop & MO_SIZE) { 414 case MO_16: 415 tcg_gen_bswap16_i64(swap, val, 0); 416 break; 417 case MO_32: 418 tcg_gen_bswap32_i64(swap, val, 0); 419 break; 420 case MO_64: 421 tcg_gen_bswap64_i64(swap, val); 422 break; 423 default: 424 g_assert_not_reached(); 425 } 426 val = swap; 427 memop &= ~MO_BSWAP; 428 oi = make_memop_idx(memop, idx); 429 } 430 431 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 432 opc = INDEX_op_qemu_st_a32_i64; 433 } else { 434 opc = INDEX_op_qemu_st_a64_i64; 435 } 436 gen_ldst_i64(opc, val, addr, oi); 437 plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 438 439 if (swap) { 440 tcg_temp_free_i64(swap); 441 } 442 } 443 444 void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 445 MemOp memop, TCGType addr_type) 446 { 447 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 448 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 449 tcg_gen_qemu_st_i64_int(val, addr, idx, memop); 450 } 451 452 /* 453 * Return true if @mop, without knowledge of the pointer alignment, 454 * does not require 16-byte atomicity, and it would be adventagous 455 * to avoid a call to a helper function. 456 */ 457 static bool use_two_i64_for_i128(MemOp mop) 458 { 459 /* Two softmmu tlb lookups is larger than one function call. */ 460 if (tcg_use_softmmu) { 461 return false; 462 } 463 464 /* 465 * For user-only, two 64-bit operations may well be smaller than a call. 466 * Determine if that would be legal for the requested atomicity. 467 */ 468 switch (mop & MO_ATOM_MASK) { 469 case MO_ATOM_NONE: 470 case MO_ATOM_IFALIGN_PAIR: 471 return true; 472 case MO_ATOM_IFALIGN: 473 case MO_ATOM_SUBALIGN: 474 case MO_ATOM_WITHIN16: 475 case MO_ATOM_WITHIN16_PAIR: 476 return false; 477 default: 478 g_assert_not_reached(); 479 } 480 } 481 482 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) 483 { 484 MemOp mop_1 = orig, mop_2; 485 486 /* Reduce the size to 64-bit. */ 487 mop_1 = (mop_1 & ~MO_SIZE) | MO_64; 488 489 /* Retain the alignment constraints of the original. */ 490 switch (orig & MO_AMASK) { 491 case MO_UNALN: 492 case MO_ALIGN_2: 493 case MO_ALIGN_4: 494 mop_2 = mop_1; 495 break; 496 case MO_ALIGN_8: 497 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ 498 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 499 mop_2 = mop_1; 500 break; 501 case MO_ALIGN: 502 /* Second has 8-byte alignment; first has 16-byte alignment. */ 503 mop_2 = mop_1; 504 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; 505 break; 506 case MO_ALIGN_16: 507 case MO_ALIGN_32: 508 case MO_ALIGN_64: 509 /* Second has 8-byte alignment; first retains original. */ 510 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 511 break; 512 default: 513 g_assert_not_reached(); 514 } 515 516 /* Use a memory ordering implemented by the host. */ 517 if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) { 518 mop_1 &= ~MO_BSWAP; 519 mop_2 &= ~MO_BSWAP; 520 } 521 522 ret[0] = mop_1; 523 ret[1] = mop_2; 524 } 525 526 static TCGv_i64 maybe_extend_addr64(TCGTemp *addr) 527 { 528 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 529 TCGv_i64 a64 = tcg_temp_ebb_new_i64(); 530 tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr)); 531 return a64; 532 } 533 return temp_tcgv_i64(addr); 534 } 535 536 static void maybe_free_addr64(TCGv_i64 a64) 537 { 538 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 539 tcg_temp_free_i64(a64); 540 } 541 } 542 543 static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, 544 TCGArg idx, MemOp memop) 545 { 546 MemOpIdx orig_oi; 547 TCGv_i64 ext_addr = NULL; 548 TCGOpcode opc; 549 550 check_max_alignment(memop_alignment_bits(memop)); 551 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 552 553 /* In serial mode, reduce atomicity. */ 554 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 555 memop &= ~MO_ATOM_MASK; 556 memop |= MO_ATOM_NONE; 557 } 558 orig_oi = make_memop_idx(memop, idx); 559 560 /* TODO: For now, force 32-bit hosts to use the helper. */ 561 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 562 TCGv_i64 lo, hi; 563 bool need_bswap = false; 564 MemOpIdx oi = orig_oi; 565 566 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 567 lo = TCGV128_HIGH(val); 568 hi = TCGV128_LOW(val); 569 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 570 need_bswap = true; 571 } else { 572 lo = TCGV128_LOW(val); 573 hi = TCGV128_HIGH(val); 574 } 575 576 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 577 opc = INDEX_op_qemu_ld_a32_i128; 578 } else { 579 opc = INDEX_op_qemu_ld_a64_i128; 580 } 581 gen_ldst(opc, TCG_TYPE_I128, tcgv_i64_temp(lo), 582 tcgv_i64_temp(hi), addr, oi); 583 584 if (need_bswap) { 585 tcg_gen_bswap64_i64(lo, lo); 586 tcg_gen_bswap64_i64(hi, hi); 587 } 588 } else if (use_two_i64_for_i128(memop)) { 589 MemOp mop[2]; 590 TCGTemp *addr_p8; 591 TCGv_i64 x, y; 592 bool need_bswap; 593 594 canonicalize_memop_i128_as_i64(mop, memop); 595 need_bswap = (mop[0] ^ memop) & MO_BSWAP; 596 597 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 598 opc = INDEX_op_qemu_ld_a32_i64; 599 } else { 600 opc = INDEX_op_qemu_ld_a64_i64; 601 } 602 603 /* 604 * Since there are no global TCGv_i128, there is no visible state 605 * changed if the second load faults. Load directly into the two 606 * subwords. 607 */ 608 if ((memop & MO_BSWAP) == MO_LE) { 609 x = TCGV128_LOW(val); 610 y = TCGV128_HIGH(val); 611 } else { 612 x = TCGV128_HIGH(val); 613 y = TCGV128_LOW(val); 614 } 615 616 gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx)); 617 618 if (need_bswap) { 619 tcg_gen_bswap64_i64(x, x); 620 } 621 622 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 623 TCGv_i32 t = tcg_temp_ebb_new_i32(); 624 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 625 addr_p8 = tcgv_i32_temp(t); 626 } else { 627 TCGv_i64 t = tcg_temp_ebb_new_i64(); 628 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 629 addr_p8 = tcgv_i64_temp(t); 630 } 631 632 gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx)); 633 tcg_temp_free_internal(addr_p8); 634 635 if (need_bswap) { 636 tcg_gen_bswap64_i64(y, y); 637 } 638 } else { 639 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 640 ext_addr = tcg_temp_ebb_new_i64(); 641 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 642 addr = tcgv_i64_temp(ext_addr); 643 } 644 gen_helper_ld_i128(val, tcg_env, temp_tcgv_i64(addr), 645 tcg_constant_i32(orig_oi)); 646 } 647 648 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 649 QEMU_PLUGIN_MEM_R); 650 } 651 652 void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 653 MemOp memop, TCGType addr_type) 654 { 655 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 656 tcg_debug_assert((memop & MO_SIZE) == MO_128); 657 tcg_debug_assert((memop & MO_SIGN) == 0); 658 tcg_gen_qemu_ld_i128_int(val, addr, idx, memop); 659 } 660 661 static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, 662 TCGArg idx, MemOp memop) 663 { 664 MemOpIdx orig_oi; 665 TCGv_i64 ext_addr = NULL; 666 TCGOpcode opc; 667 668 check_max_alignment(memop_alignment_bits(memop)); 669 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); 670 671 /* In serial mode, reduce atomicity. */ 672 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 673 memop &= ~MO_ATOM_MASK; 674 memop |= MO_ATOM_NONE; 675 } 676 orig_oi = make_memop_idx(memop, idx); 677 678 /* TODO: For now, force 32-bit hosts to use the helper. */ 679 680 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 681 TCGv_i64 lo, hi; 682 MemOpIdx oi = orig_oi; 683 bool need_bswap = false; 684 685 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 686 lo = tcg_temp_ebb_new_i64(); 687 hi = tcg_temp_ebb_new_i64(); 688 tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val)); 689 tcg_gen_bswap64_i64(hi, TCGV128_LOW(val)); 690 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 691 need_bswap = true; 692 } else { 693 lo = TCGV128_LOW(val); 694 hi = TCGV128_HIGH(val); 695 } 696 697 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 698 opc = INDEX_op_qemu_st_a32_i128; 699 } else { 700 opc = INDEX_op_qemu_st_a64_i128; 701 } 702 gen_ldst(opc, TCG_TYPE_I128, tcgv_i64_temp(lo), 703 tcgv_i64_temp(hi), addr, oi); 704 705 if (need_bswap) { 706 tcg_temp_free_i64(lo); 707 tcg_temp_free_i64(hi); 708 } 709 } else if (use_two_i64_for_i128(memop)) { 710 MemOp mop[2]; 711 TCGTemp *addr_p8; 712 TCGv_i64 x, y, b = NULL; 713 714 canonicalize_memop_i128_as_i64(mop, memop); 715 716 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 717 opc = INDEX_op_qemu_st_a32_i64; 718 } else { 719 opc = INDEX_op_qemu_st_a64_i64; 720 } 721 722 if ((memop & MO_BSWAP) == MO_LE) { 723 x = TCGV128_LOW(val); 724 y = TCGV128_HIGH(val); 725 } else { 726 x = TCGV128_HIGH(val); 727 y = TCGV128_LOW(val); 728 } 729 730 if ((mop[0] ^ memop) & MO_BSWAP) { 731 b = tcg_temp_ebb_new_i64(); 732 tcg_gen_bswap64_i64(b, x); 733 x = b; 734 } 735 736 gen_ldst_i64(opc, x, addr, make_memop_idx(mop[0], idx)); 737 738 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 739 TCGv_i32 t = tcg_temp_ebb_new_i32(); 740 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 741 addr_p8 = tcgv_i32_temp(t); 742 } else { 743 TCGv_i64 t = tcg_temp_ebb_new_i64(); 744 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 745 addr_p8 = tcgv_i64_temp(t); 746 } 747 748 if (b) { 749 tcg_gen_bswap64_i64(b, y); 750 gen_ldst_i64(opc, b, addr_p8, make_memop_idx(mop[1], idx)); 751 tcg_temp_free_i64(b); 752 } else { 753 gen_ldst_i64(opc, y, addr_p8, make_memop_idx(mop[1], idx)); 754 } 755 tcg_temp_free_internal(addr_p8); 756 } else { 757 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 758 ext_addr = tcg_temp_ebb_new_i64(); 759 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 760 addr = tcgv_i64_temp(ext_addr); 761 } 762 gen_helper_st_i128(tcg_env, temp_tcgv_i64(addr), val, 763 tcg_constant_i32(orig_oi)); 764 } 765 766 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 767 QEMU_PLUGIN_MEM_W); 768 } 769 770 void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 771 MemOp memop, TCGType addr_type) 772 { 773 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 774 tcg_debug_assert((memop & MO_SIZE) == MO_128); 775 tcg_debug_assert((memop & MO_SIGN) == 0); 776 tcg_gen_qemu_st_i128_int(val, addr, idx, memop); 777 } 778 779 void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 780 { 781 switch (opc & MO_SSIZE) { 782 case MO_SB: 783 tcg_gen_ext8s_i32(ret, val); 784 break; 785 case MO_UB: 786 tcg_gen_ext8u_i32(ret, val); 787 break; 788 case MO_SW: 789 tcg_gen_ext16s_i32(ret, val); 790 break; 791 case MO_UW: 792 tcg_gen_ext16u_i32(ret, val); 793 break; 794 case MO_UL: 795 case MO_SL: 796 tcg_gen_mov_i32(ret, val); 797 break; 798 default: 799 g_assert_not_reached(); 800 } 801 } 802 803 void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 804 { 805 switch (opc & MO_SSIZE) { 806 case MO_SB: 807 tcg_gen_ext8s_i64(ret, val); 808 break; 809 case MO_UB: 810 tcg_gen_ext8u_i64(ret, val); 811 break; 812 case MO_SW: 813 tcg_gen_ext16s_i64(ret, val); 814 break; 815 case MO_UW: 816 tcg_gen_ext16u_i64(ret, val); 817 break; 818 case MO_SL: 819 tcg_gen_ext32s_i64(ret, val); 820 break; 821 case MO_UL: 822 tcg_gen_ext32u_i64(ret, val); 823 break; 824 case MO_UQ: 825 case MO_SQ: 826 tcg_gen_mov_i64(ret, val); 827 break; 828 default: 829 g_assert_not_reached(); 830 } 831 } 832 833 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64, 834 TCGv_i32, TCGv_i32, TCGv_i32); 835 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64, 836 TCGv_i64, TCGv_i64, TCGv_i32); 837 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64, 838 TCGv_i128, TCGv_i128, TCGv_i32); 839 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64, 840 TCGv_i32, TCGv_i32); 841 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64, 842 TCGv_i64, TCGv_i32); 843 844 #ifdef CONFIG_ATOMIC64 845 # define WITH_ATOMIC64(X) X, 846 #else 847 # define WITH_ATOMIC64(X) 848 #endif 849 #if HAVE_CMPXCHG128 850 # define WITH_ATOMIC128(X) X, 851 #else 852 # define WITH_ATOMIC128(X) 853 #endif 854 855 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 856 [MO_8] = gen_helper_atomic_cmpxchgb, 857 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 858 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 859 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 860 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 861 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 862 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 863 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) 864 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) 865 }; 866 867 static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 868 TCGv_i32 cmpv, TCGv_i32 newv, 869 TCGArg idx, MemOp memop) 870 { 871 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 872 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 873 874 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 875 876 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN); 877 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 878 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 879 tcg_temp_free_i32(t2); 880 881 if (memop & MO_SIGN) { 882 tcg_gen_ext_i32(retv, t1, memop); 883 } else { 884 tcg_gen_mov_i32(retv, t1); 885 } 886 tcg_temp_free_i32(t1); 887 } 888 889 void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 890 TCGv_i32 cmpv, TCGv_i32 newv, 891 TCGArg idx, MemOp memop, 892 TCGType addr_type) 893 { 894 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 895 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 896 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 897 } 898 899 static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 900 TCGv_i32 cmpv, TCGv_i32 newv, 901 TCGArg idx, MemOp memop) 902 { 903 gen_atomic_cx_i32 gen; 904 TCGv_i64 a64; 905 MemOpIdx oi; 906 907 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 908 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 909 return; 910 } 911 912 memop = tcg_canonicalize_memop(memop, 0, 0); 913 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 914 tcg_debug_assert(gen != NULL); 915 916 oi = make_memop_idx(memop & ~MO_SIGN, idx); 917 a64 = maybe_extend_addr64(addr); 918 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 919 maybe_free_addr64(a64); 920 921 if (memop & MO_SIGN) { 922 tcg_gen_ext_i32(retv, retv, memop); 923 } 924 } 925 926 void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 927 TCGv_i32 cmpv, TCGv_i32 newv, 928 TCGArg idx, MemOp memop, 929 TCGType addr_type) 930 { 931 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 932 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 933 tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 934 } 935 936 static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 937 TCGv_i64 cmpv, TCGv_i64 newv, 938 TCGArg idx, MemOp memop) 939 { 940 TCGv_i64 t1, t2; 941 942 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 943 tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 944 TCGV_LOW(newv), idx, memop); 945 if (memop & MO_SIGN) { 946 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 947 } else { 948 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 949 } 950 return; 951 } 952 953 t1 = tcg_temp_ebb_new_i64(); 954 t2 = tcg_temp_ebb_new_i64(); 955 956 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 957 958 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN); 959 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 960 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 961 tcg_temp_free_i64(t2); 962 963 if (memop & MO_SIGN) { 964 tcg_gen_ext_i64(retv, t1, memop); 965 } else { 966 tcg_gen_mov_i64(retv, t1); 967 } 968 tcg_temp_free_i64(t1); 969 } 970 971 void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 972 TCGv_i64 cmpv, TCGv_i64 newv, 973 TCGArg idx, MemOp memop, 974 TCGType addr_type) 975 { 976 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 977 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 978 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 979 } 980 981 static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 982 TCGv_i64 cmpv, TCGv_i64 newv, 983 TCGArg idx, MemOp memop) 984 { 985 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 986 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 987 return; 988 } 989 990 if ((memop & MO_SIZE) == MO_64) { 991 gen_atomic_cx_i64 gen; 992 993 memop = tcg_canonicalize_memop(memop, 1, 0); 994 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 995 if (gen) { 996 MemOpIdx oi = make_memop_idx(memop, idx); 997 TCGv_i64 a64 = maybe_extend_addr64(addr); 998 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 999 maybe_free_addr64(a64); 1000 return; 1001 } 1002 1003 gen_helper_exit_atomic(tcg_env); 1004 1005 /* 1006 * Produce a result for a well-formed opcode stream. This satisfies 1007 * liveness for set before used, which happens before this dead code 1008 * is removed. 1009 */ 1010 tcg_gen_movi_i64(retv, 0); 1011 return; 1012 } 1013 1014 if (TCG_TARGET_REG_BITS == 32) { 1015 tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 1016 TCGV_LOW(newv), idx, memop); 1017 if (memop & MO_SIGN) { 1018 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 1019 } else { 1020 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 1021 } 1022 } else { 1023 TCGv_i32 c32 = tcg_temp_ebb_new_i32(); 1024 TCGv_i32 n32 = tcg_temp_ebb_new_i32(); 1025 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 1026 1027 tcg_gen_extrl_i64_i32(c32, cmpv); 1028 tcg_gen_extrl_i64_i32(n32, newv); 1029 tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32, 1030 idx, memop & ~MO_SIGN); 1031 tcg_temp_free_i32(c32); 1032 tcg_temp_free_i32(n32); 1033 1034 tcg_gen_extu_i32_i64(retv, r32); 1035 tcg_temp_free_i32(r32); 1036 1037 if (memop & MO_SIGN) { 1038 tcg_gen_ext_i64(retv, retv, memop); 1039 } 1040 } 1041 } 1042 1043 void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 1044 TCGv_i64 cmpv, TCGv_i64 newv, 1045 TCGArg idx, MemOp memop, TCGType addr_type) 1046 { 1047 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1048 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 1049 tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 1050 } 1051 1052 static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1053 TCGv_i128 cmpv, TCGv_i128 newv, 1054 TCGArg idx, MemOp memop) 1055 { 1056 if (TCG_TARGET_REG_BITS == 32) { 1057 /* Inline expansion below is simply too large for 32-bit hosts. */ 1058 MemOpIdx oi = make_memop_idx(memop, idx); 1059 TCGv_i64 a64 = maybe_extend_addr64(addr); 1060 1061 gen_helper_nonatomic_cmpxchgo(retv, tcg_env, a64, cmpv, newv, 1062 tcg_constant_i32(oi)); 1063 maybe_free_addr64(a64); 1064 } else { 1065 TCGv_i128 oldv = tcg_temp_ebb_new_i128(); 1066 TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); 1067 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1068 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1069 TCGv_i64 z = tcg_constant_i64(0); 1070 1071 tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop); 1072 1073 /* Compare i128 */ 1074 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); 1075 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); 1076 tcg_gen_or_i64(t0, t0, t1); 1077 1078 /* tmpv = equal ? newv : oldv */ 1079 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, 1080 TCGV128_LOW(newv), TCGV128_LOW(oldv)); 1081 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, 1082 TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); 1083 1084 /* Unconditional writeback. */ 1085 tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop); 1086 tcg_gen_mov_i128(retv, oldv); 1087 1088 tcg_temp_free_i64(t0); 1089 tcg_temp_free_i64(t1); 1090 tcg_temp_free_i128(tmpv); 1091 tcg_temp_free_i128(oldv); 1092 } 1093 } 1094 1095 void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1096 TCGv_i128 cmpv, TCGv_i128 newv, 1097 TCGArg idx, MemOp memop, 1098 TCGType addr_type) 1099 { 1100 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1101 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1102 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1103 } 1104 1105 static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1106 TCGv_i128 cmpv, TCGv_i128 newv, 1107 TCGArg idx, MemOp memop) 1108 { 1109 gen_atomic_cx_i128 gen; 1110 1111 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 1112 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1113 return; 1114 } 1115 1116 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 1117 if (gen) { 1118 MemOpIdx oi = make_memop_idx(memop, idx); 1119 TCGv_i64 a64 = maybe_extend_addr64(addr); 1120 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 1121 maybe_free_addr64(a64); 1122 return; 1123 } 1124 1125 gen_helper_exit_atomic(tcg_env); 1126 1127 /* 1128 * Produce a result for a well-formed opcode stream. This satisfies 1129 * liveness for set before used, which happens before this dead code 1130 * is removed. 1131 */ 1132 tcg_gen_movi_i64(TCGV128_LOW(retv), 0); 1133 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); 1134 } 1135 1136 void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1137 TCGv_i128 cmpv, TCGv_i128 newv, 1138 TCGArg idx, MemOp memop, 1139 TCGType addr_type) 1140 { 1141 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1142 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1143 tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1144 } 1145 1146 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1147 TCGArg idx, MemOp memop, bool new_val, 1148 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 1149 { 1150 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1151 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1152 1153 memop = tcg_canonicalize_memop(memop, 0, 0); 1154 1155 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop); 1156 tcg_gen_ext_i32(t2, val, memop); 1157 gen(t2, t1, t2); 1158 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 1159 1160 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 1161 tcg_temp_free_i32(t1); 1162 tcg_temp_free_i32(t2); 1163 } 1164 1165 static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1166 TCGArg idx, MemOp memop, void * const table[]) 1167 { 1168 gen_atomic_op_i32 gen; 1169 TCGv_i64 a64; 1170 MemOpIdx oi; 1171 1172 memop = tcg_canonicalize_memop(memop, 0, 0); 1173 1174 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1175 tcg_debug_assert(gen != NULL); 1176 1177 oi = make_memop_idx(memop & ~MO_SIGN, idx); 1178 a64 = maybe_extend_addr64(addr); 1179 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1180 maybe_free_addr64(a64); 1181 1182 if (memop & MO_SIGN) { 1183 tcg_gen_ext_i32(ret, ret, memop); 1184 } 1185 } 1186 1187 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1188 TCGArg idx, MemOp memop, bool new_val, 1189 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 1190 { 1191 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1192 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 1193 1194 memop = tcg_canonicalize_memop(memop, 1, 0); 1195 1196 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop); 1197 tcg_gen_ext_i64(t2, val, memop); 1198 gen(t2, t1, t2); 1199 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 1200 1201 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 1202 tcg_temp_free_i64(t1); 1203 tcg_temp_free_i64(t2); 1204 } 1205 1206 static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1207 TCGArg idx, MemOp memop, void * const table[]) 1208 { 1209 memop = tcg_canonicalize_memop(memop, 1, 0); 1210 1211 if ((memop & MO_SIZE) == MO_64) { 1212 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1213 1214 if (gen) { 1215 MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); 1216 TCGv_i64 a64 = maybe_extend_addr64(addr); 1217 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1218 maybe_free_addr64(a64); 1219 return; 1220 } 1221 1222 gen_helper_exit_atomic(tcg_env); 1223 /* Produce a result, so that we have a well-formed opcode stream 1224 with respect to uses of the result in the (dead) code following. */ 1225 tcg_gen_movi_i64(ret, 0); 1226 } else { 1227 TCGv_i32 v32 = tcg_temp_ebb_new_i32(); 1228 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 1229 1230 tcg_gen_extrl_i64_i32(v32, val); 1231 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 1232 tcg_temp_free_i32(v32); 1233 1234 tcg_gen_extu_i32_i64(ret, r32); 1235 tcg_temp_free_i32(r32); 1236 1237 if (memop & MO_SIGN) { 1238 tcg_gen_ext_i64(ret, ret, memop); 1239 } 1240 } 1241 } 1242 1243 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 1244 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 1245 [MO_8] = gen_helper_atomic_##NAME##b, \ 1246 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 1247 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 1248 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 1249 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 1250 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 1251 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 1252 }; \ 1253 void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \ 1254 TCGv_i32 val, TCGArg idx, \ 1255 MemOp memop, TCGType addr_type) \ 1256 { \ 1257 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1258 tcg_debug_assert((memop & MO_SIZE) <= MO_32); \ 1259 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1260 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 1261 } else { \ 1262 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 1263 tcg_gen_##OP##_i32); \ 1264 } \ 1265 } \ 1266 void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \ 1267 TCGv_i64 val, TCGArg idx, \ 1268 MemOp memop, TCGType addr_type) \ 1269 { \ 1270 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1271 tcg_debug_assert((memop & MO_SIZE) <= MO_64); \ 1272 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1273 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 1274 } else { \ 1275 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 1276 tcg_gen_##OP##_i64); \ 1277 } \ 1278 } 1279 1280 GEN_ATOMIC_HELPER(fetch_add, add, 0) 1281 GEN_ATOMIC_HELPER(fetch_and, and, 0) 1282 GEN_ATOMIC_HELPER(fetch_or, or, 0) 1283 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 1284 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 1285 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 1286 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 1287 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 1288 1289 GEN_ATOMIC_HELPER(add_fetch, add, 1) 1290 GEN_ATOMIC_HELPER(and_fetch, and, 1) 1291 GEN_ATOMIC_HELPER(or_fetch, or, 1) 1292 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 1293 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 1294 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 1295 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 1296 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 1297 1298 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 1299 { 1300 tcg_gen_mov_i32(r, b); 1301 } 1302 1303 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 1304 { 1305 tcg_gen_mov_i64(r, b); 1306 } 1307 1308 GEN_ATOMIC_HELPER(xchg, mov2, 0) 1309 1310 #undef GEN_ATOMIC_HELPER 1311