1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/target_page.h" 31 #include "exec/translation-block.h" 32 #include "exec/plugin-gen.h" 33 #include "tcg-internal.h" 34 #include "tcg-has.h" 35 #include "tcg-target-mo.h" 36 37 static void check_max_alignment(unsigned a_bits) 38 { 39 /* 40 * The requested alignment cannot overlap the TLB flags. 41 * FIXME: Must keep the count up-to-date with "exec/tlb-flags.h". 42 */ 43 if (tcg_use_softmmu) { 44 tcg_debug_assert(a_bits + 5 <= TARGET_PAGE_BITS); 45 } 46 } 47 48 static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 49 { 50 unsigned a_bits = memop_alignment_bits(op); 51 52 check_max_alignment(a_bits); 53 54 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 55 if (a_bits == (op & MO_SIZE)) { 56 op = (op & ~MO_AMASK) | MO_ALIGN; 57 } 58 59 switch (op & MO_SIZE) { 60 case MO_8: 61 op &= ~MO_BSWAP; 62 break; 63 case MO_16: 64 break; 65 case MO_32: 66 if (!is64) { 67 op &= ~MO_SIGN; 68 } 69 break; 70 case MO_64: 71 if (is64) { 72 op &= ~MO_SIGN; 73 break; 74 } 75 /* fall through */ 76 default: 77 g_assert_not_reached(); 78 } 79 if (st) { 80 op &= ~MO_SIGN; 81 } 82 83 /* In serial mode, reduce atomicity. */ 84 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 85 op &= ~MO_ATOM_MASK; 86 op |= MO_ATOM_NONE; 87 } 88 89 return op; 90 } 91 92 static void gen_ldst1(TCGOpcode opc, TCGType type, TCGTemp *v, 93 TCGTemp *addr, MemOpIdx oi) 94 { 95 TCGOp *op = tcg_gen_op3(opc, type, temp_arg(v), temp_arg(addr), oi); 96 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; 97 } 98 99 static void gen_ldst2(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh, 100 TCGTemp *addr, MemOpIdx oi) 101 { 102 TCGOp *op = tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh), 103 temp_arg(addr), oi); 104 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; 105 } 106 107 static void gen_ld_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) 108 { 109 if (TCG_TARGET_REG_BITS == 32) { 110 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I64, 111 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), 112 addr, oi); 113 } else { 114 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); 115 } 116 } 117 118 static void gen_st_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) 119 { 120 if (TCG_TARGET_REG_BITS == 32) { 121 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I64, 122 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), 123 addr, oi); 124 } else { 125 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); 126 } 127 } 128 129 static void tcg_gen_req_mo(TCGBar type) 130 { 131 type &= tcg_ctx->guest_mo; 132 type &= ~TCG_TARGET_DEFAULT_MO; 133 if (type) { 134 tcg_gen_mb(type | TCG_BAR_SC); 135 } 136 } 137 138 /* Only required for loads, where value might overlap addr. */ 139 static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr) 140 { 141 #ifdef CONFIG_PLUGIN 142 if (tcg_ctx->plugin_insn != NULL) { 143 /* Save a copy of the vaddr for use after a load. */ 144 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 145 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 146 tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr)); 147 } else { 148 tcg_gen_mov_i64(temp, temp_tcgv_i64(addr)); 149 } 150 return temp; 151 } 152 #endif 153 return NULL; 154 } 155 156 #ifdef CONFIG_PLUGIN 157 static void 158 plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi, 159 enum qemu_plugin_mem_rw rw) 160 { 161 if (tcg_ctx->plugin_insn != NULL) { 162 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 163 164 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 165 if (!copy_addr) { 166 copy_addr = tcg_temp_ebb_new_i64(); 167 tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr)); 168 } 169 tcg_gen_plugin_mem_cb(copy_addr, info); 170 tcg_temp_free_i64(copy_addr); 171 } else { 172 if (copy_addr) { 173 tcg_gen_plugin_mem_cb(copy_addr, info); 174 tcg_temp_free_i64(copy_addr); 175 } else { 176 tcg_gen_plugin_mem_cb(temp_tcgv_i64(orig_addr), info); 177 } 178 } 179 } 180 } 181 #endif 182 183 static void 184 plugin_gen_mem_callbacks_i32(TCGv_i32 val, 185 TCGv_i64 copy_addr, TCGTemp *orig_addr, 186 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 187 { 188 #ifdef CONFIG_PLUGIN 189 if (tcg_ctx->plugin_insn != NULL) { 190 tcg_gen_st_i32(val, tcg_env, 191 offsetof(CPUState, neg.plugin_mem_value_low) - 192 sizeof(CPUState) + (HOST_BIG_ENDIAN * 4)); 193 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 194 } 195 #endif 196 } 197 198 static void 199 plugin_gen_mem_callbacks_i64(TCGv_i64 val, 200 TCGv_i64 copy_addr, TCGTemp *orig_addr, 201 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 202 { 203 #ifdef CONFIG_PLUGIN 204 if (tcg_ctx->plugin_insn != NULL) { 205 tcg_gen_st_i64(val, tcg_env, 206 offsetof(CPUState, neg.plugin_mem_value_low) - 207 sizeof(CPUState)); 208 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 209 } 210 #endif 211 } 212 213 static void 214 plugin_gen_mem_callbacks_i128(TCGv_i128 val, 215 TCGv_i64 copy_addr, TCGTemp *orig_addr, 216 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 217 { 218 #ifdef CONFIG_PLUGIN 219 if (tcg_ctx->plugin_insn != NULL) { 220 tcg_gen_st_i64(TCGV128_LOW(val), tcg_env, 221 offsetof(CPUState, neg.plugin_mem_value_low) - 222 sizeof(CPUState)); 223 tcg_gen_st_i64(TCGV128_HIGH(val), tcg_env, 224 offsetof(CPUState, neg.plugin_mem_value_high) - 225 sizeof(CPUState)); 226 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 227 } 228 #endif 229 } 230 231 static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, 232 TCGArg idx, MemOp memop) 233 { 234 MemOp orig_memop; 235 MemOpIdx orig_oi, oi; 236 TCGv_i64 copy_addr; 237 238 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 239 orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0); 240 orig_oi = oi = make_memop_idx(memop, idx); 241 242 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 243 memop &= ~MO_BSWAP; 244 /* The bswap primitive benefits from zero-extended input. */ 245 if ((memop & MO_SSIZE) == MO_SW) { 246 memop &= ~MO_SIGN; 247 } 248 oi = make_memop_idx(memop, idx); 249 } 250 251 copy_addr = plugin_maybe_preserve_addr(addr); 252 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); 253 plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi, 254 QEMU_PLUGIN_MEM_R); 255 256 if ((orig_memop ^ memop) & MO_BSWAP) { 257 switch (orig_memop & MO_SIZE) { 258 case MO_16: 259 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 260 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 261 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 262 break; 263 case MO_32: 264 tcg_gen_bswap32_i32(val, val); 265 break; 266 default: 267 g_assert_not_reached(); 268 } 269 } 270 } 271 272 void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 273 MemOp memop, TCGType addr_type) 274 { 275 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 276 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 277 tcg_gen_qemu_ld_i32_int(val, addr, idx, memop); 278 } 279 280 static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, 281 TCGArg idx, MemOp memop) 282 { 283 TCGv_i32 swap = NULL; 284 MemOpIdx orig_oi, oi; 285 286 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 287 memop = tcg_canonicalize_memop(memop, 0, 1); 288 orig_oi = oi = make_memop_idx(memop, idx); 289 290 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 291 swap = tcg_temp_ebb_new_i32(); 292 switch (memop & MO_SIZE) { 293 case MO_16: 294 tcg_gen_bswap16_i32(swap, val, 0); 295 break; 296 case MO_32: 297 tcg_gen_bswap32_i32(swap, val); 298 break; 299 default: 300 g_assert_not_reached(); 301 } 302 val = swap; 303 memop &= ~MO_BSWAP; 304 oi = make_memop_idx(memop, idx); 305 } 306 307 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); 308 plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 309 310 if (swap) { 311 tcg_temp_free_i32(swap); 312 } 313 } 314 315 void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 316 MemOp memop, TCGType addr_type) 317 { 318 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 319 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 320 tcg_gen_qemu_st_i32_int(val, addr, idx, memop); 321 } 322 323 static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, 324 TCGArg idx, MemOp memop) 325 { 326 MemOp orig_memop; 327 MemOpIdx orig_oi, oi; 328 TCGv_i64 copy_addr; 329 330 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 331 tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop); 332 if (memop & MO_SIGN) { 333 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 334 } else { 335 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 336 } 337 return; 338 } 339 340 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 341 orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0); 342 orig_oi = oi = make_memop_idx(memop, idx); 343 344 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 345 memop &= ~MO_BSWAP; 346 /* The bswap primitive benefits from zero-extended input. */ 347 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 348 memop &= ~MO_SIGN; 349 } 350 oi = make_memop_idx(memop, idx); 351 } 352 353 copy_addr = plugin_maybe_preserve_addr(addr); 354 gen_ld_i64(val, addr, oi); 355 plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi, 356 QEMU_PLUGIN_MEM_R); 357 358 if ((orig_memop ^ memop) & MO_BSWAP) { 359 int flags = (orig_memop & MO_SIGN 360 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 361 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 362 switch (orig_memop & MO_SIZE) { 363 case MO_16: 364 tcg_gen_bswap16_i64(val, val, flags); 365 break; 366 case MO_32: 367 tcg_gen_bswap32_i64(val, val, flags); 368 break; 369 case MO_64: 370 tcg_gen_bswap64_i64(val, val); 371 break; 372 default: 373 g_assert_not_reached(); 374 } 375 } 376 } 377 378 void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 379 MemOp memop, TCGType addr_type) 380 { 381 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 382 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 383 tcg_gen_qemu_ld_i64_int(val, addr, idx, memop); 384 } 385 386 static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, 387 TCGArg idx, MemOp memop) 388 { 389 TCGv_i64 swap = NULL; 390 MemOpIdx orig_oi, oi; 391 392 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 393 tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop); 394 return; 395 } 396 397 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 398 memop = tcg_canonicalize_memop(memop, 1, 1); 399 orig_oi = oi = make_memop_idx(memop, idx); 400 401 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 402 swap = tcg_temp_ebb_new_i64(); 403 switch (memop & MO_SIZE) { 404 case MO_16: 405 tcg_gen_bswap16_i64(swap, val, 0); 406 break; 407 case MO_32: 408 tcg_gen_bswap32_i64(swap, val, 0); 409 break; 410 case MO_64: 411 tcg_gen_bswap64_i64(swap, val); 412 break; 413 default: 414 g_assert_not_reached(); 415 } 416 val = swap; 417 memop &= ~MO_BSWAP; 418 oi = make_memop_idx(memop, idx); 419 } 420 421 gen_st_i64(val, addr, oi); 422 plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 423 424 if (swap) { 425 tcg_temp_free_i64(swap); 426 } 427 } 428 429 void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 430 MemOp memop, TCGType addr_type) 431 { 432 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 433 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 434 tcg_gen_qemu_st_i64_int(val, addr, idx, memop); 435 } 436 437 /* 438 * Return true if @mop, without knowledge of the pointer alignment, 439 * does not require 16-byte atomicity, and it would be adventagous 440 * to avoid a call to a helper function. 441 */ 442 static bool use_two_i64_for_i128(MemOp mop) 443 { 444 /* Two softmmu tlb lookups is larger than one function call. */ 445 if (tcg_use_softmmu) { 446 return false; 447 } 448 449 /* 450 * For user-only, two 64-bit operations may well be smaller than a call. 451 * Determine if that would be legal for the requested atomicity. 452 */ 453 switch (mop & MO_ATOM_MASK) { 454 case MO_ATOM_NONE: 455 case MO_ATOM_IFALIGN_PAIR: 456 return true; 457 case MO_ATOM_IFALIGN: 458 case MO_ATOM_SUBALIGN: 459 case MO_ATOM_WITHIN16: 460 case MO_ATOM_WITHIN16_PAIR: 461 return false; 462 default: 463 g_assert_not_reached(); 464 } 465 } 466 467 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) 468 { 469 MemOp mop_1 = orig, mop_2; 470 471 /* Reduce the size to 64-bit. */ 472 mop_1 = (mop_1 & ~MO_SIZE) | MO_64; 473 474 /* Retain the alignment constraints of the original. */ 475 switch (orig & MO_AMASK) { 476 case MO_UNALN: 477 case MO_ALIGN_2: 478 case MO_ALIGN_4: 479 mop_2 = mop_1; 480 break; 481 case MO_ALIGN_8: 482 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ 483 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 484 mop_2 = mop_1; 485 break; 486 case MO_ALIGN: 487 /* Second has 8-byte alignment; first has 16-byte alignment. */ 488 mop_2 = mop_1; 489 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; 490 break; 491 case MO_ALIGN_16: 492 case MO_ALIGN_32: 493 case MO_ALIGN_64: 494 /* Second has 8-byte alignment; first retains original. */ 495 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 496 break; 497 default: 498 g_assert_not_reached(); 499 } 500 501 /* Use a memory ordering implemented by the host. */ 502 if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) { 503 mop_1 &= ~MO_BSWAP; 504 mop_2 &= ~MO_BSWAP; 505 } 506 507 ret[0] = mop_1; 508 ret[1] = mop_2; 509 } 510 511 static TCGv_i64 maybe_extend_addr64(TCGTemp *addr) 512 { 513 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 514 TCGv_i64 a64 = tcg_temp_ebb_new_i64(); 515 tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr)); 516 return a64; 517 } 518 return temp_tcgv_i64(addr); 519 } 520 521 static void maybe_free_addr64(TCGv_i64 a64) 522 { 523 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 524 tcg_temp_free_i64(a64); 525 } 526 } 527 528 static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, 529 TCGArg idx, MemOp memop) 530 { 531 MemOpIdx orig_oi; 532 TCGv_i64 ext_addr = NULL; 533 534 check_max_alignment(memop_alignment_bits(memop)); 535 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 536 537 /* In serial mode, reduce atomicity. */ 538 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 539 memop &= ~MO_ATOM_MASK; 540 memop |= MO_ATOM_NONE; 541 } 542 orig_oi = make_memop_idx(memop, idx); 543 544 /* TODO: For now, force 32-bit hosts to use the helper. */ 545 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 546 TCGv_i64 lo, hi; 547 bool need_bswap = false; 548 MemOpIdx oi = orig_oi; 549 550 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 551 lo = TCGV128_HIGH(val); 552 hi = TCGV128_LOW(val); 553 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 554 need_bswap = true; 555 } else { 556 lo = TCGV128_LOW(val); 557 hi = TCGV128_HIGH(val); 558 } 559 560 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I128, tcgv_i64_temp(lo), 561 tcgv_i64_temp(hi), addr, oi); 562 563 if (need_bswap) { 564 tcg_gen_bswap64_i64(lo, lo); 565 tcg_gen_bswap64_i64(hi, hi); 566 } 567 } else if (use_two_i64_for_i128(memop)) { 568 MemOp mop[2]; 569 TCGTemp *addr_p8; 570 TCGv_i64 x, y; 571 bool need_bswap; 572 573 canonicalize_memop_i128_as_i64(mop, memop); 574 need_bswap = (mop[0] ^ memop) & MO_BSWAP; 575 576 /* 577 * Since there are no global TCGv_i128, there is no visible state 578 * changed if the second load faults. Load directly into the two 579 * subwords. 580 */ 581 if ((memop & MO_BSWAP) == MO_LE) { 582 x = TCGV128_LOW(val); 583 y = TCGV128_HIGH(val); 584 } else { 585 x = TCGV128_HIGH(val); 586 y = TCGV128_LOW(val); 587 } 588 589 gen_ld_i64(x, addr, make_memop_idx(mop[0], idx)); 590 591 if (need_bswap) { 592 tcg_gen_bswap64_i64(x, x); 593 } 594 595 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 596 TCGv_i32 t = tcg_temp_ebb_new_i32(); 597 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 598 addr_p8 = tcgv_i32_temp(t); 599 } else { 600 TCGv_i64 t = tcg_temp_ebb_new_i64(); 601 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 602 addr_p8 = tcgv_i64_temp(t); 603 } 604 605 gen_ld_i64(y, addr_p8, make_memop_idx(mop[1], idx)); 606 tcg_temp_free_internal(addr_p8); 607 608 if (need_bswap) { 609 tcg_gen_bswap64_i64(y, y); 610 } 611 } else { 612 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 613 ext_addr = tcg_temp_ebb_new_i64(); 614 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 615 addr = tcgv_i64_temp(ext_addr); 616 } 617 gen_helper_ld_i128(val, tcg_env, temp_tcgv_i64(addr), 618 tcg_constant_i32(orig_oi)); 619 } 620 621 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 622 QEMU_PLUGIN_MEM_R); 623 } 624 625 void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 626 MemOp memop, TCGType addr_type) 627 { 628 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 629 tcg_debug_assert((memop & MO_SIZE) == MO_128); 630 tcg_debug_assert((memop & MO_SIGN) == 0); 631 tcg_gen_qemu_ld_i128_int(val, addr, idx, memop); 632 } 633 634 static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, 635 TCGArg idx, MemOp memop) 636 { 637 MemOpIdx orig_oi; 638 TCGv_i64 ext_addr = NULL; 639 640 check_max_alignment(memop_alignment_bits(memop)); 641 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); 642 643 /* In serial mode, reduce atomicity. */ 644 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 645 memop &= ~MO_ATOM_MASK; 646 memop |= MO_ATOM_NONE; 647 } 648 orig_oi = make_memop_idx(memop, idx); 649 650 /* TODO: For now, force 32-bit hosts to use the helper. */ 651 652 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 653 TCGv_i64 lo, hi; 654 MemOpIdx oi = orig_oi; 655 bool need_bswap = false; 656 657 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 658 lo = tcg_temp_ebb_new_i64(); 659 hi = tcg_temp_ebb_new_i64(); 660 tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val)); 661 tcg_gen_bswap64_i64(hi, TCGV128_LOW(val)); 662 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 663 need_bswap = true; 664 } else { 665 lo = TCGV128_LOW(val); 666 hi = TCGV128_HIGH(val); 667 } 668 669 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I128, 670 tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); 671 672 if (need_bswap) { 673 tcg_temp_free_i64(lo); 674 tcg_temp_free_i64(hi); 675 } 676 } else if (use_two_i64_for_i128(memop)) { 677 MemOp mop[2]; 678 TCGTemp *addr_p8; 679 TCGv_i64 x, y, b = NULL; 680 681 canonicalize_memop_i128_as_i64(mop, memop); 682 683 if ((memop & MO_BSWAP) == MO_LE) { 684 x = TCGV128_LOW(val); 685 y = TCGV128_HIGH(val); 686 } else { 687 x = TCGV128_HIGH(val); 688 y = TCGV128_LOW(val); 689 } 690 691 if ((mop[0] ^ memop) & MO_BSWAP) { 692 b = tcg_temp_ebb_new_i64(); 693 tcg_gen_bswap64_i64(b, x); 694 x = b; 695 } 696 697 gen_st_i64(x, addr, make_memop_idx(mop[0], idx)); 698 699 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 700 TCGv_i32 t = tcg_temp_ebb_new_i32(); 701 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 702 addr_p8 = tcgv_i32_temp(t); 703 } else { 704 TCGv_i64 t = tcg_temp_ebb_new_i64(); 705 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 706 addr_p8 = tcgv_i64_temp(t); 707 } 708 709 if (b) { 710 tcg_gen_bswap64_i64(b, y); 711 gen_st_i64(b, addr_p8, make_memop_idx(mop[1], idx)); 712 tcg_temp_free_i64(b); 713 } else { 714 gen_st_i64(y, addr_p8, make_memop_idx(mop[1], idx)); 715 } 716 tcg_temp_free_internal(addr_p8); 717 } else { 718 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 719 ext_addr = tcg_temp_ebb_new_i64(); 720 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 721 addr = tcgv_i64_temp(ext_addr); 722 } 723 gen_helper_st_i128(tcg_env, temp_tcgv_i64(addr), val, 724 tcg_constant_i32(orig_oi)); 725 } 726 727 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 728 QEMU_PLUGIN_MEM_W); 729 } 730 731 void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 732 MemOp memop, TCGType addr_type) 733 { 734 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 735 tcg_debug_assert((memop & MO_SIZE) == MO_128); 736 tcg_debug_assert((memop & MO_SIGN) == 0); 737 tcg_gen_qemu_st_i128_int(val, addr, idx, memop); 738 } 739 740 void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 741 { 742 switch (opc & MO_SSIZE) { 743 case MO_SB: 744 tcg_gen_ext8s_i32(ret, val); 745 break; 746 case MO_UB: 747 tcg_gen_ext8u_i32(ret, val); 748 break; 749 case MO_SW: 750 tcg_gen_ext16s_i32(ret, val); 751 break; 752 case MO_UW: 753 tcg_gen_ext16u_i32(ret, val); 754 break; 755 case MO_UL: 756 case MO_SL: 757 tcg_gen_mov_i32(ret, val); 758 break; 759 default: 760 g_assert_not_reached(); 761 } 762 } 763 764 void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 765 { 766 switch (opc & MO_SSIZE) { 767 case MO_SB: 768 tcg_gen_ext8s_i64(ret, val); 769 break; 770 case MO_UB: 771 tcg_gen_ext8u_i64(ret, val); 772 break; 773 case MO_SW: 774 tcg_gen_ext16s_i64(ret, val); 775 break; 776 case MO_UW: 777 tcg_gen_ext16u_i64(ret, val); 778 break; 779 case MO_SL: 780 tcg_gen_ext32s_i64(ret, val); 781 break; 782 case MO_UL: 783 tcg_gen_ext32u_i64(ret, val); 784 break; 785 case MO_UQ: 786 case MO_SQ: 787 tcg_gen_mov_i64(ret, val); 788 break; 789 default: 790 g_assert_not_reached(); 791 } 792 } 793 794 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64, 795 TCGv_i32, TCGv_i32, TCGv_i32); 796 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64, 797 TCGv_i64, TCGv_i64, TCGv_i32); 798 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64, 799 TCGv_i128, TCGv_i128, TCGv_i32); 800 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64, 801 TCGv_i32, TCGv_i32); 802 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64, 803 TCGv_i64, TCGv_i32); 804 805 #ifdef CONFIG_ATOMIC64 806 # define WITH_ATOMIC64(X) X, 807 #else 808 # define WITH_ATOMIC64(X) 809 #endif 810 #if HAVE_CMPXCHG128 811 # define WITH_ATOMIC128(X) X, 812 #else 813 # define WITH_ATOMIC128(X) 814 #endif 815 816 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 817 [MO_8] = gen_helper_atomic_cmpxchgb, 818 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 819 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 820 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 821 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 822 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 823 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 824 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) 825 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) 826 }; 827 828 static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 829 TCGv_i32 cmpv, TCGv_i32 newv, 830 TCGArg idx, MemOp memop) 831 { 832 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 833 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 834 835 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 836 837 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN); 838 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 839 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 840 tcg_temp_free_i32(t2); 841 842 if (memop & MO_SIGN) { 843 tcg_gen_ext_i32(retv, t1, memop); 844 } else { 845 tcg_gen_mov_i32(retv, t1); 846 } 847 tcg_temp_free_i32(t1); 848 } 849 850 void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 851 TCGv_i32 cmpv, TCGv_i32 newv, 852 TCGArg idx, MemOp memop, 853 TCGType addr_type) 854 { 855 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 856 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 857 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 858 } 859 860 static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 861 TCGv_i32 cmpv, TCGv_i32 newv, 862 TCGArg idx, MemOp memop) 863 { 864 gen_atomic_cx_i32 gen; 865 TCGv_i64 a64; 866 MemOpIdx oi; 867 868 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 869 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 870 return; 871 } 872 873 memop = tcg_canonicalize_memop(memop, 0, 0); 874 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 875 tcg_debug_assert(gen != NULL); 876 877 oi = make_memop_idx(memop & ~MO_SIGN, idx); 878 a64 = maybe_extend_addr64(addr); 879 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 880 maybe_free_addr64(a64); 881 882 if (memop & MO_SIGN) { 883 tcg_gen_ext_i32(retv, retv, memop); 884 } 885 } 886 887 void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 888 TCGv_i32 cmpv, TCGv_i32 newv, 889 TCGArg idx, MemOp memop, 890 TCGType addr_type) 891 { 892 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 893 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 894 tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 895 } 896 897 static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 898 TCGv_i64 cmpv, TCGv_i64 newv, 899 TCGArg idx, MemOp memop) 900 { 901 TCGv_i64 t1, t2; 902 903 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 904 tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 905 TCGV_LOW(newv), idx, memop); 906 if (memop & MO_SIGN) { 907 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 908 } else { 909 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 910 } 911 return; 912 } 913 914 t1 = tcg_temp_ebb_new_i64(); 915 t2 = tcg_temp_ebb_new_i64(); 916 917 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 918 919 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN); 920 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 921 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 922 tcg_temp_free_i64(t2); 923 924 if (memop & MO_SIGN) { 925 tcg_gen_ext_i64(retv, t1, memop); 926 } else { 927 tcg_gen_mov_i64(retv, t1); 928 } 929 tcg_temp_free_i64(t1); 930 } 931 932 void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 933 TCGv_i64 cmpv, TCGv_i64 newv, 934 TCGArg idx, MemOp memop, 935 TCGType addr_type) 936 { 937 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 938 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 939 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 940 } 941 942 static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 943 TCGv_i64 cmpv, TCGv_i64 newv, 944 TCGArg idx, MemOp memop) 945 { 946 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 947 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 948 return; 949 } 950 951 if ((memop & MO_SIZE) == MO_64) { 952 gen_atomic_cx_i64 gen; 953 954 memop = tcg_canonicalize_memop(memop, 1, 0); 955 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 956 if (gen) { 957 MemOpIdx oi = make_memop_idx(memop, idx); 958 TCGv_i64 a64 = maybe_extend_addr64(addr); 959 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 960 maybe_free_addr64(a64); 961 return; 962 } 963 964 gen_helper_exit_atomic(tcg_env); 965 966 /* 967 * Produce a result for a well-formed opcode stream. This satisfies 968 * liveness for set before used, which happens before this dead code 969 * is removed. 970 */ 971 tcg_gen_movi_i64(retv, 0); 972 return; 973 } 974 975 if (TCG_TARGET_REG_BITS == 32) { 976 tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 977 TCGV_LOW(newv), idx, memop); 978 if (memop & MO_SIGN) { 979 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 980 } else { 981 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 982 } 983 } else { 984 TCGv_i32 c32 = tcg_temp_ebb_new_i32(); 985 TCGv_i32 n32 = tcg_temp_ebb_new_i32(); 986 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 987 988 tcg_gen_extrl_i64_i32(c32, cmpv); 989 tcg_gen_extrl_i64_i32(n32, newv); 990 tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32, 991 idx, memop & ~MO_SIGN); 992 tcg_temp_free_i32(c32); 993 tcg_temp_free_i32(n32); 994 995 tcg_gen_extu_i32_i64(retv, r32); 996 tcg_temp_free_i32(r32); 997 998 if (memop & MO_SIGN) { 999 tcg_gen_ext_i64(retv, retv, memop); 1000 } 1001 } 1002 } 1003 1004 void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 1005 TCGv_i64 cmpv, TCGv_i64 newv, 1006 TCGArg idx, MemOp memop, TCGType addr_type) 1007 { 1008 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1009 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 1010 tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 1011 } 1012 1013 static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1014 TCGv_i128 cmpv, TCGv_i128 newv, 1015 TCGArg idx, MemOp memop) 1016 { 1017 if (TCG_TARGET_REG_BITS == 32) { 1018 /* Inline expansion below is simply too large for 32-bit hosts. */ 1019 MemOpIdx oi = make_memop_idx(memop, idx); 1020 TCGv_i64 a64 = maybe_extend_addr64(addr); 1021 1022 gen_helper_nonatomic_cmpxchgo(retv, tcg_env, a64, cmpv, newv, 1023 tcg_constant_i32(oi)); 1024 maybe_free_addr64(a64); 1025 } else { 1026 TCGv_i128 oldv = tcg_temp_ebb_new_i128(); 1027 TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); 1028 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1029 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1030 TCGv_i64 z = tcg_constant_i64(0); 1031 1032 tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop); 1033 1034 /* Compare i128 */ 1035 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); 1036 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); 1037 tcg_gen_or_i64(t0, t0, t1); 1038 1039 /* tmpv = equal ? newv : oldv */ 1040 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, 1041 TCGV128_LOW(newv), TCGV128_LOW(oldv)); 1042 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, 1043 TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); 1044 1045 /* Unconditional writeback. */ 1046 tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop); 1047 tcg_gen_mov_i128(retv, oldv); 1048 1049 tcg_temp_free_i64(t0); 1050 tcg_temp_free_i64(t1); 1051 tcg_temp_free_i128(tmpv); 1052 tcg_temp_free_i128(oldv); 1053 } 1054 } 1055 1056 void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1057 TCGv_i128 cmpv, TCGv_i128 newv, 1058 TCGArg idx, MemOp memop, 1059 TCGType addr_type) 1060 { 1061 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1062 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1063 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1064 } 1065 1066 static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1067 TCGv_i128 cmpv, TCGv_i128 newv, 1068 TCGArg idx, MemOp memop) 1069 { 1070 gen_atomic_cx_i128 gen; 1071 1072 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 1073 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1074 return; 1075 } 1076 1077 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 1078 if (gen) { 1079 MemOpIdx oi = make_memop_idx(memop, idx); 1080 TCGv_i64 a64 = maybe_extend_addr64(addr); 1081 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 1082 maybe_free_addr64(a64); 1083 return; 1084 } 1085 1086 gen_helper_exit_atomic(tcg_env); 1087 1088 /* 1089 * Produce a result for a well-formed opcode stream. This satisfies 1090 * liveness for set before used, which happens before this dead code 1091 * is removed. 1092 */ 1093 tcg_gen_movi_i64(TCGV128_LOW(retv), 0); 1094 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); 1095 } 1096 1097 void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1098 TCGv_i128 cmpv, TCGv_i128 newv, 1099 TCGArg idx, MemOp memop, 1100 TCGType addr_type) 1101 { 1102 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1103 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1104 tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1105 } 1106 1107 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1108 TCGArg idx, MemOp memop, bool new_val, 1109 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 1110 { 1111 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1112 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1113 1114 memop = tcg_canonicalize_memop(memop, 0, 0); 1115 1116 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop); 1117 tcg_gen_ext_i32(t2, val, memop); 1118 gen(t2, t1, t2); 1119 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 1120 1121 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 1122 tcg_temp_free_i32(t1); 1123 tcg_temp_free_i32(t2); 1124 } 1125 1126 static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1127 TCGArg idx, MemOp memop, void * const table[]) 1128 { 1129 gen_atomic_op_i32 gen; 1130 TCGv_i64 a64; 1131 MemOpIdx oi; 1132 1133 memop = tcg_canonicalize_memop(memop, 0, 0); 1134 1135 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1136 tcg_debug_assert(gen != NULL); 1137 1138 oi = make_memop_idx(memop & ~MO_SIGN, idx); 1139 a64 = maybe_extend_addr64(addr); 1140 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1141 maybe_free_addr64(a64); 1142 1143 if (memop & MO_SIGN) { 1144 tcg_gen_ext_i32(ret, ret, memop); 1145 } 1146 } 1147 1148 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1149 TCGArg idx, MemOp memop, bool new_val, 1150 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 1151 { 1152 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1153 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 1154 1155 memop = tcg_canonicalize_memop(memop, 1, 0); 1156 1157 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop); 1158 tcg_gen_ext_i64(t2, val, memop); 1159 gen(t2, t1, t2); 1160 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 1161 1162 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 1163 tcg_temp_free_i64(t1); 1164 tcg_temp_free_i64(t2); 1165 } 1166 1167 static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1168 TCGArg idx, MemOp memop, void * const table[]) 1169 { 1170 memop = tcg_canonicalize_memop(memop, 1, 0); 1171 1172 if ((memop & MO_SIZE) == MO_64) { 1173 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1174 1175 if (gen) { 1176 MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); 1177 TCGv_i64 a64 = maybe_extend_addr64(addr); 1178 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1179 maybe_free_addr64(a64); 1180 return; 1181 } 1182 1183 gen_helper_exit_atomic(tcg_env); 1184 /* Produce a result, so that we have a well-formed opcode stream 1185 with respect to uses of the result in the (dead) code following. */ 1186 tcg_gen_movi_i64(ret, 0); 1187 } else { 1188 TCGv_i32 v32 = tcg_temp_ebb_new_i32(); 1189 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 1190 1191 tcg_gen_extrl_i64_i32(v32, val); 1192 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 1193 tcg_temp_free_i32(v32); 1194 1195 tcg_gen_extu_i32_i64(ret, r32); 1196 tcg_temp_free_i32(r32); 1197 1198 if (memop & MO_SIGN) { 1199 tcg_gen_ext_i64(ret, ret, memop); 1200 } 1201 } 1202 } 1203 1204 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 1205 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 1206 [MO_8] = gen_helper_atomic_##NAME##b, \ 1207 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 1208 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 1209 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 1210 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 1211 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 1212 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 1213 }; \ 1214 void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \ 1215 TCGv_i32 val, TCGArg idx, \ 1216 MemOp memop, TCGType addr_type) \ 1217 { \ 1218 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1219 tcg_debug_assert((memop & MO_SIZE) <= MO_32); \ 1220 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1221 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 1222 } else { \ 1223 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 1224 tcg_gen_##OP##_i32); \ 1225 } \ 1226 } \ 1227 void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \ 1228 TCGv_i64 val, TCGArg idx, \ 1229 MemOp memop, TCGType addr_type) \ 1230 { \ 1231 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1232 tcg_debug_assert((memop & MO_SIZE) <= MO_64); \ 1233 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1234 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 1235 } else { \ 1236 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 1237 tcg_gen_##OP##_i64); \ 1238 } \ 1239 } 1240 1241 GEN_ATOMIC_HELPER(fetch_add, add, 0) 1242 GEN_ATOMIC_HELPER(fetch_and, and, 0) 1243 GEN_ATOMIC_HELPER(fetch_or, or, 0) 1244 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 1245 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 1246 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 1247 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 1248 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 1249 1250 GEN_ATOMIC_HELPER(add_fetch, add, 1) 1251 GEN_ATOMIC_HELPER(and_fetch, and, 1) 1252 GEN_ATOMIC_HELPER(or_fetch, or, 1) 1253 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 1254 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 1255 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 1256 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 1257 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 1258 1259 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 1260 { 1261 tcg_gen_mov_i32(r, b); 1262 } 1263 1264 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 1265 { 1266 tcg_gen_mov_i64(r, b); 1267 } 1268 1269 GEN_ATOMIC_HELPER(xchg, mov2, 0) 1270 1271 #undef GEN_ATOMIC_HELPER 1272