1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "tcg/tcg.h" 27 #include "tcg/tcg-temp-internal.h" 28 #include "tcg/tcg-op-common.h" 29 #include "tcg/tcg-mo.h" 30 #include "exec/translation-block.h" 31 #include "exec/plugin-gen.h" 32 #include "tcg-internal.h" 33 #include "tcg-has.h" 34 #include "tcg-target-mo.h" 35 36 static void check_max_alignment(unsigned a_bits) 37 { 38 /* 39 * The requested alignment cannot overlap the TLB flags. 40 * FIXME: Must keep the count up-to-date with "exec/tlb-flags.h". 41 */ 42 if (tcg_use_softmmu) { 43 tcg_debug_assert(a_bits + 5 <= tcg_ctx->page_bits); 44 } 45 } 46 47 static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st) 48 { 49 unsigned a_bits = memop_alignment_bits(op); 50 51 check_max_alignment(a_bits); 52 53 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */ 54 if (a_bits == (op & MO_SIZE)) { 55 op = (op & ~MO_AMASK) | MO_ALIGN; 56 } 57 58 switch (op & MO_SIZE) { 59 case MO_8: 60 op &= ~MO_BSWAP; 61 break; 62 case MO_16: 63 break; 64 case MO_32: 65 if (!is64) { 66 op &= ~MO_SIGN; 67 } 68 break; 69 case MO_64: 70 if (is64) { 71 op &= ~MO_SIGN; 72 break; 73 } 74 /* fall through */ 75 default: 76 g_assert_not_reached(); 77 } 78 if (st) { 79 op &= ~MO_SIGN; 80 } 81 82 /* In serial mode, reduce atomicity. */ 83 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 84 op &= ~MO_ATOM_MASK; 85 op |= MO_ATOM_NONE; 86 } 87 88 return op; 89 } 90 91 static void gen_ldst1(TCGOpcode opc, TCGType type, TCGTemp *v, 92 TCGTemp *addr, MemOpIdx oi) 93 { 94 TCGOp *op = tcg_gen_op3(opc, type, temp_arg(v), temp_arg(addr), oi); 95 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; 96 } 97 98 static void gen_ldst2(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh, 99 TCGTemp *addr, MemOpIdx oi) 100 { 101 TCGOp *op = tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh), 102 temp_arg(addr), oi); 103 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE; 104 } 105 106 static void gen_ld_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) 107 { 108 if (TCG_TARGET_REG_BITS == 32) { 109 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I64, 110 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), 111 addr, oi); 112 } else { 113 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); 114 } 115 } 116 117 static void gen_st_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi) 118 { 119 if (TCG_TARGET_REG_BITS == 32) { 120 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I64, 121 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)), 122 addr, oi); 123 } else { 124 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi); 125 } 126 } 127 128 static void tcg_gen_req_mo(TCGBar type) 129 { 130 type &= tcg_ctx->guest_mo; 131 type &= ~TCG_TARGET_DEFAULT_MO; 132 if (type) { 133 tcg_gen_mb(type | TCG_BAR_SC); 134 } 135 } 136 137 /* Only required for loads, where value might overlap addr. */ 138 static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr) 139 { 140 #ifdef CONFIG_PLUGIN 141 if (tcg_ctx->plugin_insn != NULL) { 142 /* Save a copy of the vaddr for use after a load. */ 143 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 144 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 145 tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr)); 146 } else { 147 tcg_gen_mov_i64(temp, temp_tcgv_i64(addr)); 148 } 149 return temp; 150 } 151 #endif 152 return NULL; 153 } 154 155 #ifdef CONFIG_PLUGIN 156 static void 157 plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi, 158 enum qemu_plugin_mem_rw rw) 159 { 160 if (tcg_ctx->plugin_insn != NULL) { 161 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw); 162 163 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 164 if (!copy_addr) { 165 copy_addr = tcg_temp_ebb_new_i64(); 166 tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr)); 167 } 168 tcg_gen_plugin_mem_cb(copy_addr, info); 169 tcg_temp_free_i64(copy_addr); 170 } else { 171 if (copy_addr) { 172 tcg_gen_plugin_mem_cb(copy_addr, info); 173 tcg_temp_free_i64(copy_addr); 174 } else { 175 tcg_gen_plugin_mem_cb(temp_tcgv_i64(orig_addr), info); 176 } 177 } 178 } 179 } 180 #endif 181 182 static void 183 plugin_gen_mem_callbacks_i32(TCGv_i32 val, 184 TCGv_i64 copy_addr, TCGTemp *orig_addr, 185 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 186 { 187 #ifdef CONFIG_PLUGIN 188 if (tcg_ctx->plugin_insn != NULL) { 189 tcg_gen_st_i32(val, tcg_env, 190 offsetof(CPUState, neg.plugin_mem_value_low) - 191 sizeof(CPUState) + (HOST_BIG_ENDIAN * 4)); 192 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 193 } 194 #endif 195 } 196 197 static void 198 plugin_gen_mem_callbacks_i64(TCGv_i64 val, 199 TCGv_i64 copy_addr, TCGTemp *orig_addr, 200 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 201 { 202 #ifdef CONFIG_PLUGIN 203 if (tcg_ctx->plugin_insn != NULL) { 204 tcg_gen_st_i64(val, tcg_env, 205 offsetof(CPUState, neg.plugin_mem_value_low) - 206 sizeof(CPUState)); 207 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 208 } 209 #endif 210 } 211 212 static void 213 plugin_gen_mem_callbacks_i128(TCGv_i128 val, 214 TCGv_i64 copy_addr, TCGTemp *orig_addr, 215 MemOpIdx oi, enum qemu_plugin_mem_rw rw) 216 { 217 #ifdef CONFIG_PLUGIN 218 if (tcg_ctx->plugin_insn != NULL) { 219 tcg_gen_st_i64(TCGV128_LOW(val), tcg_env, 220 offsetof(CPUState, neg.plugin_mem_value_low) - 221 sizeof(CPUState)); 222 tcg_gen_st_i64(TCGV128_HIGH(val), tcg_env, 223 offsetof(CPUState, neg.plugin_mem_value_high) - 224 sizeof(CPUState)); 225 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw); 226 } 227 #endif 228 } 229 230 static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr, 231 TCGArg idx, MemOp memop) 232 { 233 MemOp orig_memop; 234 MemOpIdx orig_oi, oi; 235 TCGv_i64 copy_addr; 236 237 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 238 orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0); 239 orig_oi = oi = make_memop_idx(memop, idx); 240 241 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 242 memop &= ~MO_BSWAP; 243 /* The bswap primitive benefits from zero-extended input. */ 244 if ((memop & MO_SSIZE) == MO_SW) { 245 memop &= ~MO_SIGN; 246 } 247 oi = make_memop_idx(memop, idx); 248 } 249 250 copy_addr = plugin_maybe_preserve_addr(addr); 251 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); 252 plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi, 253 QEMU_PLUGIN_MEM_R); 254 255 if ((orig_memop ^ memop) & MO_BSWAP) { 256 switch (orig_memop & MO_SIZE) { 257 case MO_16: 258 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN 259 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 260 : TCG_BSWAP_IZ | TCG_BSWAP_OZ)); 261 break; 262 case MO_32: 263 tcg_gen_bswap32_i32(val, val); 264 break; 265 default: 266 g_assert_not_reached(); 267 } 268 } 269 } 270 271 void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 272 MemOp memop, TCGType addr_type) 273 { 274 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 275 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 276 tcg_gen_qemu_ld_i32_int(val, addr, idx, memop); 277 } 278 279 static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr, 280 TCGArg idx, MemOp memop) 281 { 282 TCGv_i32 swap = NULL; 283 MemOpIdx orig_oi, oi; 284 285 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 286 memop = tcg_canonicalize_memop(memop, 0, 1); 287 orig_oi = oi = make_memop_idx(memop, idx); 288 289 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 290 swap = tcg_temp_ebb_new_i32(); 291 switch (memop & MO_SIZE) { 292 case MO_16: 293 tcg_gen_bswap16_i32(swap, val, 0); 294 break; 295 case MO_32: 296 tcg_gen_bswap32_i32(swap, val); 297 break; 298 default: 299 g_assert_not_reached(); 300 } 301 val = swap; 302 memop &= ~MO_BSWAP; 303 oi = make_memop_idx(memop, idx); 304 } 305 306 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi); 307 plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 308 309 if (swap) { 310 tcg_temp_free_i32(swap); 311 } 312 } 313 314 void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx, 315 MemOp memop, TCGType addr_type) 316 { 317 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 318 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 319 tcg_gen_qemu_st_i32_int(val, addr, idx, memop); 320 } 321 322 static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr, 323 TCGArg idx, MemOp memop) 324 { 325 MemOp orig_memop; 326 MemOpIdx orig_oi, oi; 327 TCGv_i64 copy_addr; 328 329 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 330 tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop); 331 if (memop & MO_SIGN) { 332 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 333 } else { 334 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 335 } 336 return; 337 } 338 339 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 340 orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0); 341 orig_oi = oi = make_memop_idx(memop, idx); 342 343 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 344 memop &= ~MO_BSWAP; 345 /* The bswap primitive benefits from zero-extended input. */ 346 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) { 347 memop &= ~MO_SIGN; 348 } 349 oi = make_memop_idx(memop, idx); 350 } 351 352 copy_addr = plugin_maybe_preserve_addr(addr); 353 gen_ld_i64(val, addr, oi); 354 plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi, 355 QEMU_PLUGIN_MEM_R); 356 357 if ((orig_memop ^ memop) & MO_BSWAP) { 358 int flags = (orig_memop & MO_SIGN 359 ? TCG_BSWAP_IZ | TCG_BSWAP_OS 360 : TCG_BSWAP_IZ | TCG_BSWAP_OZ); 361 switch (orig_memop & MO_SIZE) { 362 case MO_16: 363 tcg_gen_bswap16_i64(val, val, flags); 364 break; 365 case MO_32: 366 tcg_gen_bswap32_i64(val, val, flags); 367 break; 368 case MO_64: 369 tcg_gen_bswap64_i64(val, val); 370 break; 371 default: 372 g_assert_not_reached(); 373 } 374 } 375 } 376 377 void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 378 MemOp memop, TCGType addr_type) 379 { 380 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 381 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 382 tcg_gen_qemu_ld_i64_int(val, addr, idx, memop); 383 } 384 385 static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr, 386 TCGArg idx, MemOp memop) 387 { 388 TCGv_i64 swap = NULL; 389 MemOpIdx orig_oi, oi; 390 391 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 392 tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop); 393 return; 394 } 395 396 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST); 397 memop = tcg_canonicalize_memop(memop, 1, 1); 398 orig_oi = oi = make_memop_idx(memop, idx); 399 400 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 401 swap = tcg_temp_ebb_new_i64(); 402 switch (memop & MO_SIZE) { 403 case MO_16: 404 tcg_gen_bswap16_i64(swap, val, 0); 405 break; 406 case MO_32: 407 tcg_gen_bswap32_i64(swap, val, 0); 408 break; 409 case MO_64: 410 tcg_gen_bswap64_i64(swap, val); 411 break; 412 default: 413 g_assert_not_reached(); 414 } 415 val = swap; 416 memop &= ~MO_BSWAP; 417 oi = make_memop_idx(memop, idx); 418 } 419 420 gen_st_i64(val, addr, oi); 421 plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W); 422 423 if (swap) { 424 tcg_temp_free_i64(swap); 425 } 426 } 427 428 void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx, 429 MemOp memop, TCGType addr_type) 430 { 431 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 432 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 433 tcg_gen_qemu_st_i64_int(val, addr, idx, memop); 434 } 435 436 /* 437 * Return true if @mop, without knowledge of the pointer alignment, 438 * does not require 16-byte atomicity, and it would be adventagous 439 * to avoid a call to a helper function. 440 */ 441 static bool use_two_i64_for_i128(MemOp mop) 442 { 443 /* Two softmmu tlb lookups is larger than one function call. */ 444 if (tcg_use_softmmu) { 445 return false; 446 } 447 448 /* 449 * For user-only, two 64-bit operations may well be smaller than a call. 450 * Determine if that would be legal for the requested atomicity. 451 */ 452 switch (mop & MO_ATOM_MASK) { 453 case MO_ATOM_NONE: 454 case MO_ATOM_IFALIGN_PAIR: 455 return true; 456 case MO_ATOM_IFALIGN: 457 case MO_ATOM_SUBALIGN: 458 case MO_ATOM_WITHIN16: 459 case MO_ATOM_WITHIN16_PAIR: 460 return false; 461 default: 462 g_assert_not_reached(); 463 } 464 } 465 466 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig) 467 { 468 MemOp mop_1 = orig, mop_2; 469 470 /* Reduce the size to 64-bit. */ 471 mop_1 = (mop_1 & ~MO_SIZE) | MO_64; 472 473 /* Retain the alignment constraints of the original. */ 474 switch (orig & MO_AMASK) { 475 case MO_UNALN: 476 case MO_ALIGN_2: 477 case MO_ALIGN_4: 478 mop_2 = mop_1; 479 break; 480 case MO_ALIGN_8: 481 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */ 482 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 483 mop_2 = mop_1; 484 break; 485 case MO_ALIGN: 486 /* Second has 8-byte alignment; first has 16-byte alignment. */ 487 mop_2 = mop_1; 488 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16; 489 break; 490 case MO_ALIGN_16: 491 case MO_ALIGN_32: 492 case MO_ALIGN_64: 493 /* Second has 8-byte alignment; first retains original. */ 494 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN; 495 break; 496 default: 497 g_assert_not_reached(); 498 } 499 500 /* Use a memory ordering implemented by the host. */ 501 if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) { 502 mop_1 &= ~MO_BSWAP; 503 mop_2 &= ~MO_BSWAP; 504 } 505 506 ret[0] = mop_1; 507 ret[1] = mop_2; 508 } 509 510 static TCGv_i64 maybe_extend_addr64(TCGTemp *addr) 511 { 512 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 513 TCGv_i64 a64 = tcg_temp_ebb_new_i64(); 514 tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr)); 515 return a64; 516 } 517 return temp_tcgv_i64(addr); 518 } 519 520 static void maybe_free_addr64(TCGv_i64 a64) 521 { 522 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 523 tcg_temp_free_i64(a64); 524 } 525 } 526 527 static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr, 528 TCGArg idx, MemOp memop) 529 { 530 MemOpIdx orig_oi; 531 TCGv_i64 ext_addr = NULL; 532 533 check_max_alignment(memop_alignment_bits(memop)); 534 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD); 535 536 /* In serial mode, reduce atomicity. */ 537 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 538 memop &= ~MO_ATOM_MASK; 539 memop |= MO_ATOM_NONE; 540 } 541 orig_oi = make_memop_idx(memop, idx); 542 543 /* TODO: For now, force 32-bit hosts to use the helper. */ 544 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 545 TCGv_i64 lo, hi; 546 bool need_bswap = false; 547 MemOpIdx oi = orig_oi; 548 549 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 550 lo = TCGV128_HIGH(val); 551 hi = TCGV128_LOW(val); 552 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 553 need_bswap = true; 554 } else { 555 lo = TCGV128_LOW(val); 556 hi = TCGV128_HIGH(val); 557 } 558 559 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I128, tcgv_i64_temp(lo), 560 tcgv_i64_temp(hi), addr, oi); 561 562 if (need_bswap) { 563 tcg_gen_bswap64_i64(lo, lo); 564 tcg_gen_bswap64_i64(hi, hi); 565 } 566 } else if (use_two_i64_for_i128(memop)) { 567 MemOp mop[2]; 568 TCGTemp *addr_p8; 569 TCGv_i64 x, y; 570 bool need_bswap; 571 572 canonicalize_memop_i128_as_i64(mop, memop); 573 need_bswap = (mop[0] ^ memop) & MO_BSWAP; 574 575 /* 576 * Since there are no global TCGv_i128, there is no visible state 577 * changed if the second load faults. Load directly into the two 578 * subwords. 579 */ 580 if ((memop & MO_BSWAP) == MO_LE) { 581 x = TCGV128_LOW(val); 582 y = TCGV128_HIGH(val); 583 } else { 584 x = TCGV128_HIGH(val); 585 y = TCGV128_LOW(val); 586 } 587 588 gen_ld_i64(x, addr, make_memop_idx(mop[0], idx)); 589 590 if (need_bswap) { 591 tcg_gen_bswap64_i64(x, x); 592 } 593 594 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 595 TCGv_i32 t = tcg_temp_ebb_new_i32(); 596 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 597 addr_p8 = tcgv_i32_temp(t); 598 } else { 599 TCGv_i64 t = tcg_temp_ebb_new_i64(); 600 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 601 addr_p8 = tcgv_i64_temp(t); 602 } 603 604 gen_ld_i64(y, addr_p8, make_memop_idx(mop[1], idx)); 605 tcg_temp_free_internal(addr_p8); 606 607 if (need_bswap) { 608 tcg_gen_bswap64_i64(y, y); 609 } 610 } else { 611 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 612 ext_addr = tcg_temp_ebb_new_i64(); 613 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 614 addr = tcgv_i64_temp(ext_addr); 615 } 616 gen_helper_ld_i128(val, tcg_env, temp_tcgv_i64(addr), 617 tcg_constant_i32(orig_oi)); 618 } 619 620 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 621 QEMU_PLUGIN_MEM_R); 622 } 623 624 void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 625 MemOp memop, TCGType addr_type) 626 { 627 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 628 tcg_debug_assert((memop & MO_SIZE) == MO_128); 629 tcg_debug_assert((memop & MO_SIGN) == 0); 630 tcg_gen_qemu_ld_i128_int(val, addr, idx, memop); 631 } 632 633 static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr, 634 TCGArg idx, MemOp memop) 635 { 636 MemOpIdx orig_oi; 637 TCGv_i64 ext_addr = NULL; 638 639 check_max_alignment(memop_alignment_bits(memop)); 640 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST); 641 642 /* In serial mode, reduce atomicity. */ 643 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 644 memop &= ~MO_ATOM_MASK; 645 memop |= MO_ATOM_NONE; 646 } 647 orig_oi = make_memop_idx(memop, idx); 648 649 /* TODO: For now, force 32-bit hosts to use the helper. */ 650 651 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) { 652 TCGv_i64 lo, hi; 653 MemOpIdx oi = orig_oi; 654 bool need_bswap = false; 655 656 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) { 657 lo = tcg_temp_ebb_new_i64(); 658 hi = tcg_temp_ebb_new_i64(); 659 tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val)); 660 tcg_gen_bswap64_i64(hi, TCGV128_LOW(val)); 661 oi = make_memop_idx(memop & ~MO_BSWAP, idx); 662 need_bswap = true; 663 } else { 664 lo = TCGV128_LOW(val); 665 hi = TCGV128_HIGH(val); 666 } 667 668 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I128, 669 tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi); 670 671 if (need_bswap) { 672 tcg_temp_free_i64(lo); 673 tcg_temp_free_i64(hi); 674 } 675 } else if (use_two_i64_for_i128(memop)) { 676 MemOp mop[2]; 677 TCGTemp *addr_p8; 678 TCGv_i64 x, y, b = NULL; 679 680 canonicalize_memop_i128_as_i64(mop, memop); 681 682 if ((memop & MO_BSWAP) == MO_LE) { 683 x = TCGV128_LOW(val); 684 y = TCGV128_HIGH(val); 685 } else { 686 x = TCGV128_HIGH(val); 687 y = TCGV128_LOW(val); 688 } 689 690 if ((mop[0] ^ memop) & MO_BSWAP) { 691 b = tcg_temp_ebb_new_i64(); 692 tcg_gen_bswap64_i64(b, x); 693 x = b; 694 } 695 696 gen_st_i64(x, addr, make_memop_idx(mop[0], idx)); 697 698 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 699 TCGv_i32 t = tcg_temp_ebb_new_i32(); 700 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8); 701 addr_p8 = tcgv_i32_temp(t); 702 } else { 703 TCGv_i64 t = tcg_temp_ebb_new_i64(); 704 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8); 705 addr_p8 = tcgv_i64_temp(t); 706 } 707 708 if (b) { 709 tcg_gen_bswap64_i64(b, y); 710 gen_st_i64(b, addr_p8, make_memop_idx(mop[1], idx)); 711 tcg_temp_free_i64(b); 712 } else { 713 gen_st_i64(y, addr_p8, make_memop_idx(mop[1], idx)); 714 } 715 tcg_temp_free_internal(addr_p8); 716 } else { 717 if (tcg_ctx->addr_type == TCG_TYPE_I32) { 718 ext_addr = tcg_temp_ebb_new_i64(); 719 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr)); 720 addr = tcgv_i64_temp(ext_addr); 721 } 722 gen_helper_st_i128(tcg_env, temp_tcgv_i64(addr), val, 723 tcg_constant_i32(orig_oi)); 724 } 725 726 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi, 727 QEMU_PLUGIN_MEM_W); 728 } 729 730 void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx, 731 MemOp memop, TCGType addr_type) 732 { 733 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 734 tcg_debug_assert((memop & MO_SIZE) == MO_128); 735 tcg_debug_assert((memop & MO_SIGN) == 0); 736 tcg_gen_qemu_st_i128_int(val, addr, idx, memop); 737 } 738 739 void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc) 740 { 741 switch (opc & MO_SSIZE) { 742 case MO_SB: 743 tcg_gen_ext8s_i32(ret, val); 744 break; 745 case MO_UB: 746 tcg_gen_ext8u_i32(ret, val); 747 break; 748 case MO_SW: 749 tcg_gen_ext16s_i32(ret, val); 750 break; 751 case MO_UW: 752 tcg_gen_ext16u_i32(ret, val); 753 break; 754 case MO_UL: 755 case MO_SL: 756 tcg_gen_mov_i32(ret, val); 757 break; 758 default: 759 g_assert_not_reached(); 760 } 761 } 762 763 void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc) 764 { 765 switch (opc & MO_SSIZE) { 766 case MO_SB: 767 tcg_gen_ext8s_i64(ret, val); 768 break; 769 case MO_UB: 770 tcg_gen_ext8u_i64(ret, val); 771 break; 772 case MO_SW: 773 tcg_gen_ext16s_i64(ret, val); 774 break; 775 case MO_UW: 776 tcg_gen_ext16u_i64(ret, val); 777 break; 778 case MO_SL: 779 tcg_gen_ext32s_i64(ret, val); 780 break; 781 case MO_UL: 782 tcg_gen_ext32u_i64(ret, val); 783 break; 784 case MO_UQ: 785 case MO_SQ: 786 tcg_gen_mov_i64(ret, val); 787 break; 788 default: 789 g_assert_not_reached(); 790 } 791 } 792 793 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64, 794 TCGv_i32, TCGv_i32, TCGv_i32); 795 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64, 796 TCGv_i64, TCGv_i64, TCGv_i32); 797 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64, 798 TCGv_i128, TCGv_i128, TCGv_i32); 799 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64, 800 TCGv_i32, TCGv_i32); 801 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64, 802 TCGv_i64, TCGv_i32); 803 804 #ifdef CONFIG_ATOMIC64 805 # define WITH_ATOMIC64(X) X, 806 #else 807 # define WITH_ATOMIC64(X) 808 #endif 809 #if HAVE_CMPXCHG128 810 # define WITH_ATOMIC128(X) X, 811 #else 812 # define WITH_ATOMIC128(X) 813 #endif 814 815 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = { 816 [MO_8] = gen_helper_atomic_cmpxchgb, 817 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 818 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 819 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 820 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 821 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 822 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 823 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le) 824 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be) 825 }; 826 827 static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 828 TCGv_i32 cmpv, TCGv_i32 newv, 829 TCGArg idx, MemOp memop) 830 { 831 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 832 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 833 834 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 835 836 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN); 837 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 838 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 839 tcg_temp_free_i32(t2); 840 841 if (memop & MO_SIGN) { 842 tcg_gen_ext_i32(retv, t1, memop); 843 } else { 844 tcg_gen_mov_i32(retv, t1); 845 } 846 tcg_temp_free_i32(t1); 847 } 848 849 void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 850 TCGv_i32 cmpv, TCGv_i32 newv, 851 TCGArg idx, MemOp memop, 852 TCGType addr_type) 853 { 854 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 855 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 856 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 857 } 858 859 static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr, 860 TCGv_i32 cmpv, TCGv_i32 newv, 861 TCGArg idx, MemOp memop) 862 { 863 gen_atomic_cx_i32 gen; 864 TCGv_i64 a64; 865 MemOpIdx oi; 866 867 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 868 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 869 return; 870 } 871 872 memop = tcg_canonicalize_memop(memop, 0, 0); 873 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 874 tcg_debug_assert(gen != NULL); 875 876 oi = make_memop_idx(memop & ~MO_SIGN, idx); 877 a64 = maybe_extend_addr64(addr); 878 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 879 maybe_free_addr64(a64); 880 881 if (memop & MO_SIGN) { 882 tcg_gen_ext_i32(retv, retv, memop); 883 } 884 } 885 886 void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr, 887 TCGv_i32 cmpv, TCGv_i32 newv, 888 TCGArg idx, MemOp memop, 889 TCGType addr_type) 890 { 891 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 892 tcg_debug_assert((memop & MO_SIZE) <= MO_32); 893 tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop); 894 } 895 896 static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 897 TCGv_i64 cmpv, TCGv_i64 newv, 898 TCGArg idx, MemOp memop) 899 { 900 TCGv_i64 t1, t2; 901 902 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 903 tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 904 TCGV_LOW(newv), idx, memop); 905 if (memop & MO_SIGN) { 906 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 907 } else { 908 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 909 } 910 return; 911 } 912 913 t1 = tcg_temp_ebb_new_i64(); 914 t2 = tcg_temp_ebb_new_i64(); 915 916 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 917 918 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN); 919 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 920 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 921 tcg_temp_free_i64(t2); 922 923 if (memop & MO_SIGN) { 924 tcg_gen_ext_i64(retv, t1, memop); 925 } else { 926 tcg_gen_mov_i64(retv, t1); 927 } 928 tcg_temp_free_i64(t1); 929 } 930 931 void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 932 TCGv_i64 cmpv, TCGv_i64 newv, 933 TCGArg idx, MemOp memop, 934 TCGType addr_type) 935 { 936 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 937 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 938 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 939 } 940 941 static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr, 942 TCGv_i64 cmpv, TCGv_i64 newv, 943 TCGArg idx, MemOp memop) 944 { 945 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 946 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 947 return; 948 } 949 950 if ((memop & MO_SIZE) == MO_64) { 951 gen_atomic_cx_i64 gen; 952 953 memop = tcg_canonicalize_memop(memop, 1, 0); 954 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 955 if (gen) { 956 MemOpIdx oi = make_memop_idx(memop, idx); 957 TCGv_i64 a64 = maybe_extend_addr64(addr); 958 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 959 maybe_free_addr64(a64); 960 return; 961 } 962 963 gen_helper_exit_atomic(tcg_env); 964 965 /* 966 * Produce a result for a well-formed opcode stream. This satisfies 967 * liveness for set before used, which happens before this dead code 968 * is removed. 969 */ 970 tcg_gen_movi_i64(retv, 0); 971 return; 972 } 973 974 if (TCG_TARGET_REG_BITS == 32) { 975 tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv), 976 TCGV_LOW(newv), idx, memop); 977 if (memop & MO_SIGN) { 978 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31); 979 } else { 980 tcg_gen_movi_i32(TCGV_HIGH(retv), 0); 981 } 982 } else { 983 TCGv_i32 c32 = tcg_temp_ebb_new_i32(); 984 TCGv_i32 n32 = tcg_temp_ebb_new_i32(); 985 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 986 987 tcg_gen_extrl_i64_i32(c32, cmpv); 988 tcg_gen_extrl_i64_i32(n32, newv); 989 tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32, 990 idx, memop & ~MO_SIGN); 991 tcg_temp_free_i32(c32); 992 tcg_temp_free_i32(n32); 993 994 tcg_gen_extu_i32_i64(retv, r32); 995 tcg_temp_free_i32(r32); 996 997 if (memop & MO_SIGN) { 998 tcg_gen_ext_i64(retv, retv, memop); 999 } 1000 } 1001 } 1002 1003 void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr, 1004 TCGv_i64 cmpv, TCGv_i64 newv, 1005 TCGArg idx, MemOp memop, TCGType addr_type) 1006 { 1007 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1008 tcg_debug_assert((memop & MO_SIZE) <= MO_64); 1009 tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop); 1010 } 1011 1012 static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1013 TCGv_i128 cmpv, TCGv_i128 newv, 1014 TCGArg idx, MemOp memop) 1015 { 1016 if (TCG_TARGET_REG_BITS == 32) { 1017 /* Inline expansion below is simply too large for 32-bit hosts. */ 1018 MemOpIdx oi = make_memop_idx(memop, idx); 1019 TCGv_i64 a64 = maybe_extend_addr64(addr); 1020 1021 gen_helper_nonatomic_cmpxchgo(retv, tcg_env, a64, cmpv, newv, 1022 tcg_constant_i32(oi)); 1023 maybe_free_addr64(a64); 1024 } else { 1025 TCGv_i128 oldv = tcg_temp_ebb_new_i128(); 1026 TCGv_i128 tmpv = tcg_temp_ebb_new_i128(); 1027 TCGv_i64 t0 = tcg_temp_ebb_new_i64(); 1028 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1029 TCGv_i64 z = tcg_constant_i64(0); 1030 1031 tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop); 1032 1033 /* Compare i128 */ 1034 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv)); 1035 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv)); 1036 tcg_gen_or_i64(t0, t0, t1); 1037 1038 /* tmpv = equal ? newv : oldv */ 1039 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z, 1040 TCGV128_LOW(newv), TCGV128_LOW(oldv)); 1041 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z, 1042 TCGV128_HIGH(newv), TCGV128_HIGH(oldv)); 1043 1044 /* Unconditional writeback. */ 1045 tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop); 1046 tcg_gen_mov_i128(retv, oldv); 1047 1048 tcg_temp_free_i64(t0); 1049 tcg_temp_free_i64(t1); 1050 tcg_temp_free_i128(tmpv); 1051 tcg_temp_free_i128(oldv); 1052 } 1053 } 1054 1055 void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1056 TCGv_i128 cmpv, TCGv_i128 newv, 1057 TCGArg idx, MemOp memop, 1058 TCGType addr_type) 1059 { 1060 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1061 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1062 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1063 } 1064 1065 static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr, 1066 TCGv_i128 cmpv, TCGv_i128 newv, 1067 TCGArg idx, MemOp memop) 1068 { 1069 gen_atomic_cx_i128 gen; 1070 1071 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) { 1072 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1073 return; 1074 } 1075 1076 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 1077 if (gen) { 1078 MemOpIdx oi = make_memop_idx(memop, idx); 1079 TCGv_i64 a64 = maybe_extend_addr64(addr); 1080 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi)); 1081 maybe_free_addr64(a64); 1082 return; 1083 } 1084 1085 gen_helper_exit_atomic(tcg_env); 1086 1087 /* 1088 * Produce a result for a well-formed opcode stream. This satisfies 1089 * liveness for set before used, which happens before this dead code 1090 * is removed. 1091 */ 1092 tcg_gen_movi_i64(TCGV128_LOW(retv), 0); 1093 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0); 1094 } 1095 1096 void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr, 1097 TCGv_i128 cmpv, TCGv_i128 newv, 1098 TCGArg idx, MemOp memop, 1099 TCGType addr_type) 1100 { 1101 tcg_debug_assert(addr_type == tcg_ctx->addr_type); 1102 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128); 1103 tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop); 1104 } 1105 1106 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1107 TCGArg idx, MemOp memop, bool new_val, 1108 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 1109 { 1110 TCGv_i32 t1 = tcg_temp_ebb_new_i32(); 1111 TCGv_i32 t2 = tcg_temp_ebb_new_i32(); 1112 1113 memop = tcg_canonicalize_memop(memop, 0, 0); 1114 1115 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop); 1116 tcg_gen_ext_i32(t2, val, memop); 1117 gen(t2, t1, t2); 1118 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop); 1119 1120 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 1121 tcg_temp_free_i32(t1); 1122 tcg_temp_free_i32(t2); 1123 } 1124 1125 static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val, 1126 TCGArg idx, MemOp memop, void * const table[]) 1127 { 1128 gen_atomic_op_i32 gen; 1129 TCGv_i64 a64; 1130 MemOpIdx oi; 1131 1132 memop = tcg_canonicalize_memop(memop, 0, 0); 1133 1134 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1135 tcg_debug_assert(gen != NULL); 1136 1137 oi = make_memop_idx(memop & ~MO_SIGN, idx); 1138 a64 = maybe_extend_addr64(addr); 1139 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1140 maybe_free_addr64(a64); 1141 1142 if (memop & MO_SIGN) { 1143 tcg_gen_ext_i32(ret, ret, memop); 1144 } 1145 } 1146 1147 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1148 TCGArg idx, MemOp memop, bool new_val, 1149 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 1150 { 1151 TCGv_i64 t1 = tcg_temp_ebb_new_i64(); 1152 TCGv_i64 t2 = tcg_temp_ebb_new_i64(); 1153 1154 memop = tcg_canonicalize_memop(memop, 1, 0); 1155 1156 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop); 1157 tcg_gen_ext_i64(t2, val, memop); 1158 gen(t2, t1, t2); 1159 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop); 1160 1161 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 1162 tcg_temp_free_i64(t1); 1163 tcg_temp_free_i64(t2); 1164 } 1165 1166 static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val, 1167 TCGArg idx, MemOp memop, void * const table[]) 1168 { 1169 memop = tcg_canonicalize_memop(memop, 1, 0); 1170 1171 if ((memop & MO_SIZE) == MO_64) { 1172 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 1173 1174 if (gen) { 1175 MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx); 1176 TCGv_i64 a64 = maybe_extend_addr64(addr); 1177 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi)); 1178 maybe_free_addr64(a64); 1179 return; 1180 } 1181 1182 gen_helper_exit_atomic(tcg_env); 1183 /* Produce a result, so that we have a well-formed opcode stream 1184 with respect to uses of the result in the (dead) code following. */ 1185 tcg_gen_movi_i64(ret, 0); 1186 } else { 1187 TCGv_i32 v32 = tcg_temp_ebb_new_i32(); 1188 TCGv_i32 r32 = tcg_temp_ebb_new_i32(); 1189 1190 tcg_gen_extrl_i64_i32(v32, val); 1191 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 1192 tcg_temp_free_i32(v32); 1193 1194 tcg_gen_extu_i32_i64(ret, r32); 1195 tcg_temp_free_i32(r32); 1196 1197 if (memop & MO_SIGN) { 1198 tcg_gen_ext_i64(ret, ret, memop); 1199 } 1200 } 1201 } 1202 1203 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 1204 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \ 1205 [MO_8] = gen_helper_atomic_##NAME##b, \ 1206 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 1207 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 1208 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 1209 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 1210 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 1211 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 1212 }; \ 1213 void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \ 1214 TCGv_i32 val, TCGArg idx, \ 1215 MemOp memop, TCGType addr_type) \ 1216 { \ 1217 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1218 tcg_debug_assert((memop & MO_SIZE) <= MO_32); \ 1219 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1220 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 1221 } else { \ 1222 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 1223 tcg_gen_##OP##_i32); \ 1224 } \ 1225 } \ 1226 void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \ 1227 TCGv_i64 val, TCGArg idx, \ 1228 MemOp memop, TCGType addr_type) \ 1229 { \ 1230 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \ 1231 tcg_debug_assert((memop & MO_SIZE) <= MO_64); \ 1232 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \ 1233 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 1234 } else { \ 1235 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 1236 tcg_gen_##OP##_i64); \ 1237 } \ 1238 } 1239 1240 GEN_ATOMIC_HELPER(fetch_add, add, 0) 1241 GEN_ATOMIC_HELPER(fetch_and, and, 0) 1242 GEN_ATOMIC_HELPER(fetch_or, or, 0) 1243 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 1244 GEN_ATOMIC_HELPER(fetch_smin, smin, 0) 1245 GEN_ATOMIC_HELPER(fetch_umin, umin, 0) 1246 GEN_ATOMIC_HELPER(fetch_smax, smax, 0) 1247 GEN_ATOMIC_HELPER(fetch_umax, umax, 0) 1248 1249 GEN_ATOMIC_HELPER(add_fetch, add, 1) 1250 GEN_ATOMIC_HELPER(and_fetch, and, 1) 1251 GEN_ATOMIC_HELPER(or_fetch, or, 1) 1252 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 1253 GEN_ATOMIC_HELPER(smin_fetch, smin, 1) 1254 GEN_ATOMIC_HELPER(umin_fetch, umin, 1) 1255 GEN_ATOMIC_HELPER(smax_fetch, smax, 1) 1256 GEN_ATOMIC_HELPER(umax_fetch, umax, 1) 1257 1258 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 1259 { 1260 tcg_gen_mov_i32(r, b); 1261 } 1262 1263 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 1264 { 1265 tcg_gen_mov_i64(r, b); 1266 } 1267 1268 GEN_ATOMIC_HELPER(xchg, mov2, 0) 1269 1270 #undef GEN_ATOMIC_HELPER 1271