1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "tcg/tcg.h"
27 #include "tcg/tcg-temp-internal.h"
28 #include "tcg/tcg-op-common.h"
29 #include "tcg/tcg-mo.h"
30 #include "exec/target_page.h"
31 #include "exec/translation-block.h"
32 #include "exec/plugin-gen.h"
33 #include "tcg-internal.h"
34 #include "tcg-has.h"
35 #include "tcg-target-mo.h"
36
check_max_alignment(unsigned a_bits)37 static void check_max_alignment(unsigned a_bits)
38 {
39 /*
40 * The requested alignment cannot overlap the TLB flags.
41 * FIXME: Must keep the count up-to-date with "exec/tlb-flags.h".
42 */
43 if (tcg_use_softmmu) {
44 tcg_debug_assert(a_bits + 5 <= TARGET_PAGE_BITS);
45 }
46 }
47
tcg_canonicalize_memop(MemOp op,bool is64,bool st)48 static MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
49 {
50 unsigned a_bits = memop_alignment_bits(op);
51
52 check_max_alignment(a_bits);
53
54 /* Prefer MO_ALIGN+MO_XX over MO_ALIGN_XX+MO_XX */
55 if (a_bits == (op & MO_SIZE)) {
56 op = (op & ~MO_AMASK) | MO_ALIGN;
57 }
58
59 switch (op & MO_SIZE) {
60 case MO_8:
61 op &= ~MO_BSWAP;
62 break;
63 case MO_16:
64 break;
65 case MO_32:
66 if (!is64) {
67 op &= ~MO_SIGN;
68 }
69 break;
70 case MO_64:
71 if (is64) {
72 op &= ~MO_SIGN;
73 break;
74 }
75 /* fall through */
76 default:
77 g_assert_not_reached();
78 }
79 if (st) {
80 op &= ~MO_SIGN;
81 }
82
83 /* In serial mode, reduce atomicity. */
84 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
85 op &= ~MO_ATOM_MASK;
86 op |= MO_ATOM_NONE;
87 }
88
89 return op;
90 }
91
gen_ldst1(TCGOpcode opc,TCGType type,TCGTemp * v,TCGTemp * addr,MemOpIdx oi)92 static void gen_ldst1(TCGOpcode opc, TCGType type, TCGTemp *v,
93 TCGTemp *addr, MemOpIdx oi)
94 {
95 TCGOp *op = tcg_gen_op3(opc, type, temp_arg(v), temp_arg(addr), oi);
96 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE;
97 }
98
gen_ldst2(TCGOpcode opc,TCGType type,TCGTemp * vl,TCGTemp * vh,TCGTemp * addr,MemOpIdx oi)99 static void gen_ldst2(TCGOpcode opc, TCGType type, TCGTemp *vl, TCGTemp *vh,
100 TCGTemp *addr, MemOpIdx oi)
101 {
102 TCGOp *op = tcg_gen_op4(opc, type, temp_arg(vl), temp_arg(vh),
103 temp_arg(addr), oi);
104 TCGOP_FLAGS(op) = get_memop(oi) & MO_SIZE;
105 }
106
gen_ld_i64(TCGv_i64 v,TCGTemp * addr,MemOpIdx oi)107 static void gen_ld_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi)
108 {
109 if (TCG_TARGET_REG_BITS == 32) {
110 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I64,
111 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)),
112 addr, oi);
113 } else {
114 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi);
115 }
116 }
117
gen_st_i64(TCGv_i64 v,TCGTemp * addr,MemOpIdx oi)118 static void gen_st_i64(TCGv_i64 v, TCGTemp *addr, MemOpIdx oi)
119 {
120 if (TCG_TARGET_REG_BITS == 32) {
121 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I64,
122 tcgv_i32_temp(TCGV_LOW(v)), tcgv_i32_temp(TCGV_HIGH(v)),
123 addr, oi);
124 } else {
125 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I64, tcgv_i64_temp(v), addr, oi);
126 }
127 }
128
tcg_gen_req_mo(TCGBar type)129 static void tcg_gen_req_mo(TCGBar type)
130 {
131 type &= tcg_ctx->guest_mo;
132 type &= ~TCG_TARGET_DEFAULT_MO;
133 if (type) {
134 tcg_gen_mb(type | TCG_BAR_SC);
135 }
136 }
137
138 /* Only required for loads, where value might overlap addr. */
plugin_maybe_preserve_addr(TCGTemp * addr)139 static TCGv_i64 plugin_maybe_preserve_addr(TCGTemp *addr)
140 {
141 #ifdef CONFIG_PLUGIN
142 if (tcg_ctx->plugin_insn != NULL) {
143 /* Save a copy of the vaddr for use after a load. */
144 TCGv_i64 temp = tcg_temp_ebb_new_i64();
145 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
146 tcg_gen_extu_i32_i64(temp, temp_tcgv_i32(addr));
147 } else {
148 tcg_gen_mov_i64(temp, temp_tcgv_i64(addr));
149 }
150 return temp;
151 }
152 #endif
153 return NULL;
154 }
155
156 #ifdef CONFIG_PLUGIN
157 static void
plugin_gen_mem_callbacks(TCGv_i64 copy_addr,TCGTemp * orig_addr,MemOpIdx oi,enum qemu_plugin_mem_rw rw)158 plugin_gen_mem_callbacks(TCGv_i64 copy_addr, TCGTemp *orig_addr, MemOpIdx oi,
159 enum qemu_plugin_mem_rw rw)
160 {
161 if (tcg_ctx->plugin_insn != NULL) {
162 qemu_plugin_meminfo_t info = make_plugin_meminfo(oi, rw);
163
164 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
165 if (!copy_addr) {
166 copy_addr = tcg_temp_ebb_new_i64();
167 tcg_gen_extu_i32_i64(copy_addr, temp_tcgv_i32(orig_addr));
168 }
169 tcg_gen_plugin_mem_cb(copy_addr, info);
170 tcg_temp_free_i64(copy_addr);
171 } else {
172 if (copy_addr) {
173 tcg_gen_plugin_mem_cb(copy_addr, info);
174 tcg_temp_free_i64(copy_addr);
175 } else {
176 tcg_gen_plugin_mem_cb(temp_tcgv_i64(orig_addr), info);
177 }
178 }
179 }
180 }
181 #endif
182
183 static void
plugin_gen_mem_callbacks_i32(TCGv_i32 val,TCGv_i64 copy_addr,TCGTemp * orig_addr,MemOpIdx oi,enum qemu_plugin_mem_rw rw)184 plugin_gen_mem_callbacks_i32(TCGv_i32 val,
185 TCGv_i64 copy_addr, TCGTemp *orig_addr,
186 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
187 {
188 #ifdef CONFIG_PLUGIN
189 if (tcg_ctx->plugin_insn != NULL) {
190 tcg_gen_st_i32(val, tcg_env,
191 offsetof(CPUState, neg.plugin_mem_value_low) -
192 sizeof(CPUState) + (HOST_BIG_ENDIAN * 4));
193 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
194 }
195 #endif
196 }
197
198 static void
plugin_gen_mem_callbacks_i64(TCGv_i64 val,TCGv_i64 copy_addr,TCGTemp * orig_addr,MemOpIdx oi,enum qemu_plugin_mem_rw rw)199 plugin_gen_mem_callbacks_i64(TCGv_i64 val,
200 TCGv_i64 copy_addr, TCGTemp *orig_addr,
201 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
202 {
203 #ifdef CONFIG_PLUGIN
204 if (tcg_ctx->plugin_insn != NULL) {
205 tcg_gen_st_i64(val, tcg_env,
206 offsetof(CPUState, neg.plugin_mem_value_low) -
207 sizeof(CPUState));
208 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
209 }
210 #endif
211 }
212
213 static void
plugin_gen_mem_callbacks_i128(TCGv_i128 val,TCGv_i64 copy_addr,TCGTemp * orig_addr,MemOpIdx oi,enum qemu_plugin_mem_rw rw)214 plugin_gen_mem_callbacks_i128(TCGv_i128 val,
215 TCGv_i64 copy_addr, TCGTemp *orig_addr,
216 MemOpIdx oi, enum qemu_plugin_mem_rw rw)
217 {
218 #ifdef CONFIG_PLUGIN
219 if (tcg_ctx->plugin_insn != NULL) {
220 tcg_gen_st_i64(TCGV128_LOW(val), tcg_env,
221 offsetof(CPUState, neg.plugin_mem_value_low) -
222 sizeof(CPUState));
223 tcg_gen_st_i64(TCGV128_HIGH(val), tcg_env,
224 offsetof(CPUState, neg.plugin_mem_value_high) -
225 sizeof(CPUState));
226 plugin_gen_mem_callbacks(copy_addr, orig_addr, oi, rw);
227 }
228 #endif
229 }
230
tcg_gen_qemu_ld_i32_int(TCGv_i32 val,TCGTemp * addr,TCGArg idx,MemOp memop)231 static void tcg_gen_qemu_ld_i32_int(TCGv_i32 val, TCGTemp *addr,
232 TCGArg idx, MemOp memop)
233 {
234 MemOp orig_memop;
235 MemOpIdx orig_oi, oi;
236 TCGv_i64 copy_addr;
237
238 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
239 orig_memop = memop = tcg_canonicalize_memop(memop, 0, 0);
240 orig_oi = oi = make_memop_idx(memop, idx);
241
242 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
243 memop &= ~MO_BSWAP;
244 /* The bswap primitive benefits from zero-extended input. */
245 if ((memop & MO_SSIZE) == MO_SW) {
246 memop &= ~MO_SIGN;
247 }
248 oi = make_memop_idx(memop, idx);
249 }
250
251 copy_addr = plugin_maybe_preserve_addr(addr);
252 gen_ldst1(INDEX_op_qemu_ld, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi);
253 plugin_gen_mem_callbacks_i32(val, copy_addr, addr, orig_oi,
254 QEMU_PLUGIN_MEM_R);
255
256 if ((orig_memop ^ memop) & MO_BSWAP) {
257 switch (orig_memop & MO_SIZE) {
258 case MO_16:
259 tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
260 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
261 : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
262 break;
263 case MO_32:
264 tcg_gen_bswap32_i32(val, val);
265 break;
266 default:
267 g_assert_not_reached();
268 }
269 }
270 }
271
tcg_gen_qemu_ld_i32_chk(TCGv_i32 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)272 void tcg_gen_qemu_ld_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx,
273 MemOp memop, TCGType addr_type)
274 {
275 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
276 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
277 tcg_gen_qemu_ld_i32_int(val, addr, idx, memop);
278 }
279
tcg_gen_qemu_st_i32_int(TCGv_i32 val,TCGTemp * addr,TCGArg idx,MemOp memop)280 static void tcg_gen_qemu_st_i32_int(TCGv_i32 val, TCGTemp *addr,
281 TCGArg idx, MemOp memop)
282 {
283 TCGv_i32 swap = NULL;
284 MemOpIdx orig_oi, oi;
285
286 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
287 memop = tcg_canonicalize_memop(memop, 0, 1);
288 orig_oi = oi = make_memop_idx(memop, idx);
289
290 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
291 swap = tcg_temp_ebb_new_i32();
292 switch (memop & MO_SIZE) {
293 case MO_16:
294 tcg_gen_bswap16_i32(swap, val, 0);
295 break;
296 case MO_32:
297 tcg_gen_bswap32_i32(swap, val);
298 break;
299 default:
300 g_assert_not_reached();
301 }
302 val = swap;
303 memop &= ~MO_BSWAP;
304 oi = make_memop_idx(memop, idx);
305 }
306
307 gen_ldst1(INDEX_op_qemu_st, TCG_TYPE_I32, tcgv_i32_temp(val), addr, oi);
308 plugin_gen_mem_callbacks_i32(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W);
309
310 if (swap) {
311 tcg_temp_free_i32(swap);
312 }
313 }
314
tcg_gen_qemu_st_i32_chk(TCGv_i32 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)315 void tcg_gen_qemu_st_i32_chk(TCGv_i32 val, TCGTemp *addr, TCGArg idx,
316 MemOp memop, TCGType addr_type)
317 {
318 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
319 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
320 tcg_gen_qemu_st_i32_int(val, addr, idx, memop);
321 }
322
tcg_gen_qemu_ld_i64_int(TCGv_i64 val,TCGTemp * addr,TCGArg idx,MemOp memop)323 static void tcg_gen_qemu_ld_i64_int(TCGv_i64 val, TCGTemp *addr,
324 TCGArg idx, MemOp memop)
325 {
326 MemOp orig_memop;
327 MemOpIdx orig_oi, oi;
328 TCGv_i64 copy_addr;
329
330 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
331 tcg_gen_qemu_ld_i32_int(TCGV_LOW(val), addr, idx, memop);
332 if (memop & MO_SIGN) {
333 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
334 } else {
335 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
336 }
337 return;
338 }
339
340 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
341 orig_memop = memop = tcg_canonicalize_memop(memop, 1, 0);
342 orig_oi = oi = make_memop_idx(memop, idx);
343
344 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
345 memop &= ~MO_BSWAP;
346 /* The bswap primitive benefits from zero-extended input. */
347 if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
348 memop &= ~MO_SIGN;
349 }
350 oi = make_memop_idx(memop, idx);
351 }
352
353 copy_addr = plugin_maybe_preserve_addr(addr);
354 gen_ld_i64(val, addr, oi);
355 plugin_gen_mem_callbacks_i64(val, copy_addr, addr, orig_oi,
356 QEMU_PLUGIN_MEM_R);
357
358 if ((orig_memop ^ memop) & MO_BSWAP) {
359 int flags = (orig_memop & MO_SIGN
360 ? TCG_BSWAP_IZ | TCG_BSWAP_OS
361 : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
362 switch (orig_memop & MO_SIZE) {
363 case MO_16:
364 tcg_gen_bswap16_i64(val, val, flags);
365 break;
366 case MO_32:
367 tcg_gen_bswap32_i64(val, val, flags);
368 break;
369 case MO_64:
370 tcg_gen_bswap64_i64(val, val);
371 break;
372 default:
373 g_assert_not_reached();
374 }
375 }
376 }
377
tcg_gen_qemu_ld_i64_chk(TCGv_i64 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)378 void tcg_gen_qemu_ld_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx,
379 MemOp memop, TCGType addr_type)
380 {
381 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
382 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
383 tcg_gen_qemu_ld_i64_int(val, addr, idx, memop);
384 }
385
tcg_gen_qemu_st_i64_int(TCGv_i64 val,TCGTemp * addr,TCGArg idx,MemOp memop)386 static void tcg_gen_qemu_st_i64_int(TCGv_i64 val, TCGTemp *addr,
387 TCGArg idx, MemOp memop)
388 {
389 TCGv_i64 swap = NULL;
390 MemOpIdx orig_oi, oi;
391
392 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
393 tcg_gen_qemu_st_i32_int(TCGV_LOW(val), addr, idx, memop);
394 return;
395 }
396
397 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
398 memop = tcg_canonicalize_memop(memop, 1, 1);
399 orig_oi = oi = make_memop_idx(memop, idx);
400
401 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
402 swap = tcg_temp_ebb_new_i64();
403 switch (memop & MO_SIZE) {
404 case MO_16:
405 tcg_gen_bswap16_i64(swap, val, 0);
406 break;
407 case MO_32:
408 tcg_gen_bswap32_i64(swap, val, 0);
409 break;
410 case MO_64:
411 tcg_gen_bswap64_i64(swap, val);
412 break;
413 default:
414 g_assert_not_reached();
415 }
416 val = swap;
417 memop &= ~MO_BSWAP;
418 oi = make_memop_idx(memop, idx);
419 }
420
421 gen_st_i64(val, addr, oi);
422 plugin_gen_mem_callbacks_i64(val, NULL, addr, orig_oi, QEMU_PLUGIN_MEM_W);
423
424 if (swap) {
425 tcg_temp_free_i64(swap);
426 }
427 }
428
tcg_gen_qemu_st_i64_chk(TCGv_i64 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)429 void tcg_gen_qemu_st_i64_chk(TCGv_i64 val, TCGTemp *addr, TCGArg idx,
430 MemOp memop, TCGType addr_type)
431 {
432 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
433 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
434 tcg_gen_qemu_st_i64_int(val, addr, idx, memop);
435 }
436
437 /*
438 * Return true if @mop, without knowledge of the pointer alignment,
439 * does not require 16-byte atomicity, and it would be adventagous
440 * to avoid a call to a helper function.
441 */
use_two_i64_for_i128(MemOp mop)442 static bool use_two_i64_for_i128(MemOp mop)
443 {
444 /* Two softmmu tlb lookups is larger than one function call. */
445 if (tcg_use_softmmu) {
446 return false;
447 }
448
449 /*
450 * For user-only, two 64-bit operations may well be smaller than a call.
451 * Determine if that would be legal for the requested atomicity.
452 */
453 switch (mop & MO_ATOM_MASK) {
454 case MO_ATOM_NONE:
455 case MO_ATOM_IFALIGN_PAIR:
456 return true;
457 case MO_ATOM_IFALIGN:
458 case MO_ATOM_SUBALIGN:
459 case MO_ATOM_WITHIN16:
460 case MO_ATOM_WITHIN16_PAIR:
461 return false;
462 default:
463 g_assert_not_reached();
464 }
465 }
466
canonicalize_memop_i128_as_i64(MemOp ret[2],MemOp orig)467 static void canonicalize_memop_i128_as_i64(MemOp ret[2], MemOp orig)
468 {
469 MemOp mop_1 = orig, mop_2;
470
471 /* Reduce the size to 64-bit. */
472 mop_1 = (mop_1 & ~MO_SIZE) | MO_64;
473
474 /* Retain the alignment constraints of the original. */
475 switch (orig & MO_AMASK) {
476 case MO_UNALN:
477 case MO_ALIGN_2:
478 case MO_ALIGN_4:
479 mop_2 = mop_1;
480 break;
481 case MO_ALIGN_8:
482 /* Prefer MO_ALIGN+MO_64 to MO_ALIGN_8+MO_64. */
483 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
484 mop_2 = mop_1;
485 break;
486 case MO_ALIGN:
487 /* Second has 8-byte alignment; first has 16-byte alignment. */
488 mop_2 = mop_1;
489 mop_1 = (mop_1 & ~MO_AMASK) | MO_ALIGN_16;
490 break;
491 case MO_ALIGN_16:
492 case MO_ALIGN_32:
493 case MO_ALIGN_64:
494 /* Second has 8-byte alignment; first retains original. */
495 mop_2 = (mop_1 & ~MO_AMASK) | MO_ALIGN;
496 break;
497 default:
498 g_assert_not_reached();
499 }
500
501 /* Use a memory ordering implemented by the host. */
502 if ((orig & MO_BSWAP) && !tcg_target_has_memory_bswap(mop_1)) {
503 mop_1 &= ~MO_BSWAP;
504 mop_2 &= ~MO_BSWAP;
505 }
506
507 ret[0] = mop_1;
508 ret[1] = mop_2;
509 }
510
maybe_extend_addr64(TCGTemp * addr)511 static TCGv_i64 maybe_extend_addr64(TCGTemp *addr)
512 {
513 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
514 TCGv_i64 a64 = tcg_temp_ebb_new_i64();
515 tcg_gen_extu_i32_i64(a64, temp_tcgv_i32(addr));
516 return a64;
517 }
518 return temp_tcgv_i64(addr);
519 }
520
maybe_free_addr64(TCGv_i64 a64)521 static void maybe_free_addr64(TCGv_i64 a64)
522 {
523 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
524 tcg_temp_free_i64(a64);
525 }
526 }
527
tcg_gen_qemu_ld_i128_int(TCGv_i128 val,TCGTemp * addr,TCGArg idx,MemOp memop)528 static void tcg_gen_qemu_ld_i128_int(TCGv_i128 val, TCGTemp *addr,
529 TCGArg idx, MemOp memop)
530 {
531 MemOpIdx orig_oi;
532 TCGv_i64 ext_addr = NULL;
533
534 check_max_alignment(memop_alignment_bits(memop));
535 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
536
537 /* In serial mode, reduce atomicity. */
538 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
539 memop &= ~MO_ATOM_MASK;
540 memop |= MO_ATOM_NONE;
541 }
542 orig_oi = make_memop_idx(memop, idx);
543
544 /* TODO: For now, force 32-bit hosts to use the helper. */
545 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) {
546 TCGv_i64 lo, hi;
547 bool need_bswap = false;
548 MemOpIdx oi = orig_oi;
549
550 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
551 lo = TCGV128_HIGH(val);
552 hi = TCGV128_LOW(val);
553 oi = make_memop_idx(memop & ~MO_BSWAP, idx);
554 need_bswap = true;
555 } else {
556 lo = TCGV128_LOW(val);
557 hi = TCGV128_HIGH(val);
558 }
559
560 gen_ldst2(INDEX_op_qemu_ld2, TCG_TYPE_I128, tcgv_i64_temp(lo),
561 tcgv_i64_temp(hi), addr, oi);
562
563 if (need_bswap) {
564 tcg_gen_bswap64_i64(lo, lo);
565 tcg_gen_bswap64_i64(hi, hi);
566 }
567 } else if (use_two_i64_for_i128(memop)) {
568 MemOp mop[2];
569 TCGTemp *addr_p8;
570 TCGv_i64 x, y;
571 bool need_bswap;
572
573 canonicalize_memop_i128_as_i64(mop, memop);
574 need_bswap = (mop[0] ^ memop) & MO_BSWAP;
575
576 /*
577 * Since there are no global TCGv_i128, there is no visible state
578 * changed if the second load faults. Load directly into the two
579 * subwords.
580 */
581 if ((memop & MO_BSWAP) == MO_LE) {
582 x = TCGV128_LOW(val);
583 y = TCGV128_HIGH(val);
584 } else {
585 x = TCGV128_HIGH(val);
586 y = TCGV128_LOW(val);
587 }
588
589 gen_ld_i64(x, addr, make_memop_idx(mop[0], idx));
590
591 if (need_bswap) {
592 tcg_gen_bswap64_i64(x, x);
593 }
594
595 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
596 TCGv_i32 t = tcg_temp_ebb_new_i32();
597 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8);
598 addr_p8 = tcgv_i32_temp(t);
599 } else {
600 TCGv_i64 t = tcg_temp_ebb_new_i64();
601 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8);
602 addr_p8 = tcgv_i64_temp(t);
603 }
604
605 gen_ld_i64(y, addr_p8, make_memop_idx(mop[1], idx));
606 tcg_temp_free_internal(addr_p8);
607
608 if (need_bswap) {
609 tcg_gen_bswap64_i64(y, y);
610 }
611 } else {
612 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
613 ext_addr = tcg_temp_ebb_new_i64();
614 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr));
615 addr = tcgv_i64_temp(ext_addr);
616 }
617 gen_helper_ld_i128(val, tcg_env, temp_tcgv_i64(addr),
618 tcg_constant_i32(orig_oi));
619 }
620
621 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi,
622 QEMU_PLUGIN_MEM_R);
623 }
624
tcg_gen_qemu_ld_i128_chk(TCGv_i128 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)625 void tcg_gen_qemu_ld_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx,
626 MemOp memop, TCGType addr_type)
627 {
628 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
629 tcg_debug_assert((memop & MO_SIZE) == MO_128);
630 tcg_debug_assert((memop & MO_SIGN) == 0);
631 tcg_gen_qemu_ld_i128_int(val, addr, idx, memop);
632 }
633
tcg_gen_qemu_st_i128_int(TCGv_i128 val,TCGTemp * addr,TCGArg idx,MemOp memop)634 static void tcg_gen_qemu_st_i128_int(TCGv_i128 val, TCGTemp *addr,
635 TCGArg idx, MemOp memop)
636 {
637 MemOpIdx orig_oi;
638 TCGv_i64 ext_addr = NULL;
639
640 check_max_alignment(memop_alignment_bits(memop));
641 tcg_gen_req_mo(TCG_MO_ST_LD | TCG_MO_ST_ST);
642
643 /* In serial mode, reduce atomicity. */
644 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
645 memop &= ~MO_ATOM_MASK;
646 memop |= MO_ATOM_NONE;
647 }
648 orig_oi = make_memop_idx(memop, idx);
649
650 /* TODO: For now, force 32-bit hosts to use the helper. */
651
652 if (TCG_TARGET_HAS_qemu_ldst_i128 && TCG_TARGET_REG_BITS == 64) {
653 TCGv_i64 lo, hi;
654 MemOpIdx oi = orig_oi;
655 bool need_bswap = false;
656
657 if ((memop & MO_BSWAP) && !tcg_target_has_memory_bswap(memop)) {
658 lo = tcg_temp_ebb_new_i64();
659 hi = tcg_temp_ebb_new_i64();
660 tcg_gen_bswap64_i64(lo, TCGV128_HIGH(val));
661 tcg_gen_bswap64_i64(hi, TCGV128_LOW(val));
662 oi = make_memop_idx(memop & ~MO_BSWAP, idx);
663 need_bswap = true;
664 } else {
665 lo = TCGV128_LOW(val);
666 hi = TCGV128_HIGH(val);
667 }
668
669 gen_ldst2(INDEX_op_qemu_st2, TCG_TYPE_I128,
670 tcgv_i64_temp(lo), tcgv_i64_temp(hi), addr, oi);
671
672 if (need_bswap) {
673 tcg_temp_free_i64(lo);
674 tcg_temp_free_i64(hi);
675 }
676 } else if (use_two_i64_for_i128(memop)) {
677 MemOp mop[2];
678 TCGTemp *addr_p8;
679 TCGv_i64 x, y, b = NULL;
680
681 canonicalize_memop_i128_as_i64(mop, memop);
682
683 if ((memop & MO_BSWAP) == MO_LE) {
684 x = TCGV128_LOW(val);
685 y = TCGV128_HIGH(val);
686 } else {
687 x = TCGV128_HIGH(val);
688 y = TCGV128_LOW(val);
689 }
690
691 if ((mop[0] ^ memop) & MO_BSWAP) {
692 b = tcg_temp_ebb_new_i64();
693 tcg_gen_bswap64_i64(b, x);
694 x = b;
695 }
696
697 gen_st_i64(x, addr, make_memop_idx(mop[0], idx));
698
699 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
700 TCGv_i32 t = tcg_temp_ebb_new_i32();
701 tcg_gen_addi_i32(t, temp_tcgv_i32(addr), 8);
702 addr_p8 = tcgv_i32_temp(t);
703 } else {
704 TCGv_i64 t = tcg_temp_ebb_new_i64();
705 tcg_gen_addi_i64(t, temp_tcgv_i64(addr), 8);
706 addr_p8 = tcgv_i64_temp(t);
707 }
708
709 if (b) {
710 tcg_gen_bswap64_i64(b, y);
711 gen_st_i64(b, addr_p8, make_memop_idx(mop[1], idx));
712 tcg_temp_free_i64(b);
713 } else {
714 gen_st_i64(y, addr_p8, make_memop_idx(mop[1], idx));
715 }
716 tcg_temp_free_internal(addr_p8);
717 } else {
718 if (tcg_ctx->addr_type == TCG_TYPE_I32) {
719 ext_addr = tcg_temp_ebb_new_i64();
720 tcg_gen_extu_i32_i64(ext_addr, temp_tcgv_i32(addr));
721 addr = tcgv_i64_temp(ext_addr);
722 }
723 gen_helper_st_i128(tcg_env, temp_tcgv_i64(addr), val,
724 tcg_constant_i32(orig_oi));
725 }
726
727 plugin_gen_mem_callbacks_i128(val, ext_addr, addr, orig_oi,
728 QEMU_PLUGIN_MEM_W);
729 }
730
tcg_gen_qemu_st_i128_chk(TCGv_i128 val,TCGTemp * addr,TCGArg idx,MemOp memop,TCGType addr_type)731 void tcg_gen_qemu_st_i128_chk(TCGv_i128 val, TCGTemp *addr, TCGArg idx,
732 MemOp memop, TCGType addr_type)
733 {
734 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
735 tcg_debug_assert((memop & MO_SIZE) == MO_128);
736 tcg_debug_assert((memop & MO_SIGN) == 0);
737 tcg_gen_qemu_st_i128_int(val, addr, idx, memop);
738 }
739
tcg_gen_ext_i32(TCGv_i32 ret,TCGv_i32 val,MemOp opc)740 void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
741 {
742 switch (opc & MO_SSIZE) {
743 case MO_SB:
744 tcg_gen_ext8s_i32(ret, val);
745 break;
746 case MO_UB:
747 tcg_gen_ext8u_i32(ret, val);
748 break;
749 case MO_SW:
750 tcg_gen_ext16s_i32(ret, val);
751 break;
752 case MO_UW:
753 tcg_gen_ext16u_i32(ret, val);
754 break;
755 case MO_UL:
756 case MO_SL:
757 tcg_gen_mov_i32(ret, val);
758 break;
759 default:
760 g_assert_not_reached();
761 }
762 }
763
tcg_gen_ext_i64(TCGv_i64 ret,TCGv_i64 val,MemOp opc)764 void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
765 {
766 switch (opc & MO_SSIZE) {
767 case MO_SB:
768 tcg_gen_ext8s_i64(ret, val);
769 break;
770 case MO_UB:
771 tcg_gen_ext8u_i64(ret, val);
772 break;
773 case MO_SW:
774 tcg_gen_ext16s_i64(ret, val);
775 break;
776 case MO_UW:
777 tcg_gen_ext16u_i64(ret, val);
778 break;
779 case MO_SL:
780 tcg_gen_ext32s_i64(ret, val);
781 break;
782 case MO_UL:
783 tcg_gen_ext32u_i64(ret, val);
784 break;
785 case MO_UQ:
786 case MO_SQ:
787 tcg_gen_mov_i64(ret, val);
788 break;
789 default:
790 g_assert_not_reached();
791 }
792 }
793
794 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv_i64,
795 TCGv_i32, TCGv_i32, TCGv_i32);
796 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv_i64,
797 TCGv_i64, TCGv_i64, TCGv_i32);
798 typedef void (*gen_atomic_cx_i128)(TCGv_i128, TCGv_env, TCGv_i64,
799 TCGv_i128, TCGv_i128, TCGv_i32);
800 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv_i64,
801 TCGv_i32, TCGv_i32);
802 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv_i64,
803 TCGv_i64, TCGv_i32);
804
805 #ifdef CONFIG_ATOMIC64
806 # define WITH_ATOMIC64(X) X,
807 #else
808 # define WITH_ATOMIC64(X)
809 #endif
810 #if HAVE_CMPXCHG128
811 # define WITH_ATOMIC128(X) X,
812 #else
813 # define WITH_ATOMIC128(X)
814 #endif
815
816 static void * const table_cmpxchg[(MO_SIZE | MO_BSWAP) + 1] = {
817 [MO_8] = gen_helper_atomic_cmpxchgb,
818 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
819 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
820 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
821 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
822 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
823 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
824 WITH_ATOMIC128([MO_128 | MO_LE] = gen_helper_atomic_cmpxchgo_le)
825 WITH_ATOMIC128([MO_128 | MO_BE] = gen_helper_atomic_cmpxchgo_be)
826 };
827
tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv,TCGTemp * addr,TCGv_i32 cmpv,TCGv_i32 newv,TCGArg idx,MemOp memop)828 static void tcg_gen_nonatomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
829 TCGv_i32 cmpv, TCGv_i32 newv,
830 TCGArg idx, MemOp memop)
831 {
832 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
833 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
834
835 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
836
837 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop & ~MO_SIGN);
838 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
839 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
840 tcg_temp_free_i32(t2);
841
842 if (memop & MO_SIGN) {
843 tcg_gen_ext_i32(retv, t1, memop);
844 } else {
845 tcg_gen_mov_i32(retv, t1);
846 }
847 tcg_temp_free_i32(t1);
848 }
849
tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv,TCGTemp * addr,TCGv_i32 cmpv,TCGv_i32 newv,TCGArg idx,MemOp memop,TCGType addr_type)850 void tcg_gen_nonatomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
851 TCGv_i32 cmpv, TCGv_i32 newv,
852 TCGArg idx, MemOp memop,
853 TCGType addr_type)
854 {
855 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
856 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
857 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
858 }
859
tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv,TCGTemp * addr,TCGv_i32 cmpv,TCGv_i32 newv,TCGArg idx,MemOp memop)860 static void tcg_gen_atomic_cmpxchg_i32_int(TCGv_i32 retv, TCGTemp *addr,
861 TCGv_i32 cmpv, TCGv_i32 newv,
862 TCGArg idx, MemOp memop)
863 {
864 gen_atomic_cx_i32 gen;
865 TCGv_i64 a64;
866 MemOpIdx oi;
867
868 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
869 tcg_gen_nonatomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
870 return;
871 }
872
873 memop = tcg_canonicalize_memop(memop, 0, 0);
874 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
875 tcg_debug_assert(gen != NULL);
876
877 oi = make_memop_idx(memop & ~MO_SIGN, idx);
878 a64 = maybe_extend_addr64(addr);
879 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
880 maybe_free_addr64(a64);
881
882 if (memop & MO_SIGN) {
883 tcg_gen_ext_i32(retv, retv, memop);
884 }
885 }
886
tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv,TCGTemp * addr,TCGv_i32 cmpv,TCGv_i32 newv,TCGArg idx,MemOp memop,TCGType addr_type)887 void tcg_gen_atomic_cmpxchg_i32_chk(TCGv_i32 retv, TCGTemp *addr,
888 TCGv_i32 cmpv, TCGv_i32 newv,
889 TCGArg idx, MemOp memop,
890 TCGType addr_type)
891 {
892 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
893 tcg_debug_assert((memop & MO_SIZE) <= MO_32);
894 tcg_gen_atomic_cmpxchg_i32_int(retv, addr, cmpv, newv, idx, memop);
895 }
896
tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv,TCGTemp * addr,TCGv_i64 cmpv,TCGv_i64 newv,TCGArg idx,MemOp memop)897 static void tcg_gen_nonatomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
898 TCGv_i64 cmpv, TCGv_i64 newv,
899 TCGArg idx, MemOp memop)
900 {
901 TCGv_i64 t1, t2;
902
903 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
904 tcg_gen_nonatomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
905 TCGV_LOW(newv), idx, memop);
906 if (memop & MO_SIGN) {
907 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
908 } else {
909 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
910 }
911 return;
912 }
913
914 t1 = tcg_temp_ebb_new_i64();
915 t2 = tcg_temp_ebb_new_i64();
916
917 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
918
919 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop & ~MO_SIGN);
920 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
921 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
922 tcg_temp_free_i64(t2);
923
924 if (memop & MO_SIGN) {
925 tcg_gen_ext_i64(retv, t1, memop);
926 } else {
927 tcg_gen_mov_i64(retv, t1);
928 }
929 tcg_temp_free_i64(t1);
930 }
931
tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv,TCGTemp * addr,TCGv_i64 cmpv,TCGv_i64 newv,TCGArg idx,MemOp memop,TCGType addr_type)932 void tcg_gen_nonatomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
933 TCGv_i64 cmpv, TCGv_i64 newv,
934 TCGArg idx, MemOp memop,
935 TCGType addr_type)
936 {
937 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
938 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
939 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
940 }
941
tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv,TCGTemp * addr,TCGv_i64 cmpv,TCGv_i64 newv,TCGArg idx,MemOp memop)942 static void tcg_gen_atomic_cmpxchg_i64_int(TCGv_i64 retv, TCGTemp *addr,
943 TCGv_i64 cmpv, TCGv_i64 newv,
944 TCGArg idx, MemOp memop)
945 {
946 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
947 tcg_gen_nonatomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
948 return;
949 }
950
951 if ((memop & MO_SIZE) == MO_64) {
952 gen_atomic_cx_i64 gen;
953
954 memop = tcg_canonicalize_memop(memop, 1, 0);
955 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
956 if (gen) {
957 MemOpIdx oi = make_memop_idx(memop, idx);
958 TCGv_i64 a64 = maybe_extend_addr64(addr);
959 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
960 maybe_free_addr64(a64);
961 return;
962 }
963
964 gen_helper_exit_atomic(tcg_env);
965
966 /*
967 * Produce a result for a well-formed opcode stream. This satisfies
968 * liveness for set before used, which happens before this dead code
969 * is removed.
970 */
971 tcg_gen_movi_i64(retv, 0);
972 return;
973 }
974
975 if (TCG_TARGET_REG_BITS == 32) {
976 tcg_gen_atomic_cmpxchg_i32_int(TCGV_LOW(retv), addr, TCGV_LOW(cmpv),
977 TCGV_LOW(newv), idx, memop);
978 if (memop & MO_SIGN) {
979 tcg_gen_sari_i32(TCGV_HIGH(retv), TCGV_LOW(retv), 31);
980 } else {
981 tcg_gen_movi_i32(TCGV_HIGH(retv), 0);
982 }
983 } else {
984 TCGv_i32 c32 = tcg_temp_ebb_new_i32();
985 TCGv_i32 n32 = tcg_temp_ebb_new_i32();
986 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
987
988 tcg_gen_extrl_i64_i32(c32, cmpv);
989 tcg_gen_extrl_i64_i32(n32, newv);
990 tcg_gen_atomic_cmpxchg_i32_int(r32, addr, c32, n32,
991 idx, memop & ~MO_SIGN);
992 tcg_temp_free_i32(c32);
993 tcg_temp_free_i32(n32);
994
995 tcg_gen_extu_i32_i64(retv, r32);
996 tcg_temp_free_i32(r32);
997
998 if (memop & MO_SIGN) {
999 tcg_gen_ext_i64(retv, retv, memop);
1000 }
1001 }
1002 }
1003
tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv,TCGTemp * addr,TCGv_i64 cmpv,TCGv_i64 newv,TCGArg idx,MemOp memop,TCGType addr_type)1004 void tcg_gen_atomic_cmpxchg_i64_chk(TCGv_i64 retv, TCGTemp *addr,
1005 TCGv_i64 cmpv, TCGv_i64 newv,
1006 TCGArg idx, MemOp memop, TCGType addr_type)
1007 {
1008 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1009 tcg_debug_assert((memop & MO_SIZE) <= MO_64);
1010 tcg_gen_atomic_cmpxchg_i64_int(retv, addr, cmpv, newv, idx, memop);
1011 }
1012
tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv,TCGTemp * addr,TCGv_i128 cmpv,TCGv_i128 newv,TCGArg idx,MemOp memop)1013 static void tcg_gen_nonatomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
1014 TCGv_i128 cmpv, TCGv_i128 newv,
1015 TCGArg idx, MemOp memop)
1016 {
1017 if (TCG_TARGET_REG_BITS == 32) {
1018 /* Inline expansion below is simply too large for 32-bit hosts. */
1019 MemOpIdx oi = make_memop_idx(memop, idx);
1020 TCGv_i64 a64 = maybe_extend_addr64(addr);
1021
1022 gen_helper_nonatomic_cmpxchgo(retv, tcg_env, a64, cmpv, newv,
1023 tcg_constant_i32(oi));
1024 maybe_free_addr64(a64);
1025 } else {
1026 TCGv_i128 oldv = tcg_temp_ebb_new_i128();
1027 TCGv_i128 tmpv = tcg_temp_ebb_new_i128();
1028 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1029 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1030 TCGv_i64 z = tcg_constant_i64(0);
1031
1032 tcg_gen_qemu_ld_i128_int(oldv, addr, idx, memop);
1033
1034 /* Compare i128 */
1035 tcg_gen_xor_i64(t0, TCGV128_LOW(oldv), TCGV128_LOW(cmpv));
1036 tcg_gen_xor_i64(t1, TCGV128_HIGH(oldv), TCGV128_HIGH(cmpv));
1037 tcg_gen_or_i64(t0, t0, t1);
1038
1039 /* tmpv = equal ? newv : oldv */
1040 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_LOW(tmpv), t0, z,
1041 TCGV128_LOW(newv), TCGV128_LOW(oldv));
1042 tcg_gen_movcond_i64(TCG_COND_EQ, TCGV128_HIGH(tmpv), t0, z,
1043 TCGV128_HIGH(newv), TCGV128_HIGH(oldv));
1044
1045 /* Unconditional writeback. */
1046 tcg_gen_qemu_st_i128_int(tmpv, addr, idx, memop);
1047 tcg_gen_mov_i128(retv, oldv);
1048
1049 tcg_temp_free_i64(t0);
1050 tcg_temp_free_i64(t1);
1051 tcg_temp_free_i128(tmpv);
1052 tcg_temp_free_i128(oldv);
1053 }
1054 }
1055
tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv,TCGTemp * addr,TCGv_i128 cmpv,TCGv_i128 newv,TCGArg idx,MemOp memop,TCGType addr_type)1056 void tcg_gen_nonatomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
1057 TCGv_i128 cmpv, TCGv_i128 newv,
1058 TCGArg idx, MemOp memop,
1059 TCGType addr_type)
1060 {
1061 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1062 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
1063 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1064 }
1065
tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv,TCGTemp * addr,TCGv_i128 cmpv,TCGv_i128 newv,TCGArg idx,MemOp memop)1066 static void tcg_gen_atomic_cmpxchg_i128_int(TCGv_i128 retv, TCGTemp *addr,
1067 TCGv_i128 cmpv, TCGv_i128 newv,
1068 TCGArg idx, MemOp memop)
1069 {
1070 gen_atomic_cx_i128 gen;
1071
1072 if (!(tcg_ctx->gen_tb->cflags & CF_PARALLEL)) {
1073 tcg_gen_nonatomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1074 return;
1075 }
1076
1077 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
1078 if (gen) {
1079 MemOpIdx oi = make_memop_idx(memop, idx);
1080 TCGv_i64 a64 = maybe_extend_addr64(addr);
1081 gen(retv, tcg_env, a64, cmpv, newv, tcg_constant_i32(oi));
1082 maybe_free_addr64(a64);
1083 return;
1084 }
1085
1086 gen_helper_exit_atomic(tcg_env);
1087
1088 /*
1089 * Produce a result for a well-formed opcode stream. This satisfies
1090 * liveness for set before used, which happens before this dead code
1091 * is removed.
1092 */
1093 tcg_gen_movi_i64(TCGV128_LOW(retv), 0);
1094 tcg_gen_movi_i64(TCGV128_HIGH(retv), 0);
1095 }
1096
tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv,TCGTemp * addr,TCGv_i128 cmpv,TCGv_i128 newv,TCGArg idx,MemOp memop,TCGType addr_type)1097 void tcg_gen_atomic_cmpxchg_i128_chk(TCGv_i128 retv, TCGTemp *addr,
1098 TCGv_i128 cmpv, TCGv_i128 newv,
1099 TCGArg idx, MemOp memop,
1100 TCGType addr_type)
1101 {
1102 tcg_debug_assert(addr_type == tcg_ctx->addr_type);
1103 tcg_debug_assert((memop & (MO_SIZE | MO_SIGN)) == MO_128);
1104 tcg_gen_atomic_cmpxchg_i128_int(retv, addr, cmpv, newv, idx, memop);
1105 }
1106
do_nonatomic_op_i32(TCGv_i32 ret,TCGTemp * addr,TCGv_i32 val,TCGArg idx,MemOp memop,bool new_val,void (* gen)(TCGv_i32,TCGv_i32,TCGv_i32))1107 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
1108 TCGArg idx, MemOp memop, bool new_val,
1109 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
1110 {
1111 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1112 TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1113
1114 memop = tcg_canonicalize_memop(memop, 0, 0);
1115
1116 tcg_gen_qemu_ld_i32_int(t1, addr, idx, memop);
1117 tcg_gen_ext_i32(t2, val, memop);
1118 gen(t2, t1, t2);
1119 tcg_gen_qemu_st_i32_int(t2, addr, idx, memop);
1120
1121 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
1122 tcg_temp_free_i32(t1);
1123 tcg_temp_free_i32(t2);
1124 }
1125
do_atomic_op_i32(TCGv_i32 ret,TCGTemp * addr,TCGv_i32 val,TCGArg idx,MemOp memop,void * const table[])1126 static void do_atomic_op_i32(TCGv_i32 ret, TCGTemp *addr, TCGv_i32 val,
1127 TCGArg idx, MemOp memop, void * const table[])
1128 {
1129 gen_atomic_op_i32 gen;
1130 TCGv_i64 a64;
1131 MemOpIdx oi;
1132
1133 memop = tcg_canonicalize_memop(memop, 0, 0);
1134
1135 gen = table[memop & (MO_SIZE | MO_BSWAP)];
1136 tcg_debug_assert(gen != NULL);
1137
1138 oi = make_memop_idx(memop & ~MO_SIGN, idx);
1139 a64 = maybe_extend_addr64(addr);
1140 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi));
1141 maybe_free_addr64(a64);
1142
1143 if (memop & MO_SIGN) {
1144 tcg_gen_ext_i32(ret, ret, memop);
1145 }
1146 }
1147
do_nonatomic_op_i64(TCGv_i64 ret,TCGTemp * addr,TCGv_i64 val,TCGArg idx,MemOp memop,bool new_val,void (* gen)(TCGv_i64,TCGv_i64,TCGv_i64))1148 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
1149 TCGArg idx, MemOp memop, bool new_val,
1150 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
1151 {
1152 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1153 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
1154
1155 memop = tcg_canonicalize_memop(memop, 1, 0);
1156
1157 tcg_gen_qemu_ld_i64_int(t1, addr, idx, memop);
1158 tcg_gen_ext_i64(t2, val, memop);
1159 gen(t2, t1, t2);
1160 tcg_gen_qemu_st_i64_int(t2, addr, idx, memop);
1161
1162 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
1163 tcg_temp_free_i64(t1);
1164 tcg_temp_free_i64(t2);
1165 }
1166
do_atomic_op_i64(TCGv_i64 ret,TCGTemp * addr,TCGv_i64 val,TCGArg idx,MemOp memop,void * const table[])1167 static void do_atomic_op_i64(TCGv_i64 ret, TCGTemp *addr, TCGv_i64 val,
1168 TCGArg idx, MemOp memop, void * const table[])
1169 {
1170 memop = tcg_canonicalize_memop(memop, 1, 0);
1171
1172 if ((memop & MO_SIZE) == MO_64) {
1173 gen_atomic_op_i64 gen = table[memop & (MO_SIZE | MO_BSWAP)];
1174
1175 if (gen) {
1176 MemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
1177 TCGv_i64 a64 = maybe_extend_addr64(addr);
1178 gen(ret, tcg_env, a64, val, tcg_constant_i32(oi));
1179 maybe_free_addr64(a64);
1180 return;
1181 }
1182
1183 gen_helper_exit_atomic(tcg_env);
1184 /* Produce a result, so that we have a well-formed opcode stream
1185 with respect to uses of the result in the (dead) code following. */
1186 tcg_gen_movi_i64(ret, 0);
1187 } else {
1188 TCGv_i32 v32 = tcg_temp_ebb_new_i32();
1189 TCGv_i32 r32 = tcg_temp_ebb_new_i32();
1190
1191 tcg_gen_extrl_i64_i32(v32, val);
1192 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
1193 tcg_temp_free_i32(v32);
1194
1195 tcg_gen_extu_i32_i64(ret, r32);
1196 tcg_temp_free_i32(r32);
1197
1198 if (memop & MO_SIGN) {
1199 tcg_gen_ext_i64(ret, ret, memop);
1200 }
1201 }
1202 }
1203
1204 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
1205 static void * const table_##NAME[(MO_SIZE | MO_BSWAP) + 1] = { \
1206 [MO_8] = gen_helper_atomic_##NAME##b, \
1207 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
1208 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
1209 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
1210 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
1211 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
1212 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
1213 }; \
1214 void tcg_gen_atomic_##NAME##_i32_chk(TCGv_i32 ret, TCGTemp *addr, \
1215 TCGv_i32 val, TCGArg idx, \
1216 MemOp memop, TCGType addr_type) \
1217 { \
1218 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1219 tcg_debug_assert((memop & MO_SIZE) <= MO_32); \
1220 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1221 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
1222 } else { \
1223 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
1224 tcg_gen_##OP##_i32); \
1225 } \
1226 } \
1227 void tcg_gen_atomic_##NAME##_i64_chk(TCGv_i64 ret, TCGTemp *addr, \
1228 TCGv_i64 val, TCGArg idx, \
1229 MemOp memop, TCGType addr_type) \
1230 { \
1231 tcg_debug_assert(addr_type == tcg_ctx->addr_type); \
1232 tcg_debug_assert((memop & MO_SIZE) <= MO_64); \
1233 if (tcg_ctx->gen_tb->cflags & CF_PARALLEL) { \
1234 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
1235 } else { \
1236 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
1237 tcg_gen_##OP##_i64); \
1238 } \
1239 }
1240
1241 GEN_ATOMIC_HELPER(fetch_add, add, 0)
1242 GEN_ATOMIC_HELPER(fetch_and, and, 0)
1243 GEN_ATOMIC_HELPER(fetch_or, or, 0)
1244 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
1245 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
1246 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
1247 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
1248 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
1249
1250 GEN_ATOMIC_HELPER(add_fetch, add, 1)
1251 GEN_ATOMIC_HELPER(and_fetch, and, 1)
1252 GEN_ATOMIC_HELPER(or_fetch, or, 1)
1253 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
1254 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
1255 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
1256 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
1257 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
1258
tcg_gen_mov2_i32(TCGv_i32 r,TCGv_i32 a,TCGv_i32 b)1259 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
1260 {
1261 tcg_gen_mov_i32(r, b);
1262 }
1263
tcg_gen_mov2_i64(TCGv_i64 r,TCGv_i64 a,TCGv_i64 b)1264 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
1265 {
1266 tcg_gen_mov_i64(r, b);
1267 }
1268
1269 GEN_ATOMIC_HELPER(xchg, mov2, 0)
1270
1271 #undef GEN_ATOMIC_HELPER
1272