1 /*
2 * S/390 memory access helper routines
3 *
4 * Copyright (c) 2009 Ulrich Hecht
5 * Copyright (c) 2009 Alexander Graf
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/osdep.h"
22 #include "qemu/log.h"
23 #include "cpu.h"
24 #include "s390x-internal.h"
25 #include "tcg_s390x.h"
26 #include "exec/helper-proto.h"
27 #include "exec/cpu-common.h"
28 #include "exec/cputlb.h"
29 #include "exec/page-protection.h"
30 #include "accel/tcg/cpu-ldst.h"
31 #include "accel/tcg/probe.h"
32 #include "exec/target_page.h"
33 #include "exec/tlb-flags.h"
34 #include "accel/tcg/cpu-ops.h"
35 #include "accel/tcg/helper-retaddr.h"
36 #include "qemu/int128.h"
37 #include "qemu/atomic128.h"
38
39 #if defined(CONFIG_USER_ONLY)
40 #include "user/page-protection.h"
41 #else
42 #include "hw/s390x/storage-keys.h"
43 #include "hw/boards.h"
44 #endif
45
46 #ifdef CONFIG_USER_ONLY
47 # define user_or_likely(X) true
48 #else
49 # define user_or_likely(X) likely(X)
50 #endif
51
52 /*****************************************************************************/
53 /* Softmmu support */
54
55 /* #define DEBUG_HELPER */
56 #ifdef DEBUG_HELPER
57 #define HELPER_LOG(x...) qemu_log(x)
58 #else
59 #define HELPER_LOG(x...)
60 #endif
61
psw_key_valid(CPUS390XState * env,uint8_t psw_key)62 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
63 {
64 uint16_t pkm = env->cregs[3] >> 16;
65
66 if (env->psw.mask & PSW_MASK_PSTATE) {
67 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
68 return pkm & (0x8000 >> psw_key);
69 }
70 return true;
71 }
72
is_destructive_overlap(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t len)73 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest,
74 uint64_t src, uint32_t len)
75 {
76 if (!len || src == dest) {
77 return false;
78 }
79 /* Take care of wrapping at the end of address space. */
80 if (unlikely(wrap_address(env, src + len - 1) < src)) {
81 return dest > src || dest <= wrap_address(env, src + len - 1);
82 }
83 return dest > src && dest <= src + len - 1;
84 }
85
86 /* Trigger a SPECIFICATION exception if an address or a length is not
87 naturally aligned. */
check_alignment(CPUS390XState * env,uint64_t v,int wordsize,uintptr_t ra)88 static inline void check_alignment(CPUS390XState *env, uint64_t v,
89 int wordsize, uintptr_t ra)
90 {
91 if (v % wordsize) {
92 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
93 }
94 }
95
96 /* Load a value from memory according to its size. */
cpu_ldusize_data_ra(CPUS390XState * env,uint64_t addr,int wordsize,uintptr_t ra)97 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
98 int wordsize, uintptr_t ra)
99 {
100 switch (wordsize) {
101 case 1:
102 return cpu_ldub_data_ra(env, addr, ra);
103 case 2:
104 return cpu_lduw_data_ra(env, addr, ra);
105 default:
106 abort();
107 }
108 }
109
110 /* Store a to memory according to its size. */
cpu_stsize_data_ra(CPUS390XState * env,uint64_t addr,uint64_t value,int wordsize,uintptr_t ra)111 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
112 uint64_t value, int wordsize,
113 uintptr_t ra)
114 {
115 switch (wordsize) {
116 case 1:
117 cpu_stb_data_ra(env, addr, value, ra);
118 break;
119 case 2:
120 cpu_stw_data_ra(env, addr, value, ra);
121 break;
122 default:
123 abort();
124 }
125 }
126
127 /* An access covers at most 4096 bytes and therefore at most two pages. */
128 typedef struct S390Access {
129 target_ulong vaddr1;
130 target_ulong vaddr2;
131 void *haddr1;
132 void *haddr2;
133 uint16_t size1;
134 uint16_t size2;
135 /*
136 * If we can't access the host page directly, we'll have to do I/O access
137 * via ld/st helpers. These are internal details, so we store the
138 * mmu idx to do the access here instead of passing it around in the
139 * helpers.
140 */
141 int mmu_idx;
142 } S390Access;
143
144 /*
145 * With nonfault=1, return the PGM_ exception that would have been injected
146 * into the guest; return 0 if no exception was detected.
147 *
148 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec.
149 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr.
150 */
s390_probe_access(CPUArchState * env,target_ulong addr,int size,MMUAccessType access_type,int mmu_idx,bool nonfault,void ** phost,uintptr_t ra)151 static inline int s390_probe_access(CPUArchState *env, target_ulong addr,
152 int size, MMUAccessType access_type,
153 int mmu_idx, bool nonfault,
154 void **phost, uintptr_t ra)
155 {
156 int flags = probe_access_flags(env, addr, size, access_type, mmu_idx,
157 nonfault, phost, ra);
158
159 if (unlikely(flags & TLB_INVALID_MASK)) {
160 #ifdef CONFIG_USER_ONLY
161 /* Address is in TEC in system mode; see s390_cpu_record_sigsegv. */
162 env->__excp_addr = addr & TARGET_PAGE_MASK;
163 return (page_get_flags(addr) & PAGE_VALID
164 ? PGM_PROTECTION : PGM_ADDRESSING);
165 #else
166 return env->tlb_fill_exc;
167 #endif
168 }
169
170 #ifndef CONFIG_USER_ONLY
171 if (unlikely(flags & TLB_WATCHPOINT)) {
172 /* S390 does not presently use transaction attributes. */
173 cpu_check_watchpoint(env_cpu(env), addr, size,
174 MEMTXATTRS_UNSPECIFIED,
175 (access_type == MMU_DATA_STORE
176 ? BP_MEM_WRITE : BP_MEM_READ), ra);
177 }
178 #endif
179
180 return 0;
181 }
182
access_prepare_nf(S390Access * access,CPUS390XState * env,bool nonfault,vaddr vaddr1,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)183 static int access_prepare_nf(S390Access *access, CPUS390XState *env,
184 bool nonfault, vaddr vaddr1, int size,
185 MMUAccessType access_type,
186 int mmu_idx, uintptr_t ra)
187 {
188 int size1, size2, exc;
189
190 assert(size > 0 && size <= 4096);
191
192 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)),
193 size2 = size - size1;
194
195 memset(access, 0, sizeof(*access));
196 access->vaddr1 = vaddr1;
197 access->size1 = size1;
198 access->size2 = size2;
199 access->mmu_idx = mmu_idx;
200
201 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault,
202 &access->haddr1, ra);
203 if (unlikely(exc)) {
204 return exc;
205 }
206 if (unlikely(size2)) {
207 /* The access crosses page boundaries. */
208 vaddr vaddr2 = wrap_address(env, vaddr1 + size1);
209
210 access->vaddr2 = vaddr2;
211 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx,
212 nonfault, &access->haddr2, ra);
213 if (unlikely(exc)) {
214 return exc;
215 }
216 }
217 return 0;
218 }
219
access_prepare(S390Access * ret,CPUS390XState * env,vaddr vaddr,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)220 static inline void access_prepare(S390Access *ret, CPUS390XState *env,
221 vaddr vaddr, int size,
222 MMUAccessType access_type, int mmu_idx,
223 uintptr_t ra)
224 {
225 int exc = access_prepare_nf(ret, env, false, vaddr, size,
226 access_type, mmu_idx, ra);
227 assert(!exc);
228 }
229
230 /* Helper to handle memset on a single page. */
do_access_memset(CPUS390XState * env,vaddr vaddr,char * haddr,uint8_t byte,uint16_t size,int mmu_idx,uintptr_t ra)231 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr,
232 uint8_t byte, uint16_t size, int mmu_idx,
233 uintptr_t ra)
234 {
235 if (user_or_likely(haddr)) {
236 memset(haddr, byte, size);
237 } else {
238 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
239 for (int i = 0; i < size; i++) {
240 cpu_stb_mmu(env, vaddr + i, byte, oi, ra);
241 }
242 }
243 }
244
access_memset(CPUS390XState * env,S390Access * desta,uint8_t byte,uintptr_t ra)245 static void access_memset(CPUS390XState *env, S390Access *desta,
246 uint8_t byte, uintptr_t ra)
247 {
248 set_helper_retaddr(ra);
249 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1,
250 desta->mmu_idx, ra);
251 if (unlikely(desta->size2)) {
252 do_access_memset(env, desta->vaddr2, desta->haddr2, byte,
253 desta->size2, desta->mmu_idx, ra);
254 }
255 clear_helper_retaddr();
256 }
257
access_get_byte(CPUS390XState * env,S390Access * access,int offset,uintptr_t ra)258 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access,
259 int offset, uintptr_t ra)
260 {
261 target_ulong vaddr = access->vaddr1;
262 void *haddr = access->haddr1;
263
264 if (unlikely(offset >= access->size1)) {
265 offset -= access->size1;
266 vaddr = access->vaddr2;
267 haddr = access->haddr2;
268 }
269
270 if (user_or_likely(haddr)) {
271 return ldub_p(haddr + offset);
272 } else {
273 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
274 return cpu_ldb_mmu(env, vaddr + offset, oi, ra);
275 }
276 }
277
access_set_byte(CPUS390XState * env,S390Access * access,int offset,uint8_t byte,uintptr_t ra)278 static void access_set_byte(CPUS390XState *env, S390Access *access,
279 int offset, uint8_t byte, uintptr_t ra)
280 {
281 target_ulong vaddr = access->vaddr1;
282 void *haddr = access->haddr1;
283
284 if (unlikely(offset >= access->size1)) {
285 offset -= access->size1;
286 vaddr = access->vaddr2;
287 haddr = access->haddr2;
288 }
289
290 if (user_or_likely(haddr)) {
291 stb_p(haddr + offset, byte);
292 } else {
293 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
294 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra);
295 }
296 }
297
298 /*
299 * Move data with the same semantics as memmove() in case ranges don't overlap
300 * or src > dest. Undefined behavior on destructive overlaps.
301 */
access_memmove(CPUS390XState * env,S390Access * desta,S390Access * srca,uintptr_t ra)302 static void access_memmove(CPUS390XState *env, S390Access *desta,
303 S390Access *srca, uintptr_t ra)
304 {
305 int len = desta->size1 + desta->size2;
306
307 assert(len == srca->size1 + srca->size2);
308
309 /* Fallback to slow access in case we don't have access to all host pages */
310 if (user_or_likely(desta->haddr1 &&
311 srca->haddr1 &&
312 (!desta->size2 || desta->haddr2) &&
313 (!srca->size2 || srca->haddr2))) {
314 int diff = desta->size1 - srca->size1;
315
316 if (likely(diff == 0)) {
317 memmove(desta->haddr1, srca->haddr1, srca->size1);
318 if (unlikely(srca->size2)) {
319 memmove(desta->haddr2, srca->haddr2, srca->size2);
320 }
321 } else if (diff > 0) {
322 memmove(desta->haddr1, srca->haddr1, srca->size1);
323 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff);
324 if (likely(desta->size2)) {
325 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2);
326 }
327 } else {
328 diff = -diff;
329 memmove(desta->haddr1, srca->haddr1, desta->size1);
330 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff);
331 if (likely(srca->size2)) {
332 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2);
333 }
334 }
335 } else {
336 for (int i = 0; i < len; i++) {
337 uint8_t byte = access_get_byte(env, srca, i, ra);
338 access_set_byte(env, desta, i, byte, ra);
339 }
340 }
341 }
342
mmu_idx_from_as(uint8_t as)343 static int mmu_idx_from_as(uint8_t as)
344 {
345 switch (as) {
346 case AS_PRIMARY:
347 return MMU_PRIMARY_IDX;
348 case AS_SECONDARY:
349 return MMU_SECONDARY_IDX;
350 case AS_HOME:
351 return MMU_HOME_IDX;
352 default:
353 /* FIXME AS_ACCREG */
354 g_assert_not_reached();
355 }
356 }
357
358 /* and on array */
do_helper_nc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)359 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
360 uint64_t src, uintptr_t ra)
361 {
362 const int mmu_idx = s390x_env_mmu_index(env, false);
363 S390Access srca1, srca2, desta;
364 uint32_t i;
365 uint8_t c = 0;
366
367 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
368 __func__, l, dest, src);
369
370 /* NC always processes one more byte than specified - maximum is 256 */
371 l++;
372
373 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
374 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
375 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
376 set_helper_retaddr(ra);
377
378 for (i = 0; i < l; i++) {
379 const uint8_t x = access_get_byte(env, &srca1, i, ra) &
380 access_get_byte(env, &srca2, i, ra);
381
382 c |= x;
383 access_set_byte(env, &desta, i, x, ra);
384 }
385
386 clear_helper_retaddr();
387 return c != 0;
388 }
389
HELPER(nc)390 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
391 uint64_t src)
392 {
393 return do_helper_nc(env, l, dest, src, GETPC());
394 }
395
396 /* xor on array */
do_helper_xc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)397 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
398 uint64_t src, uintptr_t ra)
399 {
400 const int mmu_idx = s390x_env_mmu_index(env, false);
401 S390Access srca1, srca2, desta;
402 uint32_t i;
403 uint8_t c = 0;
404
405 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
406 __func__, l, dest, src);
407
408 /* XC always processes one more byte than specified - maximum is 256 */
409 l++;
410
411 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
412 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
413 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
414
415 /* xor with itself is the same as memset(0) */
416 if (src == dest) {
417 access_memset(env, &desta, 0, ra);
418 return 0;
419 }
420
421 set_helper_retaddr(ra);
422 for (i = 0; i < l; i++) {
423 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^
424 access_get_byte(env, &srca2, i, ra);
425
426 c |= x;
427 access_set_byte(env, &desta, i, x, ra);
428 }
429 clear_helper_retaddr();
430 return c != 0;
431 }
432
HELPER(xc)433 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
434 uint64_t src)
435 {
436 return do_helper_xc(env, l, dest, src, GETPC());
437 }
438
439 /* or on array */
do_helper_oc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)440 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
441 uint64_t src, uintptr_t ra)
442 {
443 const int mmu_idx = s390x_env_mmu_index(env, false);
444 S390Access srca1, srca2, desta;
445 uint32_t i;
446 uint8_t c = 0;
447
448 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
449 __func__, l, dest, src);
450
451 /* OC always processes one more byte than specified - maximum is 256 */
452 l++;
453
454 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
455 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
456 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
457 set_helper_retaddr(ra);
458
459 for (i = 0; i < l; i++) {
460 const uint8_t x = access_get_byte(env, &srca1, i, ra) |
461 access_get_byte(env, &srca2, i, ra);
462
463 c |= x;
464 access_set_byte(env, &desta, i, x, ra);
465 }
466
467 clear_helper_retaddr();
468 return c != 0;
469 }
470
HELPER(oc)471 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
472 uint64_t src)
473 {
474 return do_helper_oc(env, l, dest, src, GETPC());
475 }
476
477 /* memmove */
do_helper_mvc(CPUS390XState * env,uint32_t l,uint64_t dest,uint64_t src,uintptr_t ra)478 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
479 uint64_t src, uintptr_t ra)
480 {
481 const int mmu_idx = s390x_env_mmu_index(env, false);
482 S390Access srca, desta;
483 uint32_t i;
484
485 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
486 __func__, l, dest, src);
487
488 /* MVC always copies one more byte than specified - maximum is 256 */
489 l++;
490
491 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
492 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
493
494 /*
495 * "When the operands overlap, the result is obtained as if the operands
496 * were processed one byte at a time". Only non-destructive overlaps
497 * behave like memmove().
498 */
499 if (dest == src + 1) {
500 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra);
501 } else if (!is_destructive_overlap(env, dest, src, l)) {
502 access_memmove(env, &desta, &srca, ra);
503 } else {
504 set_helper_retaddr(ra);
505 for (i = 0; i < l; i++) {
506 uint8_t byte = access_get_byte(env, &srca, i, ra);
507
508 access_set_byte(env, &desta, i, byte, ra);
509 }
510 clear_helper_retaddr();
511 }
512
513 return env->cc_op;
514 }
515
HELPER(mvc)516 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
517 {
518 do_helper_mvc(env, l, dest, src, GETPC());
519 }
520
521 /* move right to left */
HELPER(mvcrl)522 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src)
523 {
524 const int mmu_idx = s390x_env_mmu_index(env, false);
525 const uint64_t ra = GETPC();
526 S390Access srca, desta;
527 int32_t i;
528
529 /* MVCRL always copies one more byte than specified - maximum is 256 */
530 l &= 0xff;
531 l++;
532
533 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
534 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
535
536 set_helper_retaddr(ra);
537 for (i = l - 1; i >= 0; i--) {
538 uint8_t byte = access_get_byte(env, &srca, i, ra);
539 access_set_byte(env, &desta, i, byte, ra);
540 }
541 clear_helper_retaddr();
542 }
543
544 /* move inverse */
HELPER(mvcin)545 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
546 {
547 const int mmu_idx = s390x_env_mmu_index(env, false);
548 S390Access srca, desta;
549 uintptr_t ra = GETPC();
550 int i;
551
552 /* MVCIN always copies one more byte than specified - maximum is 256 */
553 l++;
554
555 src = wrap_address(env, src - l + 1);
556 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
557 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
558
559 set_helper_retaddr(ra);
560 for (i = 0; i < l; i++) {
561 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra);
562 access_set_byte(env, &desta, i, x, ra);
563 }
564 clear_helper_retaddr();
565 }
566
567 /* move numerics */
HELPER(mvn)568 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
569 {
570 const int mmu_idx = s390x_env_mmu_index(env, false);
571 S390Access srca1, srca2, desta;
572 uintptr_t ra = GETPC();
573 int i;
574
575 /* MVN always copies one more byte than specified - maximum is 256 */
576 l++;
577
578 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
579 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
580 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
581
582 set_helper_retaddr(ra);
583 for (i = 0; i < l; i++) {
584 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) |
585 (access_get_byte(env, &srca2, i, ra) & 0xf0);
586
587 access_set_byte(env, &desta, i, x, ra);
588 }
589 clear_helper_retaddr();
590 }
591
592 /* move with offset */
HELPER(mvo)593 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
594 {
595 const int mmu_idx = s390x_env_mmu_index(env, false);
596 /* MVO always processes one more byte than specified - maximum is 16 */
597 const int len_dest = (l >> 4) + 1;
598 const int len_src = (l & 0xf) + 1;
599 uintptr_t ra = GETPC();
600 uint8_t byte_dest, byte_src;
601 S390Access srca, desta;
602 int i, j;
603
604 access_prepare(&srca, env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra);
605 access_prepare(&desta, env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra);
606
607 /* Handle rightmost byte */
608 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra);
609
610 set_helper_retaddr(ra);
611 byte_src = access_get_byte(env, &srca, len_src - 1, ra);
612 byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
613 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra);
614
615 /* Process remaining bytes from right to left */
616 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) {
617 byte_dest = byte_src >> 4;
618 if (j >= 0) {
619 byte_src = access_get_byte(env, &srca, j, ra);
620 } else {
621 byte_src = 0;
622 }
623 byte_dest |= byte_src << 4;
624 access_set_byte(env, &desta, i, byte_dest, ra);
625 }
626 clear_helper_retaddr();
627 }
628
629 /* move zones */
HELPER(mvz)630 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
631 {
632 const int mmu_idx = s390x_env_mmu_index(env, false);
633 S390Access srca1, srca2, desta;
634 uintptr_t ra = GETPC();
635 int i;
636
637 /* MVZ always copies one more byte than specified - maximum is 256 */
638 l++;
639
640 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
641 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
642 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
643
644 set_helper_retaddr(ra);
645 for (i = 0; i < l; i++) {
646 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) |
647 (access_get_byte(env, &srca2, i, ra) & 0x0f);
648
649 access_set_byte(env, &desta, i, x, ra);
650 }
651 clear_helper_retaddr();
652 }
653
654 /* compare unsigned byte arrays */
do_helper_clc(CPUS390XState * env,uint32_t l,uint64_t s1,uint64_t s2,uintptr_t ra)655 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
656 uint64_t s2, uintptr_t ra)
657 {
658 uint32_t i;
659 uint32_t cc = 0;
660
661 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
662 __func__, l, s1, s2);
663
664 for (i = 0; i <= l; i++) {
665 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
666 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
667 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
668 if (x < y) {
669 cc = 1;
670 break;
671 } else if (x > y) {
672 cc = 2;
673 break;
674 }
675 }
676
677 HELPER_LOG("\n");
678 return cc;
679 }
680
HELPER(clc)681 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
682 {
683 return do_helper_clc(env, l, s1, s2, GETPC());
684 }
685
686 /* compare logical under mask */
HELPER(clm)687 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
688 uint64_t addr)
689 {
690 uintptr_t ra = GETPC();
691 uint32_t cc = 0;
692
693 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
694 mask, addr);
695
696 if (!mask) {
697 /* Recognize access exceptions for the first byte */
698 probe_read(env, addr, 1, s390x_env_mmu_index(env, false), ra);
699 }
700
701 while (mask) {
702 if (mask & 8) {
703 uint8_t d = cpu_ldub_data_ra(env, addr, ra);
704 uint8_t r = extract32(r1, 24, 8);
705 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
706 addr);
707 if (r < d) {
708 cc = 1;
709 break;
710 } else if (r > d) {
711 cc = 2;
712 break;
713 }
714 addr++;
715 }
716 mask = (mask << 1) & 0xf;
717 r1 <<= 8;
718 }
719
720 HELPER_LOG("\n");
721 return cc;
722 }
723
get_address(CPUS390XState * env,int reg)724 static inline uint64_t get_address(CPUS390XState *env, int reg)
725 {
726 return wrap_address(env, env->regs[reg]);
727 }
728
729 /*
730 * Store the address to the given register, zeroing out unused leftmost
731 * bits in bit positions 32-63 (24-bit and 31-bit mode only).
732 */
set_address_zero(CPUS390XState * env,int reg,uint64_t address)733 static inline void set_address_zero(CPUS390XState *env, int reg,
734 uint64_t address)
735 {
736 if (env->psw.mask & PSW_MASK_64) {
737 env->regs[reg] = address;
738 } else {
739 if (!(env->psw.mask & PSW_MASK_32)) {
740 address &= 0x00ffffff;
741 } else {
742 address &= 0x7fffffff;
743 }
744 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
745 }
746 }
747
set_address(CPUS390XState * env,int reg,uint64_t address)748 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
749 {
750 if (env->psw.mask & PSW_MASK_64) {
751 /* 64-Bit mode */
752 env->regs[reg] = address;
753 } else {
754 if (!(env->psw.mask & PSW_MASK_32)) {
755 /* 24-Bit mode. According to the PoO it is implementation
756 dependent if bits 32-39 remain unchanged or are set to
757 zeros. Choose the former so that the function can also be
758 used for TRT. */
759 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
760 } else {
761 /* 31-Bit mode. According to the PoO it is implementation
762 dependent if bit 32 remains unchanged or is set to zero.
763 Choose the latter so that the function can also be used for
764 TRT. */
765 address &= 0x7fffffff;
766 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
767 }
768 }
769 }
770
wrap_length32(CPUS390XState * env,uint64_t length)771 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length)
772 {
773 if (!(env->psw.mask & PSW_MASK_64)) {
774 return (uint32_t)length;
775 }
776 return length;
777 }
778
wrap_length31(CPUS390XState * env,uint64_t length)779 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length)
780 {
781 if (!(env->psw.mask & PSW_MASK_64)) {
782 /* 24-Bit and 31-Bit mode */
783 length &= 0x7fffffff;
784 }
785 return length;
786 }
787
get_length(CPUS390XState * env,int reg)788 static inline uint64_t get_length(CPUS390XState *env, int reg)
789 {
790 return wrap_length31(env, env->regs[reg]);
791 }
792
set_length(CPUS390XState * env,int reg,uint64_t length)793 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
794 {
795 if (env->psw.mask & PSW_MASK_64) {
796 /* 64-Bit mode */
797 env->regs[reg] = length;
798 } else {
799 /* 24-Bit and 31-Bit mode */
800 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
801 }
802 }
803
804 /* search string (c is byte to search, r2 is string, r1 end of string) */
HELPER(srst)805 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
806 {
807 uintptr_t ra = GETPC();
808 uint64_t end, str;
809 uint32_t len;
810 uint8_t v, c = env->regs[0];
811
812 /* Bits 32-55 must contain all 0. */
813 if (env->regs[0] & 0xffffff00u) {
814 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
815 }
816
817 str = get_address(env, r2);
818 end = get_address(env, r1);
819
820 /* Lest we fail to service interrupts in a timely manner, limit the
821 amount of work we're willing to do. For now, let's cap at 8k. */
822 for (len = 0; len < 0x2000; ++len) {
823 if (str + len == end) {
824 /* Character not found. R1 & R2 are unmodified. */
825 env->cc_op = 2;
826 return;
827 }
828 v = cpu_ldub_data_ra(env, str + len, ra);
829 if (v == c) {
830 /* Character found. Set R1 to the location; R2 is unmodified. */
831 env->cc_op = 1;
832 set_address(env, r1, str + len);
833 return;
834 }
835 }
836
837 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
838 env->cc_op = 3;
839 set_address(env, r2, str + len);
840 }
841
HELPER(srstu)842 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
843 {
844 uintptr_t ra = GETPC();
845 uint32_t len;
846 uint16_t v, c = env->regs[0];
847 uint64_t end, str, adj_end;
848
849 /* Bits 32-47 of R0 must be zero. */
850 if (env->regs[0] & 0xffff0000u) {
851 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
852 }
853
854 str = get_address(env, r2);
855 end = get_address(env, r1);
856
857 /* If the LSB of the two addresses differ, use one extra byte. */
858 adj_end = end + ((str ^ end) & 1);
859
860 /* Lest we fail to service interrupts in a timely manner, limit the
861 amount of work we're willing to do. For now, let's cap at 8k. */
862 for (len = 0; len < 0x2000; len += 2) {
863 if (str + len == adj_end) {
864 /* End of input found. */
865 env->cc_op = 2;
866 return;
867 }
868 v = cpu_lduw_data_ra(env, str + len, ra);
869 if (v == c) {
870 /* Character found. Set R1 to the location; R2 is unmodified. */
871 env->cc_op = 1;
872 set_address(env, r1, str + len);
873 return;
874 }
875 }
876
877 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
878 env->cc_op = 3;
879 set_address(env, r2, str + len);
880 }
881
882 /* unsigned string compare (c is string terminator) */
HELPER(clst)883 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
884 {
885 uintptr_t ra = GETPC();
886 uint32_t len;
887
888 c = c & 0xff;
889 s1 = wrap_address(env, s1);
890 s2 = wrap_address(env, s2);
891
892 /* Lest we fail to service interrupts in a timely manner, limit the
893 amount of work we're willing to do. For now, let's cap at 8k. */
894 for (len = 0; len < 0x2000; ++len) {
895 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
896 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
897 if (v1 == v2) {
898 if (v1 == c) {
899 /* Equal. CC=0, and don't advance the registers. */
900 env->cc_op = 0;
901 return int128_make128(s2, s1);
902 }
903 } else {
904 /* Unequal. CC={1,2}, and advance the registers. Note that
905 the terminator need not be zero, but the string that contains
906 the terminator is by definition "low". */
907 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
908 return int128_make128(s2 + len, s1 + len);
909 }
910 }
911
912 /* CPU-determined bytes equal; advance the registers. */
913 env->cc_op = 3;
914 return int128_make128(s2 + len, s1 + len);
915 }
916
917 /* move page */
HELPER(mvpg)918 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2)
919 {
920 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK;
921 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK;
922 const int mmu_idx = s390x_env_mmu_index(env, false);
923 const bool f = extract64(r0, 11, 1);
924 const bool s = extract64(r0, 10, 1);
925 const bool cco = extract64(r0, 8, 1);
926 uintptr_t ra = GETPC();
927 S390Access srca, desta;
928 int exc;
929
930 if ((f && s) || extract64(r0, 12, 4)) {
931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
932 }
933
934 /*
935 * We always manually handle exceptions such that we can properly store
936 * r1/r2 to the lowcore on page-translation exceptions.
937 *
938 * TODO: Access key handling
939 */
940 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE,
941 MMU_DATA_LOAD, mmu_idx, ra);
942 if (exc) {
943 if (cco) {
944 return 2;
945 }
946 goto inject_exc;
947 }
948 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE,
949 MMU_DATA_STORE, mmu_idx, ra);
950 if (exc) {
951 if (cco && exc != PGM_PROTECTION) {
952 return 1;
953 }
954 goto inject_exc;
955 }
956 access_memmove(env, &desta, &srca, ra);
957 return 0; /* data moved */
958 inject_exc:
959 #if !defined(CONFIG_USER_ONLY)
960 if (exc != PGM_ADDRESSING) {
961 stq_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, trans_exc_code),
962 env->tlb_fill_tec);
963 }
964 if (exc == PGM_PAGE_TRANS) {
965 stb_phys(env_cpu(env)->as, env->psa + offsetof(LowCore, op_access_id),
966 r1 << 4 | r2);
967 }
968 #endif
969 tcg_s390_program_interrupt(env, exc, ra);
970 }
971
972 /* string copy */
HELPER(mvst)973 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
974 {
975 const int mmu_idx = s390x_env_mmu_index(env, false);
976 const uint64_t d = get_address(env, r1);
977 const uint64_t s = get_address(env, r2);
978 const uint8_t c = env->regs[0];
979 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK));
980 S390Access srca, desta;
981 uintptr_t ra = GETPC();
982 int i;
983
984 if (env->regs[0] & 0xffffff00ull) {
985 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
986 }
987
988 /*
989 * Our access should not exceed single pages, as we must not report access
990 * exceptions exceeding the actually copied range (which we don't know at
991 * this point). We might over-indicate watchpoints within the pages
992 * (if we ever care, we have to limit processing to a single byte).
993 */
994 access_prepare(&srca, env, s, len, MMU_DATA_LOAD, mmu_idx, ra);
995 access_prepare(&desta, env, d, len, MMU_DATA_STORE, mmu_idx, ra);
996
997 set_helper_retaddr(ra);
998 for (i = 0; i < len; i++) {
999 const uint8_t v = access_get_byte(env, &srca, i, ra);
1000
1001 access_set_byte(env, &desta, i, v, ra);
1002 if (v == c) {
1003 clear_helper_retaddr();
1004 set_address_zero(env, r1, d + i);
1005 return 1;
1006 }
1007 }
1008 clear_helper_retaddr();
1009 set_address_zero(env, r1, d + len);
1010 set_address_zero(env, r2, s + len);
1011 return 3;
1012 }
1013
1014 /* load access registers r1 to r3 from memory at a2 */
HELPER(lam)1015 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1016 {
1017 uintptr_t ra = GETPC();
1018 int i;
1019
1020 if (a2 & 0x3) {
1021 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1022 }
1023
1024 for (i = r1;; i = (i + 1) % 16) {
1025 env->aregs[i] = cpu_ldl_data_ra(env, a2, ra);
1026 a2 += 4;
1027
1028 if (i == r3) {
1029 break;
1030 }
1031 }
1032 }
1033
1034 /* store access registers r1 to r3 in memory at a2 */
HELPER(stam)1035 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1036 {
1037 uintptr_t ra = GETPC();
1038 int i;
1039
1040 if (a2 & 0x3) {
1041 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1042 }
1043
1044 for (i = r1;; i = (i + 1) % 16) {
1045 cpu_stl_data_ra(env, a2, env->aregs[i], ra);
1046 a2 += 4;
1047
1048 if (i == r3) {
1049 break;
1050 }
1051 }
1052 }
1053
1054 /* move long helper */
do_mvcl(CPUS390XState * env,uint64_t * dest,uint64_t * destlen,uint64_t * src,uint64_t * srclen,uint16_t pad,int wordsize,uintptr_t ra)1055 static inline uint32_t do_mvcl(CPUS390XState *env,
1056 uint64_t *dest, uint64_t *destlen,
1057 uint64_t *src, uint64_t *srclen,
1058 uint16_t pad, int wordsize, uintptr_t ra)
1059 {
1060 const int mmu_idx = s390x_env_mmu_index(env, false);
1061 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK));
1062 S390Access srca, desta;
1063 int i, cc;
1064
1065 if (*destlen == *srclen) {
1066 cc = 0;
1067 } else if (*destlen < *srclen) {
1068 cc = 1;
1069 } else {
1070 cc = 2;
1071 }
1072
1073 if (!*destlen) {
1074 return cc;
1075 }
1076
1077 /*
1078 * Only perform one type of type of operation (move/pad) at a time.
1079 * Stay within single pages.
1080 */
1081 if (*srclen) {
1082 /* Copy the src array */
1083 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len);
1084 *destlen -= len;
1085 *srclen -= len;
1086 access_prepare(&srca, env, *src, len, MMU_DATA_LOAD, mmu_idx, ra);
1087 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1088 access_memmove(env, &desta, &srca, ra);
1089 *src = wrap_address(env, *src + len);
1090 *dest = wrap_address(env, *dest + len);
1091 } else if (wordsize == 1) {
1092 /* Pad the remaining area */
1093 *destlen -= len;
1094 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1095 access_memset(env, &desta, pad, ra);
1096 *dest = wrap_address(env, *dest + len);
1097 } else {
1098 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1099 set_helper_retaddr(ra);
1100
1101 /* The remaining length selects the padding byte. */
1102 for (i = 0; i < len; (*destlen)--, i++) {
1103 if (*destlen & 1) {
1104 access_set_byte(env, &desta, i, pad, ra);
1105 } else {
1106 access_set_byte(env, &desta, i, pad >> 8, ra);
1107 }
1108 }
1109 clear_helper_retaddr();
1110 *dest = wrap_address(env, *dest + len);
1111 }
1112
1113 return *destlen ? 3 : cc;
1114 }
1115
1116 /* move long */
HELPER(mvcl)1117 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1118 {
1119 const int mmu_idx = s390x_env_mmu_index(env, false);
1120 uintptr_t ra = GETPC();
1121 uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
1122 uint64_t dest = get_address(env, r1);
1123 uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
1124 uint64_t src = get_address(env, r2);
1125 uint8_t pad = env->regs[r2 + 1] >> 24;
1126 CPUState *cs = env_cpu(env);
1127 S390Access srca, desta;
1128 uint32_t cc, cur_len;
1129
1130 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) {
1131 cc = 3;
1132 } else if (srclen == destlen) {
1133 cc = 0;
1134 } else if (destlen < srclen) {
1135 cc = 1;
1136 } else {
1137 cc = 2;
1138 }
1139
1140 /* We might have to zero-out some bits even if there was no action. */
1141 if (unlikely(!destlen || cc == 3)) {
1142 set_address_zero(env, r2, src);
1143 set_address_zero(env, r1, dest);
1144 return cc;
1145 } else if (!srclen) {
1146 set_address_zero(env, r2, src);
1147 }
1148
1149 /*
1150 * Only perform one type of type of operation (move/pad) in one step.
1151 * Stay within single pages.
1152 */
1153 while (destlen) {
1154 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK));
1155 if (!srclen) {
1156 access_prepare(&desta, env, dest, cur_len,
1157 MMU_DATA_STORE, mmu_idx, ra);
1158 access_memset(env, &desta, pad, ra);
1159 } else {
1160 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len);
1161
1162 access_prepare(&srca, env, src, cur_len,
1163 MMU_DATA_LOAD, mmu_idx, ra);
1164 access_prepare(&desta, env, dest, cur_len,
1165 MMU_DATA_STORE, mmu_idx, ra);
1166 access_memmove(env, &desta, &srca, ra);
1167 src = wrap_address(env, src + cur_len);
1168 srclen -= cur_len;
1169 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
1170 set_address_zero(env, r2, src);
1171 }
1172 dest = wrap_address(env, dest + cur_len);
1173 destlen -= cur_len;
1174 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
1175 set_address_zero(env, r1, dest);
1176
1177 /*
1178 * MVCL is interruptible. Return to the main loop if requested after
1179 * writing back all state to registers. If no interrupt will get
1180 * injected, we'll end up back in this handler and continue processing
1181 * the remaining parts.
1182 */
1183 if (destlen && unlikely(cpu_loop_exit_requested(cs))) {
1184 cpu_loop_exit_restore(cs, ra);
1185 }
1186 }
1187 return cc;
1188 }
1189
1190 /* move long extended */
HELPER(mvcle)1191 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1192 uint32_t r3)
1193 {
1194 uintptr_t ra = GETPC();
1195 uint64_t destlen = get_length(env, r1 + 1);
1196 uint64_t dest = get_address(env, r1);
1197 uint64_t srclen = get_length(env, r3 + 1);
1198 uint64_t src = get_address(env, r3);
1199 uint8_t pad = a2;
1200 uint32_t cc;
1201
1202 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
1203
1204 set_length(env, r1 + 1, destlen);
1205 set_length(env, r3 + 1, srclen);
1206 set_address(env, r1, dest);
1207 set_address(env, r3, src);
1208
1209 return cc;
1210 }
1211
1212 /* move long unicode */
HELPER(mvclu)1213 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1214 uint32_t r3)
1215 {
1216 uintptr_t ra = GETPC();
1217 uint64_t destlen = get_length(env, r1 + 1);
1218 uint64_t dest = get_address(env, r1);
1219 uint64_t srclen = get_length(env, r3 + 1);
1220 uint64_t src = get_address(env, r3);
1221 uint16_t pad = a2;
1222 uint32_t cc;
1223
1224 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
1225
1226 set_length(env, r1 + 1, destlen);
1227 set_length(env, r3 + 1, srclen);
1228 set_address(env, r1, dest);
1229 set_address(env, r3, src);
1230
1231 return cc;
1232 }
1233
1234 /* compare logical long helper */
do_clcl(CPUS390XState * env,uint64_t * src1,uint64_t * src1len,uint64_t * src3,uint64_t * src3len,uint16_t pad,uint64_t limit,int wordsize,uintptr_t ra)1235 static inline uint32_t do_clcl(CPUS390XState *env,
1236 uint64_t *src1, uint64_t *src1len,
1237 uint64_t *src3, uint64_t *src3len,
1238 uint16_t pad, uint64_t limit,
1239 int wordsize, uintptr_t ra)
1240 {
1241 uint64_t len = MAX(*src1len, *src3len);
1242 uint32_t cc = 0;
1243
1244 check_alignment(env, *src1len | *src3len, wordsize, ra);
1245
1246 if (!len) {
1247 return cc;
1248 }
1249
1250 /* Lest we fail to service interrupts in a timely manner, limit the
1251 amount of work we're willing to do. */
1252 if (len > limit) {
1253 len = limit;
1254 cc = 3;
1255 }
1256
1257 for (; len; len -= wordsize) {
1258 uint16_t v1 = pad;
1259 uint16_t v3 = pad;
1260
1261 if (*src1len) {
1262 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
1263 }
1264 if (*src3len) {
1265 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
1266 }
1267
1268 if (v1 != v3) {
1269 cc = (v1 < v3) ? 1 : 2;
1270 break;
1271 }
1272
1273 if (*src1len) {
1274 *src1 += wordsize;
1275 *src1len -= wordsize;
1276 }
1277 if (*src3len) {
1278 *src3 += wordsize;
1279 *src3len -= wordsize;
1280 }
1281 }
1282
1283 return cc;
1284 }
1285
1286
1287 /* compare logical long */
HELPER(clcl)1288 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1289 {
1290 uintptr_t ra = GETPC();
1291 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
1292 uint64_t src1 = get_address(env, r1);
1293 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
1294 uint64_t src3 = get_address(env, r2);
1295 uint8_t pad = env->regs[r2 + 1] >> 24;
1296 uint32_t cc;
1297
1298 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
1299
1300 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
1301 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
1302 set_address(env, r1, src1);
1303 set_address(env, r2, src3);
1304
1305 return cc;
1306 }
1307
1308 /* compare logical long extended memcompare insn with padding */
HELPER(clcle)1309 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1310 uint32_t r3)
1311 {
1312 uintptr_t ra = GETPC();
1313 uint64_t src1len = get_length(env, r1 + 1);
1314 uint64_t src1 = get_address(env, r1);
1315 uint64_t src3len = get_length(env, r3 + 1);
1316 uint64_t src3 = get_address(env, r3);
1317 uint8_t pad = a2;
1318 uint32_t cc;
1319
1320 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
1321
1322 set_length(env, r1 + 1, src1len);
1323 set_length(env, r3 + 1, src3len);
1324 set_address(env, r1, src1);
1325 set_address(env, r3, src3);
1326
1327 return cc;
1328 }
1329
1330 /* compare logical long unicode memcompare insn with padding */
HELPER(clclu)1331 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1332 uint32_t r3)
1333 {
1334 uintptr_t ra = GETPC();
1335 uint64_t src1len = get_length(env, r1 + 1);
1336 uint64_t src1 = get_address(env, r1);
1337 uint64_t src3len = get_length(env, r3 + 1);
1338 uint64_t src3 = get_address(env, r3);
1339 uint16_t pad = a2;
1340 uint32_t cc = 0;
1341
1342 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
1343
1344 set_length(env, r1 + 1, src1len);
1345 set_length(env, r3 + 1, src3len);
1346 set_address(env, r1, src1);
1347 set_address(env, r3, src3);
1348
1349 return cc;
1350 }
1351
1352 /* checksum */
HELPER(cksm)1353 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1,
1354 uint64_t src, uint64_t src_len)
1355 {
1356 uintptr_t ra = GETPC();
1357 uint64_t max_len, len;
1358 uint64_t cksm = (uint32_t)r1;
1359
1360 /* Lest we fail to service interrupts in a timely manner, limit the
1361 amount of work we're willing to do. For now, let's cap at 8k. */
1362 max_len = (src_len > 0x2000 ? 0x2000 : src_len);
1363
1364 /* Process full words as available. */
1365 for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
1366 cksm += (uint32_t)cpu_ldl_data_ra(env, src, ra);
1367 }
1368
1369 switch (max_len - len) {
1370 case 1:
1371 cksm += cpu_ldub_data_ra(env, src, ra) << 24;
1372 len += 1;
1373 break;
1374 case 2:
1375 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1376 len += 2;
1377 break;
1378 case 3:
1379 cksm += cpu_lduw_data_ra(env, src, ra) << 16;
1380 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
1381 len += 3;
1382 break;
1383 }
1384
1385 /* Fold the carry from the checksum. Note that we can see carry-out
1386 during folding more than once (but probably not more than twice). */
1387 while (cksm > 0xffffffffull) {
1388 cksm = (uint32_t)cksm + (cksm >> 32);
1389 }
1390
1391 /* Indicate whether or not we've processed everything. */
1392 env->cc_op = (len == src_len ? 0 : 3);
1393
1394 /* Return both cksm and processed length. */
1395 return int128_make128(cksm, len);
1396 }
1397
HELPER(pack)1398 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
1399 {
1400 uintptr_t ra = GETPC();
1401 int len_dest = len >> 4;
1402 int len_src = len & 0xf;
1403 uint8_t b;
1404
1405 dest += len_dest;
1406 src += len_src;
1407
1408 /* last byte is special, it only flips the nibbles */
1409 b = cpu_ldub_data_ra(env, src, ra);
1410 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1411 src--;
1412 len_src--;
1413
1414 /* now pack every value */
1415 while (len_dest > 0) {
1416 b = 0;
1417
1418 if (len_src >= 0) {
1419 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1420 src--;
1421 len_src--;
1422 }
1423 if (len_src >= 0) {
1424 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1425 src--;
1426 len_src--;
1427 }
1428
1429 len_dest--;
1430 dest--;
1431 cpu_stb_data_ra(env, dest, b, ra);
1432 }
1433 }
1434
do_pkau(CPUS390XState * env,uint64_t dest,uint64_t src,uint32_t srclen,int ssize,uintptr_t ra)1435 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1436 uint32_t srclen, int ssize, uintptr_t ra)
1437 {
1438 int i;
1439 /* The destination operand is always 16 bytes long. */
1440 const int destlen = 16;
1441
1442 /* The operands are processed from right to left. */
1443 src += srclen - 1;
1444 dest += destlen - 1;
1445
1446 for (i = 0; i < destlen; i++) {
1447 uint8_t b = 0;
1448
1449 /* Start with a positive sign */
1450 if (i == 0) {
1451 b = 0xc;
1452 } else if (srclen > ssize) {
1453 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1454 src -= ssize;
1455 srclen -= ssize;
1456 }
1457
1458 if (srclen > ssize) {
1459 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1460 src -= ssize;
1461 srclen -= ssize;
1462 }
1463
1464 cpu_stb_data_ra(env, dest, b, ra);
1465 dest--;
1466 }
1467 }
1468
1469
HELPER(pka)1470 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1471 uint32_t srclen)
1472 {
1473 do_pkau(env, dest, src, srclen, 1, GETPC());
1474 }
1475
HELPER(pku)1476 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1477 uint32_t srclen)
1478 {
1479 do_pkau(env, dest, src, srclen, 2, GETPC());
1480 }
1481
HELPER(unpk)1482 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1483 uint64_t src)
1484 {
1485 uintptr_t ra = GETPC();
1486 int len_dest = len >> 4;
1487 int len_src = len & 0xf;
1488 uint8_t b;
1489 int second_nibble = 0;
1490
1491 dest += len_dest;
1492 src += len_src;
1493
1494 /* last byte is special, it only flips the nibbles */
1495 b = cpu_ldub_data_ra(env, src, ra);
1496 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1497 src--;
1498 len_src--;
1499
1500 /* now pad every nibble with 0xf0 */
1501
1502 while (len_dest > 0) {
1503 uint8_t cur_byte = 0;
1504
1505 if (len_src > 0) {
1506 cur_byte = cpu_ldub_data_ra(env, src, ra);
1507 }
1508
1509 len_dest--;
1510 dest--;
1511
1512 /* only advance one nibble at a time */
1513 if (second_nibble) {
1514 cur_byte >>= 4;
1515 len_src--;
1516 src--;
1517 }
1518 second_nibble = !second_nibble;
1519
1520 /* digit */
1521 cur_byte = (cur_byte & 0xf);
1522 /* zone bits */
1523 cur_byte |= 0xf0;
1524
1525 cpu_stb_data_ra(env, dest, cur_byte, ra);
1526 }
1527 }
1528
do_unpkau(CPUS390XState * env,uint64_t dest,uint32_t destlen,int dsize,uint64_t src,uintptr_t ra)1529 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1530 uint32_t destlen, int dsize, uint64_t src,
1531 uintptr_t ra)
1532 {
1533 int i;
1534 uint32_t cc;
1535 uint8_t b;
1536 /* The source operand is always 16 bytes long. */
1537 const int srclen = 16;
1538
1539 /* The operands are processed from right to left. */
1540 src += srclen - 1;
1541 dest += destlen - dsize;
1542
1543 /* Check for the sign. */
1544 b = cpu_ldub_data_ra(env, src, ra);
1545 src--;
1546 switch (b & 0xf) {
1547 case 0xa:
1548 case 0xc:
1549 case 0xe ... 0xf:
1550 cc = 0; /* plus */
1551 break;
1552 case 0xb:
1553 case 0xd:
1554 cc = 1; /* minus */
1555 break;
1556 default:
1557 case 0x0 ... 0x9:
1558 cc = 3; /* invalid */
1559 break;
1560 }
1561
1562 /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1563 for (i = 0; i < destlen; i += dsize) {
1564 if (i == (31 * dsize)) {
1565 /* If length is 32/64 bytes, the leftmost byte is 0. */
1566 b = 0;
1567 } else if (i % (2 * dsize)) {
1568 b = cpu_ldub_data_ra(env, src, ra);
1569 src--;
1570 } else {
1571 b >>= 4;
1572 }
1573 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1574 dest -= dsize;
1575 }
1576
1577 return cc;
1578 }
1579
HELPER(unpka)1580 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1581 uint64_t src)
1582 {
1583 return do_unpkau(env, dest, destlen, 1, src, GETPC());
1584 }
1585
HELPER(unpku)1586 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1587 uint64_t src)
1588 {
1589 return do_unpkau(env, dest, destlen, 2, src, GETPC());
1590 }
1591
HELPER(tp)1592 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1593 {
1594 uintptr_t ra = GETPC();
1595 uint32_t cc = 0;
1596 int i;
1597
1598 for (i = 0; i < destlen; i++) {
1599 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1600 /* digit */
1601 cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1602
1603 if (i == (destlen - 1)) {
1604 /* sign */
1605 cc |= (b & 0xf) < 0xa ? 1 : 0;
1606 } else {
1607 /* digit */
1608 cc |= (b & 0xf) > 0x9 ? 2 : 0;
1609 }
1610 }
1611
1612 return cc;
1613 }
1614
do_helper_tr(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1615 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1616 uint64_t trans, uintptr_t ra)
1617 {
1618 uint32_t i;
1619
1620 for (i = 0; i <= len; i++) {
1621 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1622 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1623 cpu_stb_data_ra(env, array + i, new_byte, ra);
1624 }
1625
1626 return env->cc_op;
1627 }
1628
HELPER(tr)1629 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1630 uint64_t trans)
1631 {
1632 do_helper_tr(env, len, array, trans, GETPC());
1633 }
1634
HELPER(tre)1635 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array,
1636 uint64_t len, uint64_t trans)
1637 {
1638 uintptr_t ra = GETPC();
1639 uint8_t end = env->regs[0] & 0xff;
1640 uint64_t l = len;
1641 uint64_t i;
1642 uint32_t cc = 0;
1643
1644 if (!(env->psw.mask & PSW_MASK_64)) {
1645 array &= 0x7fffffff;
1646 l = (uint32_t)l;
1647 }
1648
1649 /* Lest we fail to service interrupts in a timely manner, limit the
1650 amount of work we're willing to do. For now, let's cap at 8k. */
1651 if (l > 0x2000) {
1652 l = 0x2000;
1653 cc = 3;
1654 }
1655
1656 for (i = 0; i < l; i++) {
1657 uint8_t byte, new_byte;
1658
1659 byte = cpu_ldub_data_ra(env, array + i, ra);
1660
1661 if (byte == end) {
1662 cc = 1;
1663 break;
1664 }
1665
1666 new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1667 cpu_stb_data_ra(env, array + i, new_byte, ra);
1668 }
1669
1670 env->cc_op = cc;
1671 return int128_make128(len - i, array + i);
1672 }
1673
do_helper_trt(CPUS390XState * env,int len,uint64_t array,uint64_t trans,int inc,uintptr_t ra)1674 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1675 uint64_t array, uint64_t trans,
1676 int inc, uintptr_t ra)
1677 {
1678 int i;
1679
1680 for (i = 0; i <= len; i++) {
1681 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1682 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1683
1684 if (sbyte != 0) {
1685 set_address(env, 1, array + i * inc);
1686 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1687 return (i == len) ? 2 : 1;
1688 }
1689 }
1690
1691 return 0;
1692 }
1693
do_helper_trt_fwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1694 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len,
1695 uint64_t array, uint64_t trans,
1696 uintptr_t ra)
1697 {
1698 return do_helper_trt(env, len, array, trans, 1, ra);
1699 }
1700
HELPER(trt)1701 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1702 uint64_t trans)
1703 {
1704 return do_helper_trt(env, len, array, trans, 1, GETPC());
1705 }
1706
do_helper_trt_bkwd(CPUS390XState * env,uint32_t len,uint64_t array,uint64_t trans,uintptr_t ra)1707 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len,
1708 uint64_t array, uint64_t trans,
1709 uintptr_t ra)
1710 {
1711 return do_helper_trt(env, len, array, trans, -1, ra);
1712 }
1713
HELPER(trtr)1714 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1715 uint64_t trans)
1716 {
1717 return do_helper_trt(env, len, array, trans, -1, GETPC());
1718 }
1719
1720 /* Translate one/two to one/two */
HELPER(trXX)1721 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1722 uint32_t tst, uint32_t sizes)
1723 {
1724 uintptr_t ra = GETPC();
1725 int dsize = (sizes & 1) ? 1 : 2;
1726 int ssize = (sizes & 2) ? 1 : 2;
1727 uint64_t tbl = get_address(env, 1);
1728 uint64_t dst = get_address(env, r1);
1729 uint64_t len = get_length(env, r1 + 1);
1730 uint64_t src = get_address(env, r2);
1731 uint32_t cc = 3;
1732 int i;
1733
1734 /* The lower address bits of TBL are ignored. For TROO, TROT, it's
1735 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either
1736 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */
1737 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1738 tbl &= -4096;
1739 } else {
1740 tbl &= -8;
1741 }
1742
1743 check_alignment(env, len, ssize, ra);
1744
1745 /* Lest we fail to service interrupts in a timely manner, */
1746 /* limit the amount of work we're willing to do. */
1747 for (i = 0; i < 0x2000; i++) {
1748 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1749 uint64_t tble = tbl + (sval * dsize);
1750 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1751 if (dval == tst) {
1752 cc = 1;
1753 break;
1754 }
1755 cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1756
1757 len -= ssize;
1758 src += ssize;
1759 dst += dsize;
1760
1761 if (len == 0) {
1762 cc = 0;
1763 break;
1764 }
1765 }
1766
1767 set_address(env, r1, dst);
1768 set_length(env, r1 + 1, len);
1769 set_address(env, r2, src);
1770
1771 return cc;
1772 }
1773
do_csst(CPUS390XState * env,uint32_t r3,uint64_t a1,uint64_t a2,bool parallel)1774 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
1775 uint64_t a2, bool parallel)
1776 {
1777 uint32_t mem_idx = s390x_env_mmu_index(env, false);
1778 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, mem_idx);
1779 MemOpIdx oi8 = make_memop_idx(MO_TE | MO_64, mem_idx);
1780 MemOpIdx oi4 = make_memop_idx(MO_TE | MO_32, mem_idx);
1781 MemOpIdx oi2 = make_memop_idx(MO_TE | MO_16, mem_idx);
1782 MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx);
1783 uintptr_t ra = GETPC();
1784 uint32_t fc = extract32(env->regs[0], 0, 8);
1785 uint32_t sc = extract32(env->regs[0], 8, 8);
1786 uint64_t pl = get_address(env, 1) & -16;
1787 uint64_t svh, svl;
1788 uint32_t cc;
1789
1790 /* Sanity check the function code and storage characteristic. */
1791 if (fc > 1 || sc > 3) {
1792 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1793 goto spec_exception;
1794 }
1795 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1796 goto spec_exception;
1797 }
1798 }
1799
1800 /* Sanity check the alignments. */
1801 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) {
1802 goto spec_exception;
1803 }
1804
1805 /* Sanity check writability of the store address. */
1806 probe_write(env, a2, 1 << sc, mem_idx, ra);
1807
1808 /*
1809 * Note that the compare-and-swap is atomic, and the store is atomic,
1810 * but the complete operation is not. Therefore we do not need to
1811 * assert serial context in order to implement this. That said,
1812 * restart early if we can't support either operation that is supposed
1813 * to be atomic.
1814 */
1815 if (parallel) {
1816 uint32_t max = 2;
1817 #ifdef CONFIG_ATOMIC64
1818 max = 3;
1819 #endif
1820 if ((HAVE_CMPXCHG128 ? 0 : fc + 2 > max) ||
1821 (HAVE_ATOMIC128_RW ? 0 : sc > max)) {
1822 cpu_loop_exit_atomic(env_cpu(env), ra);
1823 }
1824 }
1825
1826 /*
1827 * All loads happen before all stores. For simplicity, load the entire
1828 * store value area from the parameter list.
1829 */
1830 svh = cpu_ldq_mmu(env, pl + 16, oi8, ra);
1831 svl = cpu_ldq_mmu(env, pl + 24, oi8, ra);
1832
1833 switch (fc) {
1834 case 0:
1835 {
1836 uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra);
1837 uint32_t cv = env->regs[r3];
1838 uint32_t ov;
1839
1840 if (parallel) {
1841 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra);
1842 } else {
1843 ov = cpu_ldl_mmu(env, a1, oi4, ra);
1844 cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra);
1845 }
1846 cc = (ov != cv);
1847 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1848 }
1849 break;
1850
1851 case 1:
1852 {
1853 uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra);
1854 uint64_t cv = env->regs[r3];
1855 uint64_t ov;
1856
1857 if (parallel) {
1858 #ifdef CONFIG_ATOMIC64
1859 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra);
1860 #else
1861 /* Note that we asserted !parallel above. */
1862 g_assert_not_reached();
1863 #endif
1864 } else {
1865 ov = cpu_ldq_mmu(env, a1, oi8, ra);
1866 cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra);
1867 }
1868 cc = (ov != cv);
1869 env->regs[r3] = ov;
1870 }
1871 break;
1872
1873 case 2:
1874 {
1875 Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra);
1876 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1877 Int128 ov;
1878
1879 if (!parallel) {
1880 ov = cpu_ld16_mmu(env, a1, oi16, ra);
1881 cc = !int128_eq(ov, cv);
1882 if (cc) {
1883 nv = ov;
1884 }
1885 cpu_st16_mmu(env, a1, nv, oi16, ra);
1886 } else if (HAVE_CMPXCHG128) {
1887 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra);
1888 cc = !int128_eq(ov, cv);
1889 } else {
1890 /* Note that we asserted !parallel above. */
1891 g_assert_not_reached();
1892 }
1893
1894 env->regs[r3 + 0] = int128_gethi(ov);
1895 env->regs[r3 + 1] = int128_getlo(ov);
1896 }
1897 break;
1898
1899 default:
1900 g_assert_not_reached();
1901 }
1902
1903 /* Store only if the comparison succeeded. Note that above we use a pair
1904 of 64-bit big-endian loads, so for sc < 3 we must extract the value
1905 from the most-significant bits of svh. */
1906 if (cc == 0) {
1907 switch (sc) {
1908 case 0:
1909 cpu_stb_mmu(env, a2, svh >> 56, oi1, ra);
1910 break;
1911 case 1:
1912 cpu_stw_mmu(env, a2, svh >> 48, oi2, ra);
1913 break;
1914 case 2:
1915 cpu_stl_mmu(env, a2, svh >> 32, oi4, ra);
1916 break;
1917 case 3:
1918 cpu_stq_mmu(env, a2, svh, oi8, ra);
1919 break;
1920 case 4:
1921 cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra);
1922 break;
1923 default:
1924 g_assert_not_reached();
1925 }
1926 }
1927
1928 return cc;
1929
1930 spec_exception:
1931 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1932 }
1933
HELPER(csst)1934 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
1935 {
1936 return do_csst(env, r3, a1, a2, false);
1937 }
1938
HELPER(csst_parallel)1939 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1,
1940 uint64_t a2)
1941 {
1942 return do_csst(env, r3, a1, a2, true);
1943 }
1944
1945 #if !defined(CONFIG_USER_ONLY)
HELPER(lctlg)1946 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1947 {
1948 uintptr_t ra = GETPC();
1949 bool PERchanged = false;
1950 uint64_t src = a2;
1951 uint32_t i;
1952
1953 if (src & 0x7) {
1954 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1955 }
1956
1957 for (i = r1;; i = (i + 1) % 16) {
1958 uint64_t val = cpu_ldq_data_ra(env, src, ra);
1959 if (env->cregs[i] != val && i >= 9 && i <= 11) {
1960 PERchanged = true;
1961 }
1962 env->cregs[i] = val;
1963 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
1964 i, src, val);
1965 src += sizeof(uint64_t);
1966
1967 if (i == r3) {
1968 break;
1969 }
1970 }
1971
1972 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1973 s390_cpu_recompute_watchpoints(env_cpu(env));
1974 }
1975
1976 tlb_flush(env_cpu(env));
1977 }
1978
HELPER(lctl)1979 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1980 {
1981 uintptr_t ra = GETPC();
1982 bool PERchanged = false;
1983 uint64_t src = a2;
1984 uint32_t i;
1985
1986 if (src & 0x3) {
1987 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1988 }
1989
1990 for (i = r1;; i = (i + 1) % 16) {
1991 uint32_t val = cpu_ldl_data_ra(env, src, ra);
1992 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
1993 PERchanged = true;
1994 }
1995 env->cregs[i] = deposit64(env->cregs[i], 0, 32, val);
1996 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
1997 src += sizeof(uint32_t);
1998
1999 if (i == r3) {
2000 break;
2001 }
2002 }
2003
2004 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
2005 s390_cpu_recompute_watchpoints(env_cpu(env));
2006 }
2007
2008 tlb_flush(env_cpu(env));
2009 }
2010
HELPER(stctg)2011 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2012 {
2013 uintptr_t ra = GETPC();
2014 uint64_t dest = a2;
2015 uint32_t i;
2016
2017 if (dest & 0x7) {
2018 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2019 }
2020
2021 for (i = r1;; i = (i + 1) % 16) {
2022 cpu_stq_data_ra(env, dest, env->cregs[i], ra);
2023 dest += sizeof(uint64_t);
2024
2025 if (i == r3) {
2026 break;
2027 }
2028 }
2029 }
2030
HELPER(stctl)2031 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2032 {
2033 uintptr_t ra = GETPC();
2034 uint64_t dest = a2;
2035 uint32_t i;
2036
2037 if (dest & 0x3) {
2038 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2039 }
2040
2041 for (i = r1;; i = (i + 1) % 16) {
2042 cpu_stl_data_ra(env, dest, env->cregs[i], ra);
2043 dest += sizeof(uint32_t);
2044
2045 if (i == r3) {
2046 break;
2047 }
2048 }
2049 }
2050
HELPER(testblock)2051 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
2052 {
2053 uintptr_t ra = GETPC();
2054 int i;
2055
2056 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
2057
2058 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
2059 cpu_stq_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra);
2060 }
2061
2062 return 0;
2063 }
2064
HELPER(tprot)2065 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2)
2066 {
2067 S390CPU *cpu = env_archcpu(env);
2068 CPUState *cs = env_cpu(env);
2069
2070 /*
2071 * TODO: we currently don't handle all access protection types
2072 * (including access-list and key-controlled) as well as AR mode.
2073 */
2074 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) {
2075 /* Fetching permitted; storing permitted */
2076 return 0;
2077 }
2078
2079 if (env->int_pgm_code == PGM_PROTECTION) {
2080 /* retry if reading is possible */
2081 cs->exception_index = -1;
2082 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) {
2083 /* Fetching permitted; storing not permitted */
2084 return 1;
2085 }
2086 }
2087
2088 switch (env->int_pgm_code) {
2089 case PGM_PROTECTION:
2090 /* Fetching not permitted; storing not permitted */
2091 cs->exception_index = -1;
2092 return 2;
2093 case PGM_ADDRESSING:
2094 case PGM_TRANS_SPEC:
2095 /* exceptions forwarded to the guest */
2096 s390_cpu_virt_mem_handle_exc(cpu, GETPC());
2097 return 0;
2098 }
2099
2100 /* Translation not available */
2101 cs->exception_index = -1;
2102 return 3;
2103 }
2104
2105 /* insert storage key extended */
HELPER(iske)2106 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
2107 {
2108 static S390SKeysState *ss;
2109 static S390SKeysClass *skeyclass;
2110 uint64_t addr = wrap_address(env, r2);
2111 uint8_t key;
2112 int rc;
2113
2114 addr = mmu_real2abs(env, addr);
2115 if (!mmu_absolute_addr_valid(addr, false)) {
2116 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2117 }
2118
2119 if (unlikely(!ss)) {
2120 ss = s390_get_skeys_device();
2121 skeyclass = S390_SKEYS_GET_CLASS(ss);
2122 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2123 tlb_flush_all_cpus_synced(env_cpu(env));
2124 }
2125 }
2126
2127 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2128 if (rc) {
2129 return 0;
2130 }
2131 return key;
2132 }
2133
2134 /* set storage key extended */
HELPER(sske)2135 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
2136 {
2137 static S390SKeysState *ss;
2138 static S390SKeysClass *skeyclass;
2139 uint64_t addr = wrap_address(env, r2);
2140 uint8_t key;
2141
2142 addr = mmu_real2abs(env, addr);
2143 if (!mmu_absolute_addr_valid(addr, false)) {
2144 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2145 }
2146
2147 if (unlikely(!ss)) {
2148 ss = s390_get_skeys_device();
2149 skeyclass = S390_SKEYS_GET_CLASS(ss);
2150 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2151 tlb_flush_all_cpus_synced(env_cpu(env));
2152 }
2153 }
2154
2155 key = r1 & 0xfe;
2156 s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2157 /*
2158 * As we can only flush by virtual address and not all the entries
2159 * that point to a physical address we have to flush the whole TLB.
2160 */
2161 tlb_flush_all_cpus_synced(env_cpu(env));
2162 }
2163
2164 /* reset reference bit extended */
HELPER(rrbe)2165 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
2166 {
2167 uint64_t addr = wrap_address(env, r2);
2168 static S390SKeysState *ss;
2169 static S390SKeysClass *skeyclass;
2170 uint8_t re, key;
2171 int rc;
2172
2173 addr = mmu_real2abs(env, addr);
2174 if (!mmu_absolute_addr_valid(addr, false)) {
2175 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2176 }
2177
2178 if (unlikely(!ss)) {
2179 ss = s390_get_skeys_device();
2180 skeyclass = S390_SKEYS_GET_CLASS(ss);
2181 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2182 tlb_flush_all_cpus_synced(env_cpu(env));
2183 }
2184 }
2185
2186 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2187 if (rc) {
2188 return 0;
2189 }
2190
2191 re = key & (SK_R | SK_C);
2192 key &= ~SK_R;
2193
2194 rc = s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2195 if (rc) {
2196 return 0;
2197 }
2198 /*
2199 * As we can only flush by virtual address and not all the entries
2200 * that point to a physical address we have to flush the whole TLB.
2201 */
2202 tlb_flush_all_cpus_synced(env_cpu(env));
2203
2204 /*
2205 * cc
2206 *
2207 * 0 Reference bit zero; change bit zero
2208 * 1 Reference bit zero; change bit one
2209 * 2 Reference bit one; change bit zero
2210 * 3 Reference bit one; change bit one
2211 */
2212
2213 return re >> 1;
2214 }
2215
HELPER(mvcs)2216 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2217 uint64_t key)
2218 {
2219 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2220 S390Access srca, desta;
2221 uintptr_t ra = GETPC();
2222 int cc = 0;
2223
2224 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2225 __func__, l, a1, a2);
2226
2227 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2228 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2229 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2230 }
2231
2232 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2233 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2234 }
2235
2236 l = wrap_length32(env, l);
2237 if (l > 256) {
2238 /* max 256 */
2239 l = 256;
2240 cc = 3;
2241 } else if (!l) {
2242 return cc;
2243 }
2244
2245 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra);
2246 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra);
2247 access_memmove(env, &desta, &srca, ra);
2248 return cc;
2249 }
2250
HELPER(mvcp)2251 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2252 uint64_t key)
2253 {
2254 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2255 S390Access srca, desta;
2256 uintptr_t ra = GETPC();
2257 int cc = 0;
2258
2259 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2260 __func__, l, a1, a2);
2261
2262 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2263 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2264 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2265 }
2266
2267 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2268 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2269 }
2270
2271 l = wrap_length32(env, l);
2272 if (l > 256) {
2273 /* max 256 */
2274 l = 256;
2275 cc = 3;
2276 } else if (!l) {
2277 return cc;
2278 }
2279 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra);
2280 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra);
2281 access_memmove(env, &desta, &srca, ra);
2282 return cc;
2283 }
2284
HELPER(idte)2285 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
2286 {
2287 CPUState *cs = env_cpu(env);
2288 const uintptr_t ra = GETPC();
2289 uint64_t table, entry, raddr;
2290 uint16_t entries, i, index = 0;
2291
2292 if (r2 & 0xff000) {
2293 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2294 }
2295
2296 if (!(r2 & 0x800)) {
2297 /* invalidation-and-clearing operation */
2298 table = r1 & ASCE_ORIGIN;
2299 entries = (r2 & 0x7ff) + 1;
2300
2301 switch (r1 & ASCE_TYPE_MASK) {
2302 case ASCE_TYPE_REGION1:
2303 index = (r2 >> 53) & 0x7ff;
2304 break;
2305 case ASCE_TYPE_REGION2:
2306 index = (r2 >> 42) & 0x7ff;
2307 break;
2308 case ASCE_TYPE_REGION3:
2309 index = (r2 >> 31) & 0x7ff;
2310 break;
2311 case ASCE_TYPE_SEGMENT:
2312 index = (r2 >> 20) & 0x7ff;
2313 break;
2314 }
2315 for (i = 0; i < entries; i++) {
2316 /* addresses are not wrapped in 24/31bit mode but table index is */
2317 raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
2318 entry = cpu_ldq_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra);
2319 if (!(entry & REGION_ENTRY_I)) {
2320 /* we are allowed to not store if already invalid */
2321 entry |= REGION_ENTRY_I;
2322 cpu_stq_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra);
2323 }
2324 }
2325 }
2326
2327 /* We simply flush the complete tlb, therefore we can ignore r3. */
2328 if (m4 & 1) {
2329 tlb_flush(cs);
2330 } else {
2331 tlb_flush_all_cpus_synced(cs);
2332 }
2333 }
2334
2335 /* invalidate pte */
HELPER(ipte)2336 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
2337 uint32_t m4)
2338 {
2339 CPUState *cs = env_cpu(env);
2340 const uintptr_t ra = GETPC();
2341 uint64_t page = vaddr & TARGET_PAGE_MASK;
2342 uint64_t pte_addr, pte;
2343
2344 /* Compute the page table entry address */
2345 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN);
2346 pte_addr += VADDR_PAGE_TX(vaddr) * 8;
2347
2348 /* Mark the page table entry as invalid */
2349 pte = cpu_ldq_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra);
2350 pte |= PAGE_ENTRY_I;
2351 cpu_stq_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra);
2352
2353 /* XXX we exploit the fact that Linux passes the exact virtual
2354 address here - it's not obliged to! */
2355 if (m4 & 1) {
2356 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2357 tlb_flush_page(cs, page);
2358 /* XXX 31-bit hack */
2359 tlb_flush_page(cs, page ^ 0x80000000);
2360 } else {
2361 /* looks like we don't have a valid virtual address */
2362 tlb_flush(cs);
2363 }
2364 } else {
2365 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2366 tlb_flush_page_all_cpus_synced(cs, page);
2367 /* XXX 31-bit hack */
2368 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
2369 } else {
2370 /* looks like we don't have a valid virtual address */
2371 tlb_flush_all_cpus_synced(cs);
2372 }
2373 }
2374 }
2375
2376 /* flush local tlb */
HELPER(ptlb)2377 void HELPER(ptlb)(CPUS390XState *env)
2378 {
2379 tlb_flush(env_cpu(env));
2380 }
2381
2382 /* flush global tlb */
HELPER(purge)2383 void HELPER(purge)(CPUS390XState *env)
2384 {
2385 tlb_flush_all_cpus_synced(env_cpu(env));
2386 }
2387
2388 /* load real address */
HELPER(lra)2389 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t r1, uint64_t addr)
2390 {
2391 uint64_t asc = env->psw.mask & PSW_MASK_ASC;
2392 uint64_t ret, tec;
2393 int flags, exc, cc;
2394
2395 /* XXX incomplete - has more corner cases */
2396 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2397 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC());
2398 }
2399
2400 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec);
2401 if (exc) {
2402 cc = 3;
2403 ret = (r1 & 0xFFFFFFFF00000000ULL) | exc | 0x80000000;
2404 } else {
2405 cc = 0;
2406 ret |= addr & ~TARGET_PAGE_MASK;
2407 }
2408
2409 env->cc_op = cc;
2410 return ret;
2411 }
2412 #endif
2413
2414 /* Execute instruction. This instruction executes an insn modified with
2415 the contents of r1. It does not change the executed instruction in memory;
2416 it does not change the program counter.
2417
2418 Perform this by recording the modified instruction in env->ex_value.
2419 This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2420 */
HELPER(ex)2421 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2422 {
2423 uint64_t insn;
2424 uint8_t opc;
2425
2426 /* EXECUTE targets must be at even addresses. */
2427 if (addr & 1) {
2428 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
2429 }
2430
2431 insn = cpu_lduw_code(env, addr);
2432 opc = insn >> 8;
2433
2434 /* Or in the contents of R1[56:63]. */
2435 insn |= r1 & 0xff;
2436
2437 /* Load the rest of the instruction. */
2438 insn <<= 48;
2439 switch (get_ilen(opc)) {
2440 case 2:
2441 break;
2442 case 4:
2443 insn |= (uint64_t)cpu_lduw_code(env, addr + 2) << 32;
2444 break;
2445 case 6:
2446 insn |= (uint64_t)(uint32_t)cpu_ldl_code(env, addr + 2) << 16;
2447 break;
2448 default:
2449 g_assert_not_reached();
2450 }
2451
2452 /* The very most common cases can be sped up by avoiding a new TB. */
2453 if ((opc & 0xf0) == 0xd0) {
2454 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2455 uint64_t, uintptr_t);
2456 static const dx_helper dx[16] = {
2457 [0x0] = do_helper_trt_bkwd,
2458 [0x2] = do_helper_mvc,
2459 [0x4] = do_helper_nc,
2460 [0x5] = do_helper_clc,
2461 [0x6] = do_helper_oc,
2462 [0x7] = do_helper_xc,
2463 [0xc] = do_helper_tr,
2464 [0xd] = do_helper_trt_fwd,
2465 };
2466 dx_helper helper = dx[opc & 0xf];
2467
2468 if (helper) {
2469 uint32_t l = extract64(insn, 48, 8);
2470 uint32_t b1 = extract64(insn, 44, 4);
2471 uint32_t d1 = extract64(insn, 32, 12);
2472 uint32_t b2 = extract64(insn, 28, 4);
2473 uint32_t d2 = extract64(insn, 16, 12);
2474 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1);
2475 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2);
2476
2477 env->cc_op = helper(env, l, a1, a2, 0);
2478 env->psw.addr += ilen;
2479 return;
2480 }
2481 } else if (opc == 0x0a) {
2482 env->int_svc_code = extract64(insn, 48, 8);
2483 env->int_svc_ilen = ilen;
2484 helper_exception(env, EXCP_SVC);
2485 g_assert_not_reached();
2486 }
2487
2488 /* Record the insn we want to execute as well as the ilen to use
2489 during the execution of the target insn. This will also ensure
2490 that ex_value is non-zero, which flags that we are in a state
2491 that requires such execution. */
2492 env->ex_value = insn | ilen;
2493 env->ex_target = addr;
2494 }
2495
HELPER(mvcos)2496 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2497 uint64_t len)
2498 {
2499 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2500 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2501 const uint64_t r0 = env->regs[0];
2502 const uintptr_t ra = GETPC();
2503 uint8_t dest_key, dest_as, dest_k, dest_a;
2504 uint8_t src_key, src_as, src_k, src_a;
2505 uint64_t val;
2506 int cc = 0;
2507
2508 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2509 __func__, dest, src, len);
2510
2511 if (!(env->psw.mask & PSW_MASK_DAT)) {
2512 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2513 }
2514
2515 /* OAC (operand access control) for the first operand -> dest */
2516 val = (r0 & 0xffff0000ULL) >> 16;
2517 dest_key = (val >> 12) & 0xf;
2518 dest_as = (val >> 6) & 0x3;
2519 dest_k = (val >> 1) & 0x1;
2520 dest_a = val & 0x1;
2521
2522 /* OAC (operand access control) for the second operand -> src */
2523 val = (r0 & 0x0000ffffULL);
2524 src_key = (val >> 12) & 0xf;
2525 src_as = (val >> 6) & 0x3;
2526 src_k = (val >> 1) & 0x1;
2527 src_a = val & 0x1;
2528
2529 if (!dest_k) {
2530 dest_key = psw_key;
2531 }
2532 if (!src_k) {
2533 src_key = psw_key;
2534 }
2535 if (!dest_a) {
2536 dest_as = psw_as;
2537 }
2538 if (!src_a) {
2539 src_as = psw_as;
2540 }
2541
2542 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2543 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2544 }
2545 if (!(env->cregs[0] & CR0_SECONDARY) &&
2546 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2547 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2548 }
2549 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2550 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2551 }
2552
2553 len = wrap_length32(env, len);
2554 if (len > 4096) {
2555 cc = 3;
2556 len = 4096;
2557 }
2558
2559 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2560 if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2561 (env->psw.mask & PSW_MASK_PSTATE)) {
2562 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2563 __func__);
2564 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra);
2565 }
2566
2567 /* FIXME: Access using correct keys and AR-mode */
2568 if (len) {
2569 S390Access srca, desta;
2570
2571 access_prepare(&srca, env, src, len, MMU_DATA_LOAD,
2572 mmu_idx_from_as(src_as), ra);
2573 access_prepare(&desta, env, dest, len, MMU_DATA_STORE,
2574 mmu_idx_from_as(dest_as), ra);
2575
2576 access_memmove(env, &desta, &srca, ra);
2577 }
2578
2579 return cc;
2580 }
2581
2582 /* Decode a Unicode character. A return value < 0 indicates success, storing
2583 the UTF-32 result into OCHAR and the input length into OLEN. A return
2584 value >= 0 indicates failure, and the CC value to be returned. */
2585 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2586 uint64_t ilen, bool enh_check, uintptr_t ra,
2587 uint32_t *ochar, uint32_t *olen);
2588
2589 /* Encode a Unicode character. A return value < 0 indicates success, storing
2590 the bytes into ADDR and the output length into OLEN. A return value >= 0
2591 indicates failure, and the CC value to be returned. */
2592 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2593 uint64_t ilen, uintptr_t ra, uint32_t c,
2594 uint32_t *olen);
2595
decode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2596 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2597 bool enh_check, uintptr_t ra,
2598 uint32_t *ochar, uint32_t *olen)
2599 {
2600 uint8_t s0, s1, s2, s3;
2601 uint32_t c, l;
2602
2603 if (ilen < 1) {
2604 return 0;
2605 }
2606 s0 = cpu_ldub_data_ra(env, addr, ra);
2607 if (s0 <= 0x7f) {
2608 /* one byte character */
2609 l = 1;
2610 c = s0;
2611 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2612 /* invalid character */
2613 return 2;
2614 } else if (s0 <= 0xdf) {
2615 /* two byte character */
2616 l = 2;
2617 if (ilen < 2) {
2618 return 0;
2619 }
2620 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2621 c = s0 & 0x1f;
2622 c = (c << 6) | (s1 & 0x3f);
2623 if (enh_check && (s1 & 0xc0) != 0x80) {
2624 return 2;
2625 }
2626 } else if (s0 <= 0xef) {
2627 /* three byte character */
2628 l = 3;
2629 if (ilen < 3) {
2630 return 0;
2631 }
2632 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2633 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2634 c = s0 & 0x0f;
2635 c = (c << 6) | (s1 & 0x3f);
2636 c = (c << 6) | (s2 & 0x3f);
2637 /* Fold the byte-by-byte range descriptions in the PoO into
2638 tests against the complete value. It disallows encodings
2639 that could be smaller, and the UTF-16 surrogates. */
2640 if (enh_check
2641 && ((s1 & 0xc0) != 0x80
2642 || (s2 & 0xc0) != 0x80
2643 || c < 0x1000
2644 || (c >= 0xd800 && c <= 0xdfff))) {
2645 return 2;
2646 }
2647 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2648 /* four byte character */
2649 l = 4;
2650 if (ilen < 4) {
2651 return 0;
2652 }
2653 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2654 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2655 s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2656 c = s0 & 0x07;
2657 c = (c << 6) | (s1 & 0x3f);
2658 c = (c << 6) | (s2 & 0x3f);
2659 c = (c << 6) | (s3 & 0x3f);
2660 /* See above. */
2661 if (enh_check
2662 && ((s1 & 0xc0) != 0x80
2663 || (s2 & 0xc0) != 0x80
2664 || (s3 & 0xc0) != 0x80
2665 || c < 0x010000
2666 || c > 0x10ffff)) {
2667 return 2;
2668 }
2669 } else {
2670 /* invalid character */
2671 return 2;
2672 }
2673
2674 *ochar = c;
2675 *olen = l;
2676 return -1;
2677 }
2678
decode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2679 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2680 bool enh_check, uintptr_t ra,
2681 uint32_t *ochar, uint32_t *olen)
2682 {
2683 uint16_t s0, s1;
2684 uint32_t c, l;
2685
2686 if (ilen < 2) {
2687 return 0;
2688 }
2689 s0 = cpu_lduw_data_ra(env, addr, ra);
2690 if ((s0 & 0xfc00) != 0xd800) {
2691 /* one word character */
2692 l = 2;
2693 c = s0;
2694 } else {
2695 /* two word character */
2696 l = 4;
2697 if (ilen < 4) {
2698 return 0;
2699 }
2700 s1 = cpu_lduw_data_ra(env, addr + 2, ra);
2701 c = extract32(s0, 6, 4) + 1;
2702 c = (c << 6) | (s0 & 0x3f);
2703 c = (c << 10) | (s1 & 0x3ff);
2704 if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2705 /* invalid surrogate character */
2706 return 2;
2707 }
2708 }
2709
2710 *ochar = c;
2711 *olen = l;
2712 return -1;
2713 }
2714
decode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,bool enh_check,uintptr_t ra,uint32_t * ochar,uint32_t * olen)2715 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2716 bool enh_check, uintptr_t ra,
2717 uint32_t *ochar, uint32_t *olen)
2718 {
2719 uint32_t c;
2720
2721 if (ilen < 4) {
2722 return 0;
2723 }
2724 c = cpu_ldl_data_ra(env, addr, ra);
2725 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2726 /* invalid unicode character */
2727 return 2;
2728 }
2729
2730 *ochar = c;
2731 *olen = 4;
2732 return -1;
2733 }
2734
encode_utf8(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2735 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2736 uintptr_t ra, uint32_t c, uint32_t *olen)
2737 {
2738 uint8_t d[4];
2739 uint32_t l, i;
2740
2741 if (c <= 0x7f) {
2742 /* one byte character */
2743 l = 1;
2744 d[0] = c;
2745 } else if (c <= 0x7ff) {
2746 /* two byte character */
2747 l = 2;
2748 d[1] = 0x80 | extract32(c, 0, 6);
2749 d[0] = 0xc0 | extract32(c, 6, 5);
2750 } else if (c <= 0xffff) {
2751 /* three byte character */
2752 l = 3;
2753 d[2] = 0x80 | extract32(c, 0, 6);
2754 d[1] = 0x80 | extract32(c, 6, 6);
2755 d[0] = 0xe0 | extract32(c, 12, 4);
2756 } else {
2757 /* four byte character */
2758 l = 4;
2759 d[3] = 0x80 | extract32(c, 0, 6);
2760 d[2] = 0x80 | extract32(c, 6, 6);
2761 d[1] = 0x80 | extract32(c, 12, 6);
2762 d[0] = 0xf0 | extract32(c, 18, 3);
2763 }
2764
2765 if (ilen < l) {
2766 return 1;
2767 }
2768 for (i = 0; i < l; ++i) {
2769 cpu_stb_data_ra(env, addr + i, d[i], ra);
2770 }
2771
2772 *olen = l;
2773 return -1;
2774 }
2775
encode_utf16(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2776 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2777 uintptr_t ra, uint32_t c, uint32_t *olen)
2778 {
2779 uint16_t d0, d1;
2780
2781 if (c <= 0xffff) {
2782 /* one word character */
2783 if (ilen < 2) {
2784 return 1;
2785 }
2786 cpu_stw_data_ra(env, addr, c, ra);
2787 *olen = 2;
2788 } else {
2789 /* two word character */
2790 if (ilen < 4) {
2791 return 1;
2792 }
2793 d1 = 0xdc00 | extract32(c, 0, 10);
2794 d0 = 0xd800 | extract32(c, 10, 6);
2795 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2796 cpu_stw_data_ra(env, addr + 0, d0, ra);
2797 cpu_stw_data_ra(env, addr + 2, d1, ra);
2798 *olen = 4;
2799 }
2800
2801 return -1;
2802 }
2803
encode_utf32(CPUS390XState * env,uint64_t addr,uint64_t ilen,uintptr_t ra,uint32_t c,uint32_t * olen)2804 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2805 uintptr_t ra, uint32_t c, uint32_t *olen)
2806 {
2807 if (ilen < 4) {
2808 return 1;
2809 }
2810 cpu_stl_data_ra(env, addr, c, ra);
2811 *olen = 4;
2812 return -1;
2813 }
2814
convert_unicode(CPUS390XState * env,uint32_t r1,uint32_t r2,uint32_t m3,uintptr_t ra,decode_unicode_fn decode,encode_unicode_fn encode)2815 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2816 uint32_t r2, uint32_t m3, uintptr_t ra,
2817 decode_unicode_fn decode,
2818 encode_unicode_fn encode)
2819 {
2820 uint64_t dst = get_address(env, r1);
2821 uint64_t dlen = get_length(env, r1 + 1);
2822 uint64_t src = get_address(env, r2);
2823 uint64_t slen = get_length(env, r2 + 1);
2824 bool enh_check = m3 & 1;
2825 int cc, i;
2826
2827 /* Lest we fail to service interrupts in a timely manner, limit the
2828 amount of work we're willing to do. For now, let's cap at 256. */
2829 for (i = 0; i < 256; ++i) {
2830 uint32_t c, ilen, olen;
2831
2832 cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2833 if (unlikely(cc >= 0)) {
2834 break;
2835 }
2836 cc = encode(env, dst, dlen, ra, c, &olen);
2837 if (unlikely(cc >= 0)) {
2838 break;
2839 }
2840
2841 src += ilen;
2842 slen -= ilen;
2843 dst += olen;
2844 dlen -= olen;
2845 cc = 3;
2846 }
2847
2848 set_address(env, r1, dst);
2849 set_length(env, r1 + 1, dlen);
2850 set_address(env, r2, src);
2851 set_length(env, r2 + 1, slen);
2852
2853 return cc;
2854 }
2855
HELPER(cu12)2856 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2857 {
2858 return convert_unicode(env, r1, r2, m3, GETPC(),
2859 decode_utf8, encode_utf16);
2860 }
2861
HELPER(cu14)2862 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2863 {
2864 return convert_unicode(env, r1, r2, m3, GETPC(),
2865 decode_utf8, encode_utf32);
2866 }
2867
HELPER(cu21)2868 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2869 {
2870 return convert_unicode(env, r1, r2, m3, GETPC(),
2871 decode_utf16, encode_utf8);
2872 }
2873
HELPER(cu24)2874 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2875 {
2876 return convert_unicode(env, r1, r2, m3, GETPC(),
2877 decode_utf16, encode_utf32);
2878 }
2879
HELPER(cu41)2880 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2881 {
2882 return convert_unicode(env, r1, r2, m3, GETPC(),
2883 decode_utf32, encode_utf8);
2884 }
2885
HELPER(cu42)2886 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2887 {
2888 return convert_unicode(env, r1, r2, m3, GETPC(),
2889 decode_utf32, encode_utf16);
2890 }
2891
probe_write_access(CPUS390XState * env,uint64_t addr,uint64_t len,uintptr_t ra)2892 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len,
2893 uintptr_t ra)
2894 {
2895 const int mmu_idx = s390x_env_mmu_index(env, false);
2896
2897 /* test the actual access, not just any access to the page due to LAP */
2898 while (len) {
2899 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK);
2900 const uint64_t curlen = MIN(pagelen, len);
2901
2902 probe_write(env, addr, curlen, mmu_idx, ra);
2903 addr = wrap_address(env, addr + curlen);
2904 len -= curlen;
2905 }
2906 }
2907
HELPER(probe_write_access)2908 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len)
2909 {
2910 probe_write_access(env, addr, len, GETPC());
2911 }
2912