1 /* 2 * QEMU TCG support -- s390x vector floating point instruction support 3 * 4 * Copyright (C) 2019 Red Hat Inc 5 * 6 * Authors: 7 * David Hildenbrand <david@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 #include "qemu/osdep.h" 13 #include "qemu-common.h" 14 #include "cpu.h" 15 #include "internal.h" 16 #include "vec.h" 17 #include "tcg_s390x.h" 18 #include "tcg/tcg-gvec-desc.h" 19 #include "exec/exec-all.h" 20 #include "exec/helper-proto.h" 21 #include "fpu/softfloat.h" 22 23 #define VIC_INVALID 0x1 24 #define VIC_DIVBYZERO 0x2 25 #define VIC_OVERFLOW 0x3 26 #define VIC_UNDERFLOW 0x4 27 #define VIC_INEXACT 0x5 28 29 /* returns the VEX. If the VEX is 0, there is no trap */ 30 static uint8_t check_ieee_exc(CPUS390XState *env, uint8_t enr, bool XxC, 31 uint8_t *vec_exc) 32 { 33 uint8_t vece_exc = 0, trap_exc; 34 unsigned qemu_exc; 35 36 /* Retrieve and clear the softfloat exceptions */ 37 qemu_exc = env->fpu_status.float_exception_flags; 38 if (qemu_exc == 0) { 39 return 0; 40 } 41 env->fpu_status.float_exception_flags = 0; 42 43 vece_exc = s390_softfloat_exc_to_ieee(qemu_exc); 44 45 /* Add them to the vector-wide s390x exception bits */ 46 *vec_exc |= vece_exc; 47 48 /* Check for traps and construct the VXC */ 49 trap_exc = vece_exc & env->fpc >> 24; 50 if (trap_exc) { 51 if (trap_exc & S390_IEEE_MASK_INVALID) { 52 return enr << 4 | VIC_INVALID; 53 } else if (trap_exc & S390_IEEE_MASK_DIVBYZERO) { 54 return enr << 4 | VIC_DIVBYZERO; 55 } else if (trap_exc & S390_IEEE_MASK_OVERFLOW) { 56 return enr << 4 | VIC_OVERFLOW; 57 } else if (trap_exc & S390_IEEE_MASK_UNDERFLOW) { 58 return enr << 4 | VIC_UNDERFLOW; 59 } else if (!XxC) { 60 g_assert(trap_exc & S390_IEEE_MASK_INEXACT); 61 /* inexact has lowest priority on traps */ 62 return enr << 4 | VIC_INEXACT; 63 } 64 } 65 return 0; 66 } 67 68 static void handle_ieee_exc(CPUS390XState *env, uint8_t vxc, uint8_t vec_exc, 69 uintptr_t retaddr) 70 { 71 if (vxc) { 72 /* on traps, the fpc flags are not updated, instruction is suppressed */ 73 tcg_s390_vector_exception(env, vxc, retaddr); 74 } 75 if (vec_exc) { 76 /* indicate exceptions for all elements combined */ 77 env->fpc |= vec_exc << 16; 78 } 79 } 80 81 static float64 s390_vec_read_float64(const S390Vector *v, uint8_t enr) 82 { 83 return make_float64(s390_vec_read_element64(v, enr)); 84 } 85 86 static void s390_vec_write_float64(S390Vector *v, uint8_t enr, float64 data) 87 { 88 return s390_vec_write_element64(v, enr, data); 89 } 90 91 typedef float64 (*vop64_2_fn)(float64 a, float_status *s); 92 static void vop64_2(S390Vector *v1, const S390Vector *v2, CPUS390XState *env, 93 bool s, bool XxC, uint8_t erm, vop64_2_fn fn, 94 uintptr_t retaddr) 95 { 96 uint8_t vxc, vec_exc = 0; 97 S390Vector tmp = {}; 98 int i, old_mode; 99 100 old_mode = s390_swap_bfp_rounding_mode(env, erm); 101 for (i = 0; i < 2; i++) { 102 const float64 a = s390_vec_read_float64(v2, i); 103 104 s390_vec_write_float64(&tmp, i, fn(a, &env->fpu_status)); 105 vxc = check_ieee_exc(env, i, XxC, &vec_exc); 106 if (s || vxc) { 107 break; 108 } 109 } 110 s390_restore_bfp_rounding_mode(env, old_mode); 111 handle_ieee_exc(env, vxc, vec_exc, retaddr); 112 *v1 = tmp; 113 } 114 115 static float64 vcdg64(float64 a, float_status *s) 116 { 117 return int64_to_float64(a, s); 118 } 119 120 static float64 vcdlg64(float64 a, float_status *s) 121 { 122 return uint64_to_float64(a, s); 123 } 124 125 static float64 vcgd64(float64 a, float_status *s) 126 { 127 const float64 tmp = float64_to_int64(a, s); 128 129 return float64_is_any_nan(a) ? INT64_MIN : tmp; 130 } 131 132 static float64 vclgd64(float64 a, float_status *s) 133 { 134 const float64 tmp = float64_to_uint64(a, s); 135 136 return float64_is_any_nan(a) ? 0 : tmp; 137 } 138 139 #define DEF_GVEC_VOP2_FN(NAME, FN, BITS) \ 140 void HELPER(gvec_##NAME##BITS)(void *v1, const void *v2, CPUS390XState *env, \ 141 uint32_t desc) \ 142 { \ 143 const uint8_t erm = extract32(simd_data(desc), 4, 4); \ 144 const bool se = extract32(simd_data(desc), 3, 1); \ 145 const bool XxC = extract32(simd_data(desc), 2, 1); \ 146 \ 147 vop##BITS##_2(v1, v2, env, se, XxC, erm, FN, GETPC()); \ 148 } 149 150 #define DEF_GVEC_VOP2_64(NAME) \ 151 DEF_GVEC_VOP2_FN(NAME, NAME##64, 64) 152 153 #define DEF_GVEC_VOP2(NAME, OP) \ 154 DEF_GVEC_VOP2_FN(NAME, float64_##OP, 64) 155 156 DEF_GVEC_VOP2_64(vcdg) 157 DEF_GVEC_VOP2_64(vcdlg) 158 DEF_GVEC_VOP2_64(vcgd) 159 DEF_GVEC_VOP2_64(vclgd) 160 DEF_GVEC_VOP2(vfi, round_to_int) 161 DEF_GVEC_VOP2(vfsq, sqrt) 162 163 typedef float64 (*vop64_3_fn)(float64 a, float64 b, float_status *s); 164 static void vop64_3(S390Vector *v1, const S390Vector *v2, const S390Vector *v3, 165 CPUS390XState *env, bool s, vop64_3_fn fn, 166 uintptr_t retaddr) 167 { 168 uint8_t vxc, vec_exc = 0; 169 S390Vector tmp = {}; 170 int i; 171 172 for (i = 0; i < 2; i++) { 173 const float64 a = s390_vec_read_float64(v2, i); 174 const float64 b = s390_vec_read_float64(v3, i); 175 176 s390_vec_write_float64(&tmp, i, fn(a, b, &env->fpu_status)); 177 vxc = check_ieee_exc(env, i, false, &vec_exc); 178 if (s || vxc) { 179 break; 180 } 181 } 182 handle_ieee_exc(env, vxc, vec_exc, retaddr); 183 *v1 = tmp; 184 } 185 186 #define DEF_GVEC_VOP3(NAME, OP) \ 187 void HELPER(gvec_##NAME##64)(void *v1, const void *v2, const void *v3, \ 188 CPUS390XState *env, uint32_t desc) \ 189 { \ 190 const bool se = extract32(simd_data(desc), 3, 1); \ 191 \ 192 vop64_3(v1, v2, v3, env, se, float64_##OP, GETPC()); \ 193 } 194 195 DEF_GVEC_VOP3(vfa, add) 196 DEF_GVEC_VOP3(vfs, sub) 197 DEF_GVEC_VOP3(vfd, div) 198 DEF_GVEC_VOP3(vfm, mul) 199 200 static int wfc64(const S390Vector *v1, const S390Vector *v2, 201 CPUS390XState *env, bool signal, uintptr_t retaddr) 202 { 203 /* only the zero-indexed elements are compared */ 204 const float64 a = s390_vec_read_element64(v1, 0); 205 const float64 b = s390_vec_read_element64(v2, 0); 206 uint8_t vxc, vec_exc = 0; 207 int cmp; 208 209 if (signal) { 210 cmp = float64_compare(a, b, &env->fpu_status); 211 } else { 212 cmp = float64_compare_quiet(a, b, &env->fpu_status); 213 } 214 vxc = check_ieee_exc(env, 0, false, &vec_exc); 215 handle_ieee_exc(env, vxc, vec_exc, retaddr); 216 217 return float_comp_to_cc(env, cmp); 218 } 219 220 void HELPER(gvec_wfc64)(const void *v1, const void *v2, CPUS390XState *env, 221 uint32_t desc) 222 { 223 env->cc_op = wfc64(v1, v2, env, false, GETPC()); 224 } 225 226 void HELPER(gvec_wfk64)(const void *v1, const void *v2, CPUS390XState *env, 227 uint32_t desc) 228 { 229 env->cc_op = wfc64(v1, v2, env, true, GETPC()); 230 } 231 232 typedef bool (*vfc64_fn)(float64 a, float64 b, float_status *status); 233 static int vfc64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3, 234 CPUS390XState *env, bool s, vfc64_fn fn, uintptr_t retaddr) 235 { 236 uint8_t vxc, vec_exc = 0; 237 S390Vector tmp = {}; 238 int match = 0; 239 int i; 240 241 for (i = 0; i < 2; i++) { 242 const float64 a = s390_vec_read_element64(v2, i); 243 const float64 b = s390_vec_read_element64(v3, i); 244 245 /* swap the order of the parameters, so we can use existing functions */ 246 if (fn(b, a, &env->fpu_status)) { 247 match++; 248 s390_vec_write_element64(&tmp, i, -1ull); 249 } 250 vxc = check_ieee_exc(env, i, false, &vec_exc); 251 if (s || vxc) { 252 break; 253 } 254 } 255 256 handle_ieee_exc(env, vxc, vec_exc, retaddr); 257 *v1 = tmp; 258 if (match) { 259 return s || match == 2 ? 0 : 1; 260 } 261 return 3; 262 } 263 264 void HELPER(gvec_vfce64)(void *v1, const void *v2, const void *v3, 265 CPUS390XState *env, uint32_t desc) 266 { 267 vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC()); 268 } 269 270 void HELPER(gvec_vfce64s)(void *v1, const void *v2, const void *v3, 271 CPUS390XState *env, uint32_t desc) 272 { 273 vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC()); 274 } 275 276 void HELPER(gvec_vfce64_cc)(void *v1, const void *v2, const void *v3, 277 CPUS390XState *env, uint32_t desc) 278 { 279 env->cc_op = vfc64(v1, v2, v3, env, false, float64_eq_quiet, GETPC()); 280 } 281 282 void HELPER(gvec_vfce64s_cc)(void *v1, const void *v2, const void *v3, 283 CPUS390XState *env, uint32_t desc) 284 { 285 env->cc_op = vfc64(v1, v2, v3, env, true, float64_eq_quiet, GETPC()); 286 } 287 288 void HELPER(gvec_vfch64)(void *v1, const void *v2, const void *v3, 289 CPUS390XState *env, uint32_t desc) 290 { 291 vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC()); 292 } 293 294 void HELPER(gvec_vfch64s)(void *v1, const void *v2, const void *v3, 295 CPUS390XState *env, uint32_t desc) 296 { 297 vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC()); 298 } 299 300 void HELPER(gvec_vfch64_cc)(void *v1, const void *v2, const void *v3, 301 CPUS390XState *env, uint32_t desc) 302 { 303 env->cc_op = vfc64(v1, v2, v3, env, false, float64_lt_quiet, GETPC()); 304 } 305 306 void HELPER(gvec_vfch64s_cc)(void *v1, const void *v2, const void *v3, 307 CPUS390XState *env, uint32_t desc) 308 { 309 env->cc_op = vfc64(v1, v2, v3, env, true, float64_lt_quiet, GETPC()); 310 } 311 312 void HELPER(gvec_vfche64)(void *v1, const void *v2, const void *v3, 313 CPUS390XState *env, uint32_t desc) 314 { 315 vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC()); 316 } 317 318 void HELPER(gvec_vfche64s)(void *v1, const void *v2, const void *v3, 319 CPUS390XState *env, uint32_t desc) 320 { 321 vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC()); 322 } 323 324 void HELPER(gvec_vfche64_cc)(void *v1, const void *v2, const void *v3, 325 CPUS390XState *env, uint32_t desc) 326 { 327 env->cc_op = vfc64(v1, v2, v3, env, false, float64_le_quiet, GETPC()); 328 } 329 330 void HELPER(gvec_vfche64s_cc)(void *v1, const void *v2, const void *v3, 331 CPUS390XState *env, uint32_t desc) 332 { 333 env->cc_op = vfc64(v1, v2, v3, env, true, float64_le_quiet, GETPC()); 334 } 335 336 static void vfll32(S390Vector *v1, const S390Vector *v2, CPUS390XState *env, 337 bool s, uintptr_t retaddr) 338 { 339 uint8_t vxc, vec_exc = 0; 340 S390Vector tmp = {}; 341 int i; 342 343 for (i = 0; i < 2; i++) { 344 /* load from even element */ 345 const float32 a = s390_vec_read_element32(v2, i * 2); 346 const uint64_t ret = float32_to_float64(a, &env->fpu_status); 347 348 s390_vec_write_element64(&tmp, i, ret); 349 /* indicate the source element */ 350 vxc = check_ieee_exc(env, i * 2, false, &vec_exc); 351 if (s || vxc) { 352 break; 353 } 354 } 355 handle_ieee_exc(env, vxc, vec_exc, retaddr); 356 *v1 = tmp; 357 } 358 359 void HELPER(gvec_vfll32)(void *v1, const void *v2, CPUS390XState *env, 360 uint32_t desc) 361 { 362 vfll32(v1, v2, env, false, GETPC()); 363 } 364 365 void HELPER(gvec_vfll32s)(void *v1, const void *v2, CPUS390XState *env, 366 uint32_t desc) 367 { 368 vfll32(v1, v2, env, true, GETPC()); 369 } 370 371 static void vflr64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env, 372 bool s, bool XxC, uint8_t erm, uintptr_t retaddr) 373 { 374 uint8_t vxc, vec_exc = 0; 375 S390Vector tmp = {}; 376 int i, old_mode; 377 378 old_mode = s390_swap_bfp_rounding_mode(env, erm); 379 for (i = 0; i < 2; i++) { 380 float64 a = s390_vec_read_element64(v2, i); 381 uint32_t ret = float64_to_float32(a, &env->fpu_status); 382 383 /* place at even element */ 384 s390_vec_write_element32(&tmp, i * 2, ret); 385 /* indicate the source element */ 386 vxc = check_ieee_exc(env, i, XxC, &vec_exc); 387 if (s || vxc) { 388 break; 389 } 390 } 391 s390_restore_bfp_rounding_mode(env, old_mode); 392 handle_ieee_exc(env, vxc, vec_exc, retaddr); 393 *v1 = tmp; 394 } 395 396 void HELPER(gvec_vflr64)(void *v1, const void *v2, CPUS390XState *env, 397 uint32_t desc) 398 { 399 const uint8_t erm = extract32(simd_data(desc), 4, 4); 400 const bool XxC = extract32(simd_data(desc), 2, 1); 401 402 vflr64(v1, v2, env, false, XxC, erm, GETPC()); 403 } 404 405 void HELPER(gvec_vflr64s)(void *v1, const void *v2, CPUS390XState *env, 406 uint32_t desc) 407 { 408 const uint8_t erm = extract32(simd_data(desc), 4, 4); 409 const bool XxC = extract32(simd_data(desc), 2, 1); 410 411 vflr64(v1, v2, env, true, XxC, erm, GETPC()); 412 } 413 414 static void vfma64(S390Vector *v1, const S390Vector *v2, const S390Vector *v3, 415 const S390Vector *v4, CPUS390XState *env, bool s, int flags, 416 uintptr_t retaddr) 417 { 418 uint8_t vxc, vec_exc = 0; 419 S390Vector tmp = {}; 420 int i; 421 422 for (i = 0; i < 2; i++) { 423 const uint64_t a = s390_vec_read_element64(v2, i); 424 const uint64_t b = s390_vec_read_element64(v3, i); 425 const uint64_t c = s390_vec_read_element64(v4, i); 426 uint64_t ret = float64_muladd(a, b, c, flags, &env->fpu_status); 427 428 s390_vec_write_element64(&tmp, i, ret); 429 vxc = check_ieee_exc(env, i, false, &vec_exc); 430 if (s || vxc) { 431 break; 432 } 433 } 434 handle_ieee_exc(env, vxc, vec_exc, retaddr); 435 *v1 = tmp; 436 } 437 438 void HELPER(gvec_vfma64)(void *v1, const void *v2, const void *v3, 439 const void *v4, CPUS390XState *env, uint32_t desc) 440 { 441 vfma64(v1, v2, v3, v4, env, false, 0, GETPC()); 442 } 443 444 void HELPER(gvec_vfma64s)(void *v1, const void *v2, const void *v3, 445 const void *v4, CPUS390XState *env, uint32_t desc) 446 { 447 vfma64(v1, v2, v3, v4, env, true, 0, GETPC()); 448 } 449 450 void HELPER(gvec_vfms64)(void *v1, const void *v2, const void *v3, 451 const void *v4, CPUS390XState *env, uint32_t desc) 452 { 453 vfma64(v1, v2, v3, v4, env, false, float_muladd_negate_c, GETPC()); 454 } 455 456 void HELPER(gvec_vfms64s)(void *v1, const void *v2, const void *v3, 457 const void *v4, CPUS390XState *env, uint32_t desc) 458 { 459 vfma64(v1, v2, v3, v4, env, true, float_muladd_negate_c, GETPC()); 460 } 461 462 static int vftci64(S390Vector *v1, const S390Vector *v2, CPUS390XState *env, 463 bool s, uint16_t i3) 464 { 465 int i, match = 0; 466 467 for (i = 0; i < 2; i++) { 468 float64 a = s390_vec_read_element64(v2, i); 469 470 if (float64_dcmask(env, a) & i3) { 471 match++; 472 s390_vec_write_element64(v1, i, -1ull); 473 } else { 474 s390_vec_write_element64(v1, i, 0); 475 } 476 if (s) { 477 break; 478 } 479 } 480 481 if (match) { 482 return s || match == 2 ? 0 : 1; 483 } 484 return 3; 485 } 486 487 void HELPER(gvec_vftci64)(void *v1, const void *v2, CPUS390XState *env, 488 uint32_t desc) 489 { 490 env->cc_op = vftci64(v1, v2, env, false, simd_data(desc)); 491 } 492 493 void HELPER(gvec_vftci64s)(void *v1, const void *v2, CPUS390XState *env, 494 uint32_t desc) 495 { 496 env->cc_op = vftci64(v1, v2, env, true, simd_data(desc)); 497 } 498