1 /* 2 * crypto_helper.c - emulate v8 Crypto Extensions instructions 3 * 4 * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 */ 11 12 #include "qemu/osdep.h" 13 14 #include "cpu.h" 15 #include "exec/helper-proto.h" 16 #include "tcg/tcg-gvec-desc.h" 17 #include "crypto/aes.h" 18 #include "vec_internal.h" 19 20 union CRYPTO_STATE { 21 uint8_t bytes[16]; 22 uint32_t words[4]; 23 uint64_t l[2]; 24 }; 25 26 #ifdef HOST_WORDS_BIGENDIAN 27 #define CR_ST_BYTE(state, i) ((state).bytes[(15 - (i)) ^ 8]) 28 #define CR_ST_WORD(state, i) ((state).words[(3 - (i)) ^ 2]) 29 #else 30 #define CR_ST_BYTE(state, i) ((state).bytes[i]) 31 #define CR_ST_WORD(state, i) ((state).words[i]) 32 #endif 33 34 /* 35 * The caller has not been converted to full gvec, and so only 36 * modifies the low 16 bytes of the vector register. 37 */ 38 static void clear_tail_16(void *vd, uint32_t desc) 39 { 40 int opr_sz = simd_oprsz(desc); 41 int max_sz = simd_maxsz(desc); 42 43 assert(opr_sz == 16); 44 clear_tail(vd, opr_sz, max_sz); 45 } 46 47 static void do_crypto_aese(uint64_t *rd, uint64_t *rn, 48 uint64_t *rm, bool decrypt) 49 { 50 static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox }; 51 static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts }; 52 union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } }; 53 union CRYPTO_STATE st = { .l = { rn[0], rn[1] } }; 54 int i; 55 56 /* xor state vector with round key */ 57 rk.l[0] ^= st.l[0]; 58 rk.l[1] ^= st.l[1]; 59 60 /* combine ShiftRows operation and sbox substitution */ 61 for (i = 0; i < 16; i++) { 62 CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])]; 63 } 64 65 rd[0] = st.l[0]; 66 rd[1] = st.l[1]; 67 } 68 69 void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc) 70 { 71 intptr_t i, opr_sz = simd_oprsz(desc); 72 bool decrypt = simd_data(desc); 73 74 for (i = 0; i < opr_sz; i += 16) { 75 do_crypto_aese(vd + i, vn + i, vm + i, decrypt); 76 } 77 clear_tail(vd, opr_sz, simd_maxsz(desc)); 78 } 79 80 static void do_crypto_aesmc(uint64_t *rd, uint64_t *rm, bool decrypt) 81 { 82 static uint32_t const mc[][256] = { { 83 /* MixColumns lookup table */ 84 0x00000000, 0x03010102, 0x06020204, 0x05030306, 85 0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e, 86 0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16, 87 0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e, 88 0x30101020, 0x33111122, 0x36121224, 0x35131326, 89 0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e, 90 0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36, 91 0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e, 92 0x60202040, 0x63212142, 0x66222244, 0x65232346, 93 0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e, 94 0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56, 95 0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e, 96 0x50303060, 0x53313162, 0x56323264, 0x55333366, 97 0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e, 98 0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76, 99 0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e, 100 0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386, 101 0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e, 102 0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96, 103 0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e, 104 0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6, 105 0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae, 106 0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6, 107 0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe, 108 0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6, 109 0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce, 110 0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6, 111 0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde, 112 0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6, 113 0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee, 114 0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6, 115 0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe, 116 0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d, 117 0x97848413, 0x94858511, 0x91868617, 0x92878715, 118 0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d, 119 0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05, 120 0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d, 121 0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735, 122 0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d, 123 0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25, 124 0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d, 125 0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755, 126 0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d, 127 0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45, 128 0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d, 129 0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775, 130 0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d, 131 0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65, 132 0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d, 133 0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795, 134 0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d, 135 0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85, 136 0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd, 137 0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5, 138 0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad, 139 0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5, 140 0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd, 141 0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5, 142 0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd, 143 0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5, 144 0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd, 145 0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5, 146 0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed, 147 0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5, 148 }, { 149 /* Inverse MixColumns lookup table */ 150 0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12, 151 0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a, 152 0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362, 153 0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a, 154 0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2, 155 0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca, 156 0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382, 157 0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba, 158 0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9, 159 0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1, 160 0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9, 161 0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81, 162 0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029, 163 0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411, 164 0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859, 165 0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61, 166 0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf, 167 0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987, 168 0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf, 169 0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7, 170 0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f, 171 0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967, 172 0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f, 173 0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117, 174 0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664, 175 0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c, 176 0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14, 177 0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c, 178 0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684, 179 0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc, 180 0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4, 181 0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc, 182 0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753, 183 0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b, 184 0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23, 185 0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b, 186 0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3, 187 0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b, 188 0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3, 189 0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb, 190 0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88, 191 0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0, 192 0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8, 193 0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0, 194 0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68, 195 0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850, 196 0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418, 197 0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020, 198 0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe, 199 0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6, 200 0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e, 201 0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6, 202 0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e, 203 0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526, 204 0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e, 205 0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56, 206 0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25, 207 0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d, 208 0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255, 209 0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d, 210 0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5, 211 0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd, 212 0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5, 213 0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d, 214 } }; 215 216 union CRYPTO_STATE st = { .l = { rm[0], rm[1] } }; 217 int i; 218 219 for (i = 0; i < 16; i += 4) { 220 CR_ST_WORD(st, i >> 2) = 221 mc[decrypt][CR_ST_BYTE(st, i)] ^ 222 rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^ 223 rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^ 224 rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24); 225 } 226 227 rd[0] = st.l[0]; 228 rd[1] = st.l[1]; 229 } 230 231 void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc) 232 { 233 intptr_t i, opr_sz = simd_oprsz(desc); 234 bool decrypt = simd_data(desc); 235 236 for (i = 0; i < opr_sz; i += 16) { 237 do_crypto_aesmc(vd + i, vm + i, decrypt); 238 } 239 clear_tail(vd, opr_sz, simd_maxsz(desc)); 240 } 241 242 /* 243 * SHA-1 logical functions 244 */ 245 246 static uint32_t cho(uint32_t x, uint32_t y, uint32_t z) 247 { 248 return (x & (y ^ z)) ^ z; 249 } 250 251 static uint32_t par(uint32_t x, uint32_t y, uint32_t z) 252 { 253 return x ^ y ^ z; 254 } 255 256 static uint32_t maj(uint32_t x, uint32_t y, uint32_t z) 257 { 258 return (x & y) | ((x | y) & z); 259 } 260 261 void HELPER(crypto_sha1su0)(void *vd, void *vn, void *vm, uint32_t desc) 262 { 263 uint64_t *d = vd, *n = vn, *m = vm; 264 uint64_t d0, d1; 265 266 d0 = d[1] ^ d[0] ^ m[0]; 267 d1 = n[0] ^ d[1] ^ m[1]; 268 d[0] = d0; 269 d[1] = d1; 270 271 clear_tail_16(vd, desc); 272 } 273 274 static inline void crypto_sha1_3reg(uint64_t *rd, uint64_t *rn, 275 uint64_t *rm, uint32_t desc, 276 uint32_t (*fn)(union CRYPTO_STATE *d)) 277 { 278 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 279 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 280 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 281 int i; 282 283 for (i = 0; i < 4; i++) { 284 uint32_t t = fn(&d); 285 286 t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0) 287 + CR_ST_WORD(m, i); 288 289 CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3); 290 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2); 291 CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2); 292 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0); 293 CR_ST_WORD(d, 0) = t; 294 } 295 rd[0] = d.l[0]; 296 rd[1] = d.l[1]; 297 298 clear_tail_16(rd, desc); 299 } 300 301 static uint32_t do_sha1c(union CRYPTO_STATE *d) 302 { 303 return cho(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3)); 304 } 305 306 void HELPER(crypto_sha1c)(void *vd, void *vn, void *vm, uint32_t desc) 307 { 308 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1c); 309 } 310 311 static uint32_t do_sha1p(union CRYPTO_STATE *d) 312 { 313 return par(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3)); 314 } 315 316 void HELPER(crypto_sha1p)(void *vd, void *vn, void *vm, uint32_t desc) 317 { 318 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1p); 319 } 320 321 static uint32_t do_sha1m(union CRYPTO_STATE *d) 322 { 323 return maj(CR_ST_WORD(*d, 1), CR_ST_WORD(*d, 2), CR_ST_WORD(*d, 3)); 324 } 325 326 void HELPER(crypto_sha1m)(void *vd, void *vn, void *vm, uint32_t desc) 327 { 328 crypto_sha1_3reg(vd, vn, vm, desc, do_sha1m); 329 } 330 331 void HELPER(crypto_sha1h)(void *vd, void *vm, uint32_t desc) 332 { 333 uint64_t *rd = vd; 334 uint64_t *rm = vm; 335 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 336 337 CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2); 338 CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0; 339 340 rd[0] = m.l[0]; 341 rd[1] = m.l[1]; 342 343 clear_tail_16(vd, desc); 344 } 345 346 void HELPER(crypto_sha1su1)(void *vd, void *vm, uint32_t desc) 347 { 348 uint64_t *rd = vd; 349 uint64_t *rm = vm; 350 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 351 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 352 353 CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1); 354 CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1); 355 CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1); 356 CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1); 357 358 rd[0] = d.l[0]; 359 rd[1] = d.l[1]; 360 361 clear_tail_16(vd, desc); 362 } 363 364 /* 365 * The SHA-256 logical functions, according to 366 * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf 367 */ 368 369 static uint32_t S0(uint32_t x) 370 { 371 return ror32(x, 2) ^ ror32(x, 13) ^ ror32(x, 22); 372 } 373 374 static uint32_t S1(uint32_t x) 375 { 376 return ror32(x, 6) ^ ror32(x, 11) ^ ror32(x, 25); 377 } 378 379 static uint32_t s0(uint32_t x) 380 { 381 return ror32(x, 7) ^ ror32(x, 18) ^ (x >> 3); 382 } 383 384 static uint32_t s1(uint32_t x) 385 { 386 return ror32(x, 17) ^ ror32(x, 19) ^ (x >> 10); 387 } 388 389 void HELPER(crypto_sha256h)(void *vd, void *vn, void *vm, uint32_t desc) 390 { 391 uint64_t *rd = vd; 392 uint64_t *rn = vn; 393 uint64_t *rm = vm; 394 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 395 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 396 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 397 int i; 398 399 for (i = 0; i < 4; i++) { 400 uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2)) 401 + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0)) 402 + CR_ST_WORD(m, i); 403 404 CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2); 405 CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1); 406 CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0); 407 CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t; 408 409 t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2)) 410 + S0(CR_ST_WORD(d, 0)); 411 412 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2); 413 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1); 414 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0); 415 CR_ST_WORD(d, 0) = t; 416 } 417 418 rd[0] = d.l[0]; 419 rd[1] = d.l[1]; 420 421 clear_tail_16(vd, desc); 422 } 423 424 void HELPER(crypto_sha256h2)(void *vd, void *vn, void *vm, uint32_t desc) 425 { 426 uint64_t *rd = vd; 427 uint64_t *rn = vn; 428 uint64_t *rm = vm; 429 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 430 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 431 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 432 int i; 433 434 for (i = 0; i < 4; i++) { 435 uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2)) 436 + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0)) 437 + CR_ST_WORD(m, i); 438 439 CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2); 440 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1); 441 CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0); 442 CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t; 443 } 444 445 rd[0] = d.l[0]; 446 rd[1] = d.l[1]; 447 448 clear_tail_16(vd, desc); 449 } 450 451 void HELPER(crypto_sha256su0)(void *vd, void *vm, uint32_t desc) 452 { 453 uint64_t *rd = vd; 454 uint64_t *rm = vm; 455 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 456 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 457 458 CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1)); 459 CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2)); 460 CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3)); 461 CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0)); 462 463 rd[0] = d.l[0]; 464 rd[1] = d.l[1]; 465 466 clear_tail_16(vd, desc); 467 } 468 469 void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm, uint32_t desc) 470 { 471 uint64_t *rd = vd; 472 uint64_t *rn = vn; 473 uint64_t *rm = vm; 474 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 475 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 476 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 477 478 CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1); 479 CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2); 480 CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3); 481 CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0); 482 483 rd[0] = d.l[0]; 484 rd[1] = d.l[1]; 485 486 clear_tail_16(vd, desc); 487 } 488 489 /* 490 * The SHA-512 logical functions (same as above but using 64-bit operands) 491 */ 492 493 static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z) 494 { 495 return (x & (y ^ z)) ^ z; 496 } 497 498 static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z) 499 { 500 return (x & y) | ((x | y) & z); 501 } 502 503 static uint64_t S0_512(uint64_t x) 504 { 505 return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39); 506 } 507 508 static uint64_t S1_512(uint64_t x) 509 { 510 return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41); 511 } 512 513 static uint64_t s0_512(uint64_t x) 514 { 515 return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7); 516 } 517 518 static uint64_t s1_512(uint64_t x) 519 { 520 return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6); 521 } 522 523 void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm, uint32_t desc) 524 { 525 uint64_t *rd = vd; 526 uint64_t *rn = vn; 527 uint64_t *rm = vm; 528 uint64_t d0 = rd[0]; 529 uint64_t d1 = rd[1]; 530 531 d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]); 532 d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]); 533 534 rd[0] = d0; 535 rd[1] = d1; 536 537 clear_tail_16(vd, desc); 538 } 539 540 void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm, uint32_t desc) 541 { 542 uint64_t *rd = vd; 543 uint64_t *rn = vn; 544 uint64_t *rm = vm; 545 uint64_t d0 = rd[0]; 546 uint64_t d1 = rd[1]; 547 548 d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]); 549 d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]); 550 551 rd[0] = d0; 552 rd[1] = d1; 553 554 clear_tail_16(vd, desc); 555 } 556 557 void HELPER(crypto_sha512su0)(void *vd, void *vn, uint32_t desc) 558 { 559 uint64_t *rd = vd; 560 uint64_t *rn = vn; 561 uint64_t d0 = rd[0]; 562 uint64_t d1 = rd[1]; 563 564 d0 += s0_512(rd[1]); 565 d1 += s0_512(rn[0]); 566 567 rd[0] = d0; 568 rd[1] = d1; 569 570 clear_tail_16(vd, desc); 571 } 572 573 void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm, uint32_t desc) 574 { 575 uint64_t *rd = vd; 576 uint64_t *rn = vn; 577 uint64_t *rm = vm; 578 579 rd[0] += s1_512(rn[0]) + rm[0]; 580 rd[1] += s1_512(rn[1]) + rm[1]; 581 582 clear_tail_16(vd, desc); 583 } 584 585 void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm, uint32_t desc) 586 { 587 uint64_t *rd = vd; 588 uint64_t *rn = vn; 589 uint64_t *rm = vm; 590 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 591 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 592 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 593 uint32_t t; 594 595 t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17); 596 CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9); 597 598 t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17); 599 CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9); 600 601 t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17); 602 CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9); 603 604 t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17); 605 CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9); 606 607 rd[0] = d.l[0]; 608 rd[1] = d.l[1]; 609 610 clear_tail_16(vd, desc); 611 } 612 613 void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm, uint32_t desc) 614 { 615 uint64_t *rd = vd; 616 uint64_t *rn = vn; 617 uint64_t *rm = vm; 618 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 619 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 620 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 621 uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25); 622 623 CR_ST_WORD(d, 0) ^= t; 624 CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25); 625 CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25); 626 CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^ 627 ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26); 628 629 rd[0] = d.l[0]; 630 rd[1] = d.l[1]; 631 632 clear_tail_16(vd, desc); 633 } 634 635 static inline void QEMU_ALWAYS_INLINE 636 crypto_sm3tt(uint64_t *rd, uint64_t *rn, uint64_t *rm, 637 uint32_t desc, uint32_t opcode) 638 { 639 union CRYPTO_STATE d = { .l = { rd[0], rd[1] } }; 640 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 641 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 642 uint32_t imm2 = simd_data(desc); 643 uint32_t t; 644 645 assert(imm2 < 4); 646 647 if (opcode == 0 || opcode == 2) { 648 /* SM3TT1A, SM3TT2A */ 649 t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); 650 } else if (opcode == 1) { 651 /* SM3TT1B */ 652 t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); 653 } else if (opcode == 3) { 654 /* SM3TT2B */ 655 t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1)); 656 } else { 657 qemu_build_not_reached(); 658 } 659 660 t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2); 661 662 CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1); 663 664 if (opcode < 2) { 665 /* SM3TT1A, SM3TT1B */ 666 t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20); 667 668 CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23); 669 } else { 670 /* SM3TT2A, SM3TT2B */ 671 t += CR_ST_WORD(n, 3); 672 t ^= rol32(t, 9) ^ rol32(t, 17); 673 674 CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13); 675 } 676 677 CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3); 678 CR_ST_WORD(d, 3) = t; 679 680 rd[0] = d.l[0]; 681 rd[1] = d.l[1]; 682 683 clear_tail_16(rd, desc); 684 } 685 686 #define DO_SM3TT(NAME, OPCODE) \ 687 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ 688 { crypto_sm3tt(vd, vn, vm, desc, OPCODE); } 689 690 DO_SM3TT(crypto_sm3tt1a, 0) 691 DO_SM3TT(crypto_sm3tt1b, 1) 692 DO_SM3TT(crypto_sm3tt2a, 2) 693 DO_SM3TT(crypto_sm3tt2b, 3) 694 695 #undef DO_SM3TT 696 697 static uint8_t const sm4_sbox[] = { 698 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, 699 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, 700 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, 701 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, 702 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, 703 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, 704 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, 705 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, 706 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, 707 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, 708 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, 709 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, 710 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, 711 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, 712 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, 713 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, 714 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, 715 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, 716 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, 717 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, 718 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, 719 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, 720 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, 721 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, 722 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, 723 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, 724 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, 725 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, 726 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, 727 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, 728 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, 729 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48, 730 }; 731 732 static void do_crypto_sm4e(uint64_t *rd, uint64_t *rn, uint64_t *rm) 733 { 734 union CRYPTO_STATE d = { .l = { rn[0], rn[1] } }; 735 union CRYPTO_STATE n = { .l = { rm[0], rm[1] } }; 736 uint32_t t, i; 737 738 for (i = 0; i < 4; i++) { 739 t = CR_ST_WORD(d, (i + 1) % 4) ^ 740 CR_ST_WORD(d, (i + 2) % 4) ^ 741 CR_ST_WORD(d, (i + 3) % 4) ^ 742 CR_ST_WORD(n, i); 743 744 t = sm4_sbox[t & 0xff] | 745 sm4_sbox[(t >> 8) & 0xff] << 8 | 746 sm4_sbox[(t >> 16) & 0xff] << 16 | 747 sm4_sbox[(t >> 24) & 0xff] << 24; 748 749 CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^ 750 rol32(t, 24); 751 } 752 753 rd[0] = d.l[0]; 754 rd[1] = d.l[1]; 755 } 756 757 void HELPER(crypto_sm4e)(void *vd, void *vn, void *vm, uint32_t desc) 758 { 759 intptr_t i, opr_sz = simd_oprsz(desc); 760 761 for (i = 0; i < opr_sz; i += 16) { 762 do_crypto_sm4e(vd + i, vn + i, vm + i); 763 } 764 clear_tail(vd, opr_sz, simd_maxsz(desc)); 765 } 766 767 static void do_crypto_sm4ekey(uint64_t *rd, uint64_t *rn, uint64_t *rm) 768 { 769 union CRYPTO_STATE d; 770 union CRYPTO_STATE n = { .l = { rn[0], rn[1] } }; 771 union CRYPTO_STATE m = { .l = { rm[0], rm[1] } }; 772 uint32_t t, i; 773 774 d = n; 775 for (i = 0; i < 4; i++) { 776 t = CR_ST_WORD(d, (i + 1) % 4) ^ 777 CR_ST_WORD(d, (i + 2) % 4) ^ 778 CR_ST_WORD(d, (i + 3) % 4) ^ 779 CR_ST_WORD(m, i); 780 781 t = sm4_sbox[t & 0xff] | 782 sm4_sbox[(t >> 8) & 0xff] << 8 | 783 sm4_sbox[(t >> 16) & 0xff] << 16 | 784 sm4_sbox[(t >> 24) & 0xff] << 24; 785 786 CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23); 787 } 788 789 rd[0] = d.l[0]; 790 rd[1] = d.l[1]; 791 } 792 793 void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm, uint32_t desc) 794 { 795 intptr_t i, opr_sz = simd_oprsz(desc); 796 797 for (i = 0; i < opr_sz; i += 16) { 798 do_crypto_sm4ekey(vd + i, vn + i, vm + i); 799 } 800 clear_tail(vd, opr_sz, simd_maxsz(desc)); 801 } 802 803 void HELPER(crypto_rax1)(void *vd, void *vn, void *vm, uint32_t desc) 804 { 805 intptr_t i, opr_sz = simd_oprsz(desc); 806 uint64_t *d = vd, *n = vn, *m = vm; 807 808 for (i = 0; i < opr_sz / 8; ++i) { 809 d[i] = n[i] ^ rol64(m[i], 1); 810 } 811 clear_tail(vd, opr_sz, simd_maxsz(desc)); 812 } 813