arm/crypto/crct10dif-ce-core.S

2 // Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions
14 // Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions
62 //  /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
75 	.arch		armv8-a
76 	.fpu		crypto-neon-fp-armv8
118 	vld1.64		{q11-q12}, [buf]!
166 	// the bit order match the polynomial coefficient order.
167 	vld1.64		{q0-q1}, [buf]!
168 	vld1.64		{q2-q3}, [buf]!
169 	vld1.64		{q4-q5}, [buf]!
170 	vld1.64		{q6-q7}, [buf]!
200 	// While >= 128 data bytes remain (not counting q0-q7), fold the 128
201 	// bytes q0-q7 into them, storing the result back into q0-q7.
210 	// Now fold the 112 bytes in q0-q6 into the 16 bytes in q7.
228 	adds		len, len, #(128-16)
264 	// q1 = high order part of second chunk: q7 left-shifted by 'len' bytes.
268 	vtbl.8		q1l, {q7l-q7h}, q2l
269 	vtbl.8		q1h, {q7l-q7h}, q2h
271 	// q3 = first chunk: q7 right-shifted by '16-len' bytes.
274 	vtbl.8		q3l, {q7l-q7h}, q2l
275 	vtbl.8		q3h, {q7l-q7h}, q2h
277 	// Convert to 8-bit masks: 'len' 0x00 bytes, then '16-len' 0xff bytes.
280 	// q2 = second chunk: 'len' bytes from q0 (low-order bytes),
281 	// then '16-len' bytes from q1 (high-order bytes).
291 	// Reduce the 128-bit value M(x), stored in q7, to the final 16-bit CRC.
293 	// Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'.
297 	// x^64.  This produces a 128-bit value congruent to x^64 * M(x) and
298 	// whose low 48 bits are 0.
299 	vmull.p64	q0, q7h, FOLD_CONST_H	// high bits * x^48 * (x^80 mod G(x))
302 	// Fold the high 32 bits into the low 96 bits.  This produces a 96-bit
303 	// value congruent to x^64 * M(x) and whose low 48 bits are 0.
307 	vmull.p64	q1, q1l, FOLD_CONST_L	// high 32 bits * x^48 * (x^48 mod G(x))
310 	// Load G(x) and floor(x^48 / G(x)).
314 	vmull.p64	q1, q0h, FOLD_CONST_H	// high 32 bits * floor(x^48 / G(x))
317 	vshr.u64	q0l, q0l, #48
339 	// Load the fold-across-16-bytes constants.
368 	.quad		0x1368000000000000	// x^48 * (x^48 mod G(x))
369 	.quad		0x2d56000000000000	// x^48 * (x^80 mod G(x))
372 	.quad		0x00000001f65a57f8	// floor(x^48 / G(x))
374 // For 1 <= len <= 15, the 16-byte vector beginning at &byteshift_table[16 -
376 // ..., 0x80} XOR the index vector to shift right by '16 - len' bytes.