xref: /src/lib/libmd/amd64/sha1block.S (revision ec3242ed1906e77c9af2c54da636833a946c62b6)

/*-
 * Copyright (c) 2013 The Go Authors. All rights reserved.
 * Copyright (c) 2024 Robert Clausecker <fuz@FreeBSD.org>
 *
 * Adapted from Go's crypto/sha1/sha1block_amd64.s.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *   * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *   * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <machine/asm.h>

/*
 * SHA-1 block routine. See sha1c.c for C equivalent.
 *
 * There are 80 rounds of 4 types:
 *   - rounds 0-15 are type 1 and load data (round1 macro).
 *   - rounds 16-19 are type 1 and do not load data (round1x macro).
 *   - rounds 20-39 are type 2 and do not load data (round2 macro).
 *   - rounds 40-59 are type 3 and do not load data (round3 macro).
 *   - rounds 60-79 are type 4 and do not load data (round4 macro).
 *
 * Each round loads or shuffles the data, then computes a per-round
 * function of b, c, d, and then mixes the result into and rotates the
 * five registers a, b, c, d, e holding the intermediate results.
 *
 * The register rotation is implemented by rotating the arguments to
 * the round macros instead of by explicit move instructions.
 */
.macro	load		index
	mov		(\index)*4(%rsi), %r10d
	bswap		%r10d
	mov		%r10d, (\index)*4(%rsp)
.endm

.macro	shuffle		index
	mov		((\index   )&0xf)*4(%rsp), %r10d
	xor		((\index- 3)&0xf)*4(%rsp), %r10d
	xor		((\index- 8)&0xf)*4(%rsp), %r10d
	xor		((\index-14)&0xf)*4(%rsp), %r10d
	rol		$1, %r10d
	mov		%r10d, ((\index)&0xf)*4(%rsp)
.endm

.macro	func1		a, b, c, d, e
	mov		\d, %r9d
	xor		\c, %r9d
	and		\b, %r9d
	xor		\d, %r9d
.endm

.macro	func2		a, b, c, d, e
	mov		\b, %r9d
	xor		\c, %r9d
	xor		\d, %r9d
.endm

.macro	func3		a, b, c, d, e
	mov		\b, %r8d
	or		\c, %r8d
	and		\d, %r8d
	mov		\b, %r9d
	and		\c, %r9d
	or		%r8d, %r9d
.endm

.macro	func4		a, b, c, d, e
	func2		\a, \b, \c, \d, \e
.endm

.macro	mix		a, b, c, d, e, const
	rol		$30, \b
	add		%r9d, \e
	mov		\a, %r8d
	rol		$5, %r8d
	lea		\const(\e, %r10d, 1), \e
	add		%r8d, \e
.endm

.macro	round1		a, b, c, d, e, index
	load		\index
	func1		\a, \b, \c, \d, \e
	mix		\a, \b, \c, \d, \e, 0x5a827999
.endm

.macro	round1x		a, b, c, d, e, index
	shuffle		\index
	func1		\a, \b, \c, \d, \e
	mix		\a, \b, \c, \d, \e, 0x5a827999
.endm

.macro	round2		a, b, c, d, e, index
	shuffle		\index
	func2		\a, \b, \c, \d, \e
	mix		\a, \b, \c, \d, \e, 0x6ed9eba1
.endm

.macro	round3		a, b, c, d, e, index
	shuffle		\index
	func3		\a, \b, \c, \d, \e
	mix		\a, \b, \c, \d, \e, 0x8f1bbcdc
.endm

.macro	round4		a, b, c, d, e, index
	shuffle		\index
	func4		\a, \b, \c, \d, \e
	mix		\a, \b, \c, \d, \e, 0xca62c1d6
.endm

	// sha1block(SHA1_CTX, buf, len)
ENTRY(_libmd_sha1block_scalar)
	push		%rbp
	push		%rbx
	push		%r12
	push		%r13
	push		%r14
	push		%r15
	push		%rdi			// rdi: SHA1_CTX
	sub		$64+8, %rsp		// 64 bytes for round keys
						// plus alignment

	mov		%rdi, %rbp
						// rsi: buf
	and		$~63, %rdx		// rdx: length in blocks
	lea		(%rsi, %rdx, 1), %rdi	// rdi: end pointer
	mov		(%rbp),	%eax		// c->h0
	mov		4(%rbp), %ebx		// c->h1
	mov		8(%rbp), %ecx		// c->h2
	mov		12(%rbp), %edx		// c->h3
	mov		16(%rbp), %ebp		// c->h4

	cmp		%rsi, %rdi		// any data to process?
	je		.Lend

.Lloop:	mov		%eax, %r11d
	mov		%ebx, %r12d
	mov		%ecx, %r13d
	mov		%edx, %r14d
	mov		%ebp, %r15d

	round1		%eax, %ebx, %ecx, %edx, %ebp,  0
	round1		%ebp, %eax, %ebx, %ecx, %edx,  1
	round1		%edx, %ebp, %eax, %ebx, %ecx,  2
	round1		%ecx, %edx, %ebp, %eax, %ebx,  3
	round1		%ebx, %ecx, %edx, %ebp, %eax,  4

	round1		%eax, %ebx, %ecx, %edx, %ebp,  5
	round1		%ebp, %eax, %ebx, %ecx, %edx,  6
	round1		%edx, %ebp, %eax, %ebx, %ecx,  7
	round1		%ecx, %edx, %ebp, %eax, %ebx,  8
	round1		%ebx, %ecx, %edx, %ebp, %eax,  9

	round1		%eax, %ebx, %ecx, %edx, %ebp, 10
	round1		%ebp, %eax, %ebx, %ecx, %edx, 11
	round1		%edx, %ebp, %eax, %ebx, %ecx, 12
	round1		%ecx, %edx, %ebp, %eax, %ebx, 13
	round1		%ebx, %ecx, %edx, %ebp, %eax, 14

	round1		%eax, %ebx, %ecx, %edx, %ebp, 15
	round1x		%ebp, %eax, %ebx, %ecx, %edx, 16
	round1x		%edx, %ebp, %eax, %ebx, %ecx, 17
	round1x		%ecx, %edx, %ebp, %eax, %ebx, 18
	round1x		%ebx, %ecx, %edx, %ebp, %eax, 19

	round2		%eax, %ebx, %ecx, %edx, %ebp, 20
	round2		%ebp, %eax, %ebx, %ecx, %edx, 21
	round2		%edx, %ebp, %eax, %ebx, %ecx, 22
	round2		%ecx, %edx, %ebp, %eax, %ebx, 23
	round2		%ebx, %ecx, %edx, %ebp, %eax, 24

	round2		%eax, %ebx, %ecx, %edx, %ebp, 25
	round2		%ebp, %eax, %ebx, %ecx, %edx, 26
	round2		%edx, %ebp, %eax, %ebx, %ecx, 27
	round2		%ecx, %edx, %ebp, %eax, %ebx, 28
	round2		%ebx, %ecx, %edx, %ebp, %eax, 29

	round2		%eax, %ebx, %ecx, %edx, %ebp, 30
	round2		%ebp, %eax, %ebx, %ecx, %edx, 31
	round2		%edx, %ebp, %eax, %ebx, %ecx, 32
	round2		%ecx, %edx, %ebp, %eax, %ebx, 33
	round2		%ebx, %ecx, %edx, %ebp, %eax, 34

	round2		%eax, %ebx, %ecx, %edx, %ebp, 35
	round2		%ebp, %eax, %ebx, %ecx, %edx, 36
	round2		%edx, %ebp, %eax, %ebx, %ecx, 37
	round2		%ecx, %edx, %ebp, %eax, %ebx, 38
	round2		%ebx, %ecx, %edx, %ebp, %eax, 39

	round3		%eax, %ebx, %ecx, %edx, %ebp, 40
	round3		%ebp, %eax, %ebx, %ecx, %edx, 41
	round3		%edx, %ebp, %eax, %ebx, %ecx, 42
	round3		%ecx, %edx, %ebp, %eax, %ebx, 43
	round3		%ebx, %ecx, %edx, %ebp, %eax, 44

	round3		%eax, %ebx, %ecx, %edx, %ebp, 45
	round3		%ebp, %eax, %ebx, %ecx, %edx, 46
	round3		%edx, %ebp, %eax, %ebx, %ecx, 47
	round3		%ecx, %edx, %ebp, %eax, %ebx, 48
	round3		%ebx, %ecx, %edx, %ebp, %eax, 49

	round3		%eax, %ebx, %ecx, %edx, %ebp, 50
	round3		%ebp, %eax, %ebx, %ecx, %edx, 51
	round3		%edx, %ebp, %eax, %ebx, %ecx, 52
	round3		%ecx, %edx, %ebp, %eax, %ebx, 53
	round3		%ebx, %ecx, %edx, %ebp, %eax, 54

	round3		%eax, %ebx, %ecx, %edx, %ebp, 55
	round3		%ebp, %eax, %ebx, %ecx, %edx, 56
	round3		%edx, %ebp, %eax, %ebx, %ecx, 57
	round3		%ecx, %edx, %ebp, %eax, %ebx, 58
	round3		%ebx, %ecx, %edx, %ebp, %eax, 59

	round4		%eax, %ebx, %ecx, %edx, %ebp, 60
	round4		%ebp, %eax, %ebx, %ecx, %edx, 61
	round4		%edx, %ebp, %eax, %ebx, %ecx, 62
	round4		%ecx, %edx, %ebp, %eax, %ebx, 63
	round4		%ebx, %ecx, %edx, %ebp, %eax, 64

	round4		%eax, %ebx, %ecx, %edx, %ebp, 65
	round4		%ebp, %eax, %ebx, %ecx, %edx, 66
	round4		%edx, %ebp, %eax, %ebx, %ecx, 67
	round4		%ecx, %edx, %ebp, %eax, %ebx, 68
	round4		%ebx, %ecx, %edx, %ebp, %eax, 69

	round4		%eax, %ebx, %ecx, %edx, %ebp, 70
	round4		%ebp, %eax, %ebx, %ecx, %edx, 71
	round4		%edx, %ebp, %eax, %ebx, %ecx, 72
	round4		%ecx, %edx, %ebp, %eax, %ebx, 73
	round4		%ebx, %ecx, %edx, %ebp, %eax, 74

	round4		%eax, %ebx, %ecx, %edx, %ebp, 75
	round4		%ebp, %eax, %ebx, %ecx, %edx, 76
	round4		%edx, %ebp, %eax, %ebx, %ecx, 77
	round4		%ecx, %edx, %ebp, %eax, %ebx, 78
	round4		%ebx, %ecx, %edx, %ebp, %eax, 79

	add		%r11d, %eax
	add		%r12d, %ebx
	add		%r13d, %ecx
	add		%r14d, %edx
	add		%r15d, %ebp

	add		$64, %rsi
	cmp		%rdi, %rsi
	jb		.Lloop

.Lend:	add		$64+8, %rsp
	pop		%rdi			// SHA1_CTX
	mov		%eax, (%rdi)
	mov		%ebx, 4(%rdi)
	mov		%ecx, 8(%rdi)
	mov		%edx, 12(%rdi)
	mov		%ebp, 16(%rdi)

	pop		%r15
	pop		%r14
	pop		%r13
	pop		%r12
	pop		%rbx
	pop		%rbp
	ret
END(_libmd_sha1block_scalar)

/*
 * This is the implementation using AVX2, BMI1 and BMI2. It is based on:
 * "SHA-1 implementation with Intel(R) AVX2 instruction set extensions"
 * From http://software.intel.com/en-us/articles
 * (look for improving-the-performance-of-the-secure-hash-algorithm-1)
 * This implementation is 2x unrolled, and interleaves vector instructions,
 * used to precompute W, with scalar computation of current round
 * for optimal scheduling.
 */

	/* trivial helper macros */
.macro	update_hash	a, tb, c, d, e
	add		(%r9), \a
	mov		\a, (%r9)
	add		4(%r9), \tb
	mov		\tb, 4(%r9)
	add		8(%r9), \c
	mov		\c, 8(%r9)
	add		12(%r9), \d
	mov		\d, 12(%r9)
	add		16(%r9), \e
	mov		\e, 16(%r9)
.endm

	/* help macros for recalc, which does precomputations */
.macro	precalc0	offset
	vmovdqu		\offset(%r10), %xmm0
.endm

.macro	precalc1	offset
	vinserti128	$1, \offset(%r13), %ymm0, %ymm0
.endm

.macro	precalc2	yreg
	vpshufb		%ymm10, %ymm0, \yreg
.endm

.macro	precalc4	yreg, k_offset
	vpaddd		\k_offset(%r8), \yreg, %ymm0
.endm

.macro	precalc7	offset
	vmovdqu		%ymm0, (\offset)*2(%r14)
.endm

/*
 * Message scheduling pre-compute for rounds 0-15
 * r13      is a pointer to the even 64-byte block
 * r10      is a pointer to the odd 64-byte block
 * r14      is a pointer to the temp buffer
 * xmm0     is used as a temp register
 * yreg     is clobbered as part of the computation
 * offset   chooses a 16 byte chunk within a block
 * r8       is a pointer to the constants block
 * k_offset chooses K constants relevant to this round
 * xmm10    holds the swap mask
 */
.macro	precalc00_15	offset, yreg
	precalc0	\offset
	precalc1	\offset
	precalc2	\yreg
	precalc4	\yreg, 0
	precalc7	\offset
.endm

	/* helper macros for precalc16_31 */
.macro	precalc16	reg_sub16, reg_sub12, reg_sub4, reg
	vpalignr	$8, \reg_sub16, \reg_sub12, \reg	// w[i - 14]
	vpsrldq		$4, \reg_sub4, %ymm0			// w[i -  3]
.endm

.macro	precalc17	reg_sub16, reg_sub8, reg
	vpxor		\reg_sub8, \reg, \reg
	vpxor		\reg_sub16, %ymm0, %ymm0
.endm

.macro	precalc18	reg
	vpxor		%ymm0, \reg, \reg
	vpslldq		$12, \reg, %ymm9
.endm

.macro	precalc19	reg
	vpslld		$1, \reg, %ymm0
	vpsrld		$31, \reg, \reg
	.endm

.macro	precalc20	reg
	vpor		\reg, %ymm0, %ymm0
	vpslld		$2, %ymm9, \reg
.endm

.macro	precalc21	reg
	vpsrld		$30, %ymm9, %ymm9
	vpxor		\reg, %ymm0, %ymm0
.endm

.macro	precalc23	reg, k_offset, offset
	vpxor		%ymm9, %ymm0, \reg
	vpaddd		\k_offset(%r8), \reg, %ymm0
	vmovdqu		%ymm0, (\offset)(%r14)
.endm

/*
 * Message scheduling pre-compute for rounds 16-31
 * calculating last 32 w[i] values in 8 XMM registers
 * pre-calculate K+w[i] values and store to mem
 * for later load by ALU add instruction.
 * "brute force" vectorization for rounds 16-31 only
 * due to w[i]->w[i-3] dependency.
 + clobbers 5 input ymm registers REG_SUB*
 * uses xmm0 and xmm9 as temp registers
 * As always, r8 is a pointer to constants block
 * and r14 is a pointer to temp buffer
 */
.macro	precalc16_31	reg, reg_sub4, reg_sub8, reg_sub12, reg_sub16, k_offset, offset
	precalc16	\reg_sub16, \reg_sub12, \reg_sub4, \reg
	precalc17	\reg_sub16, \reg_sub8, \reg
	precalc18	\reg
	precalc19	\reg
	precalc20	\reg
	precalc21	\reg
	precalc23	\reg, \k_offset, \offset
.endm

	/* helper macros for precalc_32_79 */
.macro	precalc32	reg_sub8, reg_sub4
	vpalignr	$8, \reg_sub8, \reg_sub4, %ymm0
.endm

.macro	precalc33	reg_sub28, reg
	vpxor		\reg_sub28, \reg, \reg
.endm

.macro	precalc34	reg_sub16
	vpxor		\reg_sub16, %ymm0, %ymm0
.endm

.macro	precalc35	reg
	vpxor		%ymm0, \reg, \reg
.endm

.macro	precalc36	reg
	vpslld		$2, \reg, %ymm0
.endm

.macro	precalc37	reg
	vpsrld		$30, \reg, \reg
	vpor		\reg, %ymm0, \reg
.endm

.macro	precalc39	reg, k_offset, offset
	vpaddd		\k_offset(%r8), \reg, %ymm0
	vmovdqu		%ymm0, \offset(%r14)
.endm

.macro	precalc32_79	reg, reg_sub4, reg_sub8, reg_sub16, reg_sub28, k_offset, offset
	precalc32	\reg_sub8, \reg_sub4
	precalc33	\reg_sub28, \reg
	precalc34	\reg_sub16
	precalc35	\reg
	precalc36	\reg
	precalc37	\reg
	precalc39	\reg, \k_offset, \offset
.endm

.macro	precalc
	precalc00_15	0x00, %ymm15
	precalc00_15	0x10, %ymm14
	precalc00_15	0x20, %ymm13
	precalc00_15	0x30, %ymm12
	precalc16_31	%ymm8,  %ymm12, %ymm13, %ymm14, %ymm15, 0x00, 0x080
	precalc16_31	%ymm7,  %ymm8,  %ymm12, %ymm13, %ymm14, 0x20, 0x0a0
	precalc16_31	%ymm5,  %ymm7,  %ymm8,  %ymm12, %ymm13, 0x20, 0x0c0
	precalc16_31	%ymm3,  %ymm5,  %ymm7,  %ymm8,  %ymm12, 0x20, 0x0e0
	precalc32_79	%ymm15, %ymm3,  %ymm5,  %ymm8,  %ymm14, 0x20, 0x100
	precalc32_79	%ymm14, %ymm15, %ymm3,  %ymm7,  %ymm13, 0x20, 0x120
	precalc32_79	%ymm13, %ymm14, %ymm15, %ymm5,  %ymm12, 0x40, 0x140
	precalc32_79	%ymm12, %ymm13, %ymm14, %ymm3,  %ymm8,  0x40, 0x160
	precalc32_79	%ymm8,  %ymm12, %ymm13, %ymm15, %ymm7,  0x40, 0x180
	precalc32_79	%ymm7,  %ymm8,  %ymm12, %ymm14, %ymm5,  0x40, 0x1a0
	precalc32_79	%ymm5,  %ymm7,  %ymm8,  %ymm13, %ymm3,  0x40, 0x1c0
	precalc32_79	%ymm3,  %ymm5,  %ymm7,  %ymm12, %ymm15, 0x60, 0x1e0
	precalc32_79	%ymm15, %ymm3,  %ymm5,  %ymm8,  %ymm14, 0x60, 0x200
	precalc32_79	%ymm14, %ymm15, %ymm3,  %ymm7,  %ymm13, 0x60, 0x220
	precalc32_79	%ymm13, %ymm14, %ymm15, %ymm5,  %ymm12, 0x60, 0x240
	precalc32_79	%ymm12, %ymm13, %ymm14, %ymm3,  %ymm8,  0x60, 0x260
.endm

/*
 * Macros calculating individual rounds have general form
 * calc_round_pre + precalc_round + calc_round_post
 * calc_round_{pre,post} macros follow
 */
.macro	calc_f1_pre	offset, reg_a, reg_b, reg_c, reg_e
	add		\offset(%r15), \reg_e
	andn		\reg_c, \reg_a, %ebp
	add		\reg_b, \reg_e			// add F from the previous round
	rorx		$0x1b, \reg_a, %r12d
	rorx		$2, \reg_a, \reg_b		// for the next round
.endm

/*
 * Calculate F for the next round
 */
.macro	calc_f1_post	reg_a, reg_b, reg_e
	and		\reg_b, \reg_a			// b & c
	xor		%ebp, \reg_a			// F1 = (b&c) ^ (~b&d)
	add		%r12d, \reg_e
.endm

/*
 * Registers are cyclically rotated:
 * edx -> eax -> edi -> esi -> ebx -> ecx
 */
.macro	calc0
	mov		%esi, %ebx			// precalculate first round
	rorx		$2, %esi, %esi
	andn		%eax, %ebx, %ebp
	and		%edi, %ebx
	xor		%ebp, %ebx
	calc_f1_pre	0x0, %ecx, %ebx, %edi, %edx
	precalc0	0x80
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc1
	calc_f1_pre	0x4, %edx, %ecx, %esi, %eax
	precalc1	0x80
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc2
	calc_f1_pre	0x8, %eax, %edx, %ebx, %edi
	precalc2	%ymm15
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc3
	calc_f1_pre	0xc, %edi, %eax, %ecx, %esi
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc4
	calc_f1_pre	0x20, %esi, %edi, %edx, %ebx
	precalc4	%ymm15, 0x0
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc5
	calc_f1_pre	0x24, %ebx, %esi, %eax, %ecx
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc6
	calc_f1_pre	0x28, %ecx, %ebx, %edi, %edx
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc7
	calc_f1_pre	0x2c, %edx, %ecx, %esi, %eax
	precalc7	0x0
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc8
	calc_f1_pre	0x40, %eax, %edx, %ebx, %edi
	precalc0	0x90
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc9
	calc_f1_pre	0x44, %edi, %eax, %ecx, %esi
	precalc1	0x90
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc10
	calc_f1_pre	0x48, %esi, %edi, %edx, %ebx
	precalc2	%ymm14
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc11
	calc_f1_pre	0x4c, %ebx, %esi, %eax, %ecx
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc12
	calc_f1_pre	0x60, %ecx, %ebx, %edi, %edx
	precalc4	%ymm14, 0
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc13
	calc_f1_pre	0x64, %edx, %ecx, %esi, %eax
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc14
	calc_f1_pre	0x68, %eax, %edx, %ebx, %edi
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc15
	calc_f1_pre	0x6c, %edi, %eax, %ecx, %esi
	precalc7	0x10
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc16
	calc_f1_pre	0x80, %esi, %edi, %edx, %ebx
	precalc0	0xa0
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc17
	calc_f1_pre	0x84, %ebx, %esi, %eax, %ecx
	precalc1	0xa0
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc18
	calc_f1_pre	0x88, %ecx, %ebx, %edi, %edx
	precalc2	%ymm13
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc_f2_pre	offset, reg_a, reg_b, reg_e
	add		\offset(%r15), \reg_e
	add		\reg_b, \reg_e			// add F from the previous round
	rorx		$0x1b, \reg_a, %r12d
	rorx		$2, \reg_a, \reg_b		// for next round
.endm

.macro	calc_f2_post	reg_a, reg_b, reg_c, reg_e
	xor		\reg_b, \reg_a
	add		%r12d, \reg_e
	xor		\reg_c, \reg_a
.endm

.macro	calc19
	calc_f2_pre	0x8c, %edx, %ecx, %eax
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc20
	calc_f2_pre	0xa0, %eax, %edx, %edi
	precalc4	%ymm13, 0x0
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc21
	calc_f2_pre	0xa4, %edi, %eax, %esi
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc22
	calc_f2_pre	0xa8, %esi, %edi, %ebx
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc23
	calc_f2_pre	0xac, %ebx, %esi, %ecx
	precalc7	0x20
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc24
	calc_f2_pre	0xc0, %ecx, %ebx, %edx
	precalc0	0xb0
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc25
	calc_f2_pre	0xc4, %edx, %ecx, %eax
	precalc1	0xb0
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc26
	calc_f2_pre	0xc8, %eax, %edx, %edi
	precalc2	%ymm12
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc27
	calc_f2_pre	0xcc, %edi, %eax, %esi
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc28
	calc_f2_pre	0xe0, %esi, %edi, %ebx
	precalc4	%ymm12, 0x0
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc29
	calc_f2_pre	0xe4, %ebx, %esi, %ecx
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc30
	calc_f2_pre	0xe8, %ecx, %ebx, %edx
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc31
	calc_f2_pre	0xec, %edx, %ecx, %eax
	precalc7	0x30
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc32
	calc_f2_pre	0x100, %eax, %edx, %edi
	precalc16	%ymm15, %ymm14, %ymm12, %ymm8
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc33
	calc_f2_pre	0x104, %edi, %eax, %esi
	precalc17	%ymm15, %ymm13, %ymm8
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc34
	calc_f2_pre	0x108, %esi, %edi, %ebx
	precalc18	%ymm8
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc35
	calc_f2_pre	0x10c, %ebx, %esi, %ecx
	precalc19	%ymm8
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc36
	calc_f2_pre	0x120, %ecx, %ebx, %edx
	precalc20	%ymm8
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc37
	calc_f2_pre	0x124, %edx, %ecx, %eax
	precalc21	%ymm8
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc38
	calc_f2_pre	0x128, %eax, %edx, %edi
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc_f3_pre	offset, reg_e
	add		\offset(%r15), \reg_e
.endm

.macro	calc_f3_post	reg_a, reg_b, reg_c, reg_e, reg_tb
	add		\reg_tb, \reg_e		// add F from the previous round
	mov		\reg_b, %ebp
	or		\reg_a, %ebp
	rorx		$0x1b, \reg_a, %r12d
	rorx		$2, \reg_a, \reg_tb
	and		\reg_c, %ebp		// calculate F for the next round
	and		\reg_b, \reg_a
	or		%ebp, \reg_a
	add		%r12d, \reg_e
.endm

.macro	calc39
	calc_f3_pre	0x12c, %esi
	precalc23	%ymm8, 0x0, 0x80
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc40
	calc_f3_pre	0x140, %ebx
	precalc16	%ymm14, %ymm13, %ymm8, %ymm7
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc41
	calc_f3_pre	0x144, %ecx
	precalc17	%ymm14, %ymm12, %ymm7
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc42
	calc_f3_pre	0x148, %edx
	precalc18	%ymm7
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc43
	calc_f3_pre	0x14c, %eax
	precalc19	%ymm7
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc44
	calc_f3_pre	0x160, %edi
	precalc20	%ymm7
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc45
	calc_f3_pre	0x164, %esi
	precalc21	%ymm7
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc46
	calc_f3_pre	0x168, %ebx
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc47
	calc_f3_pre	0x16c, %ecx
	vpxor		%ymm9, %ymm0, %ymm7
	vpaddd		0x20(%r8), %ymm7, %ymm0
	vmovdqu		%ymm0, 0xa0(%r14)
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc48
	calc_f3_pre	0x180, %edx
	precalc16	%ymm13, %ymm12, %ymm7, %ymm5
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc49
	calc_f3_pre	0x184, %eax
	precalc17	%ymm13, %ymm8, %ymm5
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc50
	calc_f3_pre	0x188, %edi
	precalc18	%ymm5
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc51
	calc_f3_pre	0x18c, %esi
	precalc19	%ymm5
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc52
	calc_f3_pre	0x1a0, %ebx
	precalc20	%ymm5
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc53
	calc_f3_pre	0x1a4, %ecx
	precalc21	%ymm5
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc54
	calc_f3_pre	0x1a8, %edx
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc55
	calc_f3_pre	0x1ac, %eax
	precalc23	%ymm5, 0x20, 0xc0
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc56
	calc_f3_pre	0x1c0, %edi
	precalc16	%ymm12, %ymm8, %ymm5, %ymm3
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc57
	calc_f3_pre	0x1c4, %esi
	precalc17	%ymm12, %ymm7, %ymm3
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc58
	calc_f3_pre	0x1c8, %ebx
	precalc18	%ymm3
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc59
	calc_f2_pre	0x1cc, %ebx, %esi, %ecx
	precalc19	%ymm3
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc60
	calc_f2_pre	0x1e0, %ecx, %ebx, %edx
	precalc20	%ymm3
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc61
	calc_f2_pre	0x1e4, %edx, %ecx, %eax
	precalc21	%ymm3
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc62
	calc_f2_pre	0x1e8, %eax, %edx, %edi
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc63
	calc_f2_pre	0x1ec, %edi, %eax, %esi
	precalc23	%ymm3, 0x20, 0xe0
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc64
	calc_f2_pre	0x200, %esi, %edi, %ebx
	precalc32	%ymm5, %ymm3
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc65
	calc_f2_pre	0x204, %ebx, %esi, %ecx
	precalc33	%ymm14, %ymm15
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc66
	calc_f2_pre	0x208, %ecx, %ebx, %edx
	precalc34	%ymm8
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc67
	calc_f2_pre	0x20c, %edx, %ecx, %eax
	precalc35	%ymm15
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc68
	calc_f2_pre	0x220, %eax, %edx, %edi
	precalc36	%ymm15
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc69
	calc_f2_pre	0x224, %edi, %eax, %esi
	precalc37	%ymm15
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc70
	calc_f2_pre	0x228, %esi, %edi, %ebx
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc71
	calc_f2_pre	0x22c, %ebx, %esi, %ecx
	precalc39	%ymm15, 0x20, 0x100
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc72
	calc_f2_pre	0x240, %ecx, %ebx, %edx
	precalc32	%ymm3, %ymm15
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc73
	calc_f2_pre	0x244, %edx, %ecx, %eax
	precalc33	%ymm13, %ymm14
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc74
	calc_f2_pre	0x248, %eax, %edx, %edi
	precalc34	%ymm7
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc75
	calc_f2_pre	0x24c, %edi, %eax, %esi
	precalc35	%ymm14
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc76
	calc_f2_pre	0x260, %esi, %edi, %ebx
	precalc36	%ymm14
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc77
	calc_f2_pre	0x264, %ebx, %esi, %ecx
	precalc37	%ymm14
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc78
	calc_f2_pre	0x268, %ecx, %ebx, %edx
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc79
	add		0x26c(%r15), %eax
	add		%ecx, %eax
	rorx		$0x1b, %edx, %r12d
	precalc39	%ymm14, 0x20, 0x120
	add		%r12d, %eax
.endm

/*
 * Similar to calc0
 */
.macro	calc80
	mov		%ecx, %edx			// precalculate first round
	rorx		$2, %ecx, %ecx
	andn		%esi, %edx, %ebp
	and		%ebx, %edx
	xor		%ebp, %edx
	calc_f1_pre	0x10, %eax, %edx, %ebx, %edi
	precalc32	%ymm15, %ymm14
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc81
	calc_f1_pre	0x14, %edi, %eax, %ecx, %esi
	precalc33	%ymm12, %ymm13
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc82
	calc_f1_pre	0x18, %esi, %edi, %edx, %ebx
	precalc34	%ymm5
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc83
	calc_f1_pre	0x1c, %ebx, %esi, %eax, %ecx
	precalc35	%ymm13
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc84
	calc_f1_pre	0x30, %ecx, %ebx, %edi, %edx
	precalc36	%ymm13
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc85
	calc_f1_pre	0x34, %edx, %ecx, %esi, %eax
	precalc37	%ymm13
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc86
	calc_f1_pre	0x38, %eax, %edx, %ebx, %edi
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc87
	calc_f1_pre	0x3c, %edi, %eax, %ecx, %esi
	precalc39	%ymm13, 0x40, 0x140
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc88
	calc_f1_pre	0x50, %esi, %edi, %edx, %ebx
	precalc32	%ymm14, %ymm13
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc89
	calc_f1_pre	0x54, %ebx, %esi, %eax, %ecx
	precalc33	%ymm8, %ymm12
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc90
	calc_f1_pre	0x58, %ecx, %ebx, %edi, %edx
	precalc34	%ymm3
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc91
	calc_f1_pre	0x5c, %edx, %ecx, %esi, %eax
	precalc35	%ymm12
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc92
	calc_f1_pre	0x70, %eax, %edx, %ebx, %edi
	precalc36	%ymm12
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc93
	calc_f1_pre	0x74, %edi, %eax, %ecx, %esi
	precalc37	%ymm12
	calc_f1_post	%edi, %edx, %esi
.endm

.macro	calc94
	calc_f1_pre	0x78, %esi, %edi, %edx, %ebx
	calc_f1_post	%esi, %eax, %ebx
.endm

.macro	calc95
	calc_f1_pre	0x7c, %ebx, %esi, %eax, %ecx
	precalc39	%ymm12, 0x40, 0x160
	calc_f1_post	%ebx, %edi, %ecx
.endm

.macro	calc96
	calc_f1_pre	0x90, %ecx, %ebx, %edi, %edx
	precalc32	%ymm13, %ymm12
	calc_f1_post	%ecx, %esi, %edx
.endm

.macro	calc97
	calc_f1_pre	0x94, %edx, %ecx, %esi, %eax
	precalc33	%ymm7, %ymm8
	calc_f1_post	%edx, %ebx, %eax
.endm

.macro	calc98
	calc_f1_pre	0x98, %eax, %edx, %ebx, %edi
	precalc34	%ymm15
	calc_f1_post	%eax, %ecx, %edi
.endm

.macro	calc99
	calc_f2_pre	0x9c, %edi, %eax, %esi
	precalc35	%ymm8
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc100
	calc_f2_pre	0xb0, %esi, %edi, %ebx
	precalc36	%ymm8
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc101
	calc_f2_pre	0xb4, %ebx, %esi, %ecx
	precalc37	%ymm8
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc102
	calc_f2_pre	0xb8, %ecx, %ebx, %edx
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc103
	calc_f2_pre	0xbc, %edx, %ecx, %eax
	precalc39	%ymm8, 0x40, 0x180
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc104
	calc_f2_pre	0xd0, %eax, %edx, %edi
	precalc32	%ymm12, %ymm8
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc105
	calc_f2_pre	0xd4, %edi, %eax, %esi
	precalc33	%ymm5, %ymm7
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc106
	calc_f2_pre	0xd8, %esi, %edi, %ebx
	precalc34	%ymm14
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc107
	calc_f2_pre	0xdc, %ebx, %esi, %ecx
	precalc35	%ymm7
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc108
	calc_f2_pre	0xf0, %ecx, %ebx, %edx
	precalc36	%ymm7
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc109
	calc_f2_pre	0xf4, %edx, %ecx, %eax
	precalc37	%ymm7
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc110
	calc_f2_pre	0xf8, %eax, %edx, %edi
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc111
	calc_f2_pre	0xfc, %edi, %eax, %esi
	precalc39	%ymm7, 0x40, 0x1a0
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc112
	calc_f2_pre	0x110, %esi, %edi, %ebx
	precalc32	%ymm8, %ymm7
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc113
	calc_f2_pre	0x114, %ebx, %esi, %ecx
	precalc33	%ymm3, %ymm5
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc114
	calc_f2_pre	0x118, %ecx, %ebx, %edx
	precalc34	%ymm13
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc115
	calc_f2_pre	0x11c, %edx, %ecx, %eax
	precalc35	%ymm5
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc116
	calc_f2_pre	0x130, %eax, %edx, %edi
	precalc36	%ymm5
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc117
	calc_f2_pre	0x134, %edi, %eax, %esi
	precalc37	%ymm5
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc118
	calc_f2_pre	0x138, %esi, %edi, %ebx
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc119
	calc_f3_pre	0x13c, %ecx
	precalc39	%ymm5, 0x40, 0x1c0
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc120
	calc_f3_pre	0x150, %edx
	precalc32	%ymm7, %ymm5
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc121
	calc_f3_pre	0x154, %eax
	precalc33	%ymm15, %ymm3
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc122
	calc_f3_pre	0x158, %edi
	precalc34	%ymm12
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc123
	calc_f3_pre	0x15c, %esi
	precalc35	%ymm3
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc124
	calc_f3_pre	0x170, %ebx
	precalc36	%ymm3
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc125
	calc_f3_pre	0x174, %ecx
	precalc37	%ymm3
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc126
	calc_f3_pre	0x178, %edx
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc127
	calc_f3_pre	0x17c, %eax
	precalc39	%ymm3, 0x60, 0x1e0
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc128
	calc_f3_pre	0x190, %edi
	precalc32	%ymm5, %ymm3
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc129
	calc_f3_pre	0x194, %esi
	precalc33	%ymm14, %ymm15
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc130
	calc_f3_pre	0x198, %ebx
	precalc34	%ymm8
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc131
	calc_f3_pre	0x19c, %ecx
	precalc35	%ymm15
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc132
	calc_f3_pre	0x1b0, %edx
	precalc36	%ymm15
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc133
	calc_f3_pre	0x1b4, %eax
	precalc37	%ymm15
	calc_f3_post	%edx, %ebx, %esi, %eax, %ecx
.endm

.macro	calc134
	calc_f3_pre	0x1b8, %edi
	calc_f3_post	%eax, %ecx, %ebx, %edi, %edx
.endm

.macro	calc135
	calc_f3_pre	0x1bc, %esi
	precalc39	%ymm15, 0x60, 0x200
	calc_f3_post	%edi, %edx, %ecx, %esi, %eax
.endm

.macro	calc136
	calc_f3_pre	0x1d0, %ebx
	precalc32	%ymm3, %ymm15
	calc_f3_post	%esi, %eax, %edx, %ebx, %edi
.endm

.macro	calc137
	calc_f3_pre	0x1d4, %ecx
	precalc33	%ymm13, %ymm14
	calc_f3_post	%ebx, %edi, %eax, %ecx, %esi
.endm

.macro	calc138
	calc_f3_pre	0x1d8, %edx
	precalc34	%ymm7
	calc_f3_post	%ecx, %esi, %edi, %edx, %ebx
.endm

.macro	calc139
	calc_f2_pre	0x1dc, %edx, %ecx, %eax
	precalc35	%ymm14
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc140
	calc_f2_pre	0x1f0, %eax, %edx, %edi
	precalc36	%ymm14
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc141
	calc_f2_pre	0x1f4, %edi, %eax, %esi
	precalc37	%ymm14
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc142
	calc_f2_pre	0x1f8, %esi, %edi, %ebx
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc143
	calc_f2_pre	0x1fc, %ebx, %esi, %ecx
	precalc39	%ymm14, 0x60, 0x220
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc144
	calc_f2_pre	0x210, %ecx, %ebx, %edx
	precalc32	%ymm15, %ymm14
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc145
	calc_f2_pre	0x214, %edx, %ecx, %eax
	precalc33	%ymm12, %ymm13
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc146
	calc_f2_pre	0x218, %eax, %edx, %edi
	precalc34	%ymm5
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc147
	calc_f2_pre	0x21c, %edi, %eax, %esi
	precalc35	%ymm13
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc148
	calc_f2_pre	0x230, %esi, %edi, %ebx
	precalc36	%ymm13
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc149
	calc_f2_pre	0x234, %ebx, %esi, %ecx
	precalc37	%ymm13
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc150
	calc_f2_pre	0x238, %ecx, %ebx, %edx
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc151
	calc_f2_pre	0x23c, %edx, %ecx, %eax
	precalc39	%ymm13, 0x60, 0x240
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc152
	calc_f2_pre	0x250, %eax, %edx, %edi
	precalc32	%ymm14, %ymm13
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc153
	calc_f2_pre	0x254, %edi, %eax, %esi
	precalc33	%ymm8, %ymm12
	calc_f2_post	%edi, %edx, %ecx, %esi
.endm

.macro	calc154
	calc_f2_pre	0x258, %esi, %edi, %ebx
	precalc34	%ymm3
	calc_f2_post	%esi, %eax, %edx, %ebx
.endm

.macro	calc155
	calc_f2_pre	0x25c, %ebx, %esi, %ecx
	precalc35	%ymm12
	calc_f2_post	%ebx, %edi, %eax, %ecx
.endm

.macro	calc156
	calc_f2_pre	0x270, %ecx, %ebx, %edx
	precalc36	%ymm12
	calc_f2_post	%ecx, %esi, %edi, %edx
.endm

.macro	calc157
	calc_f2_pre	0x274, %edx, %ecx, %eax
	precalc37	%ymm12
	calc_f2_post	%edx, %ebx, %esi, %eax
.endm

.macro	calc158
	calc_f2_pre	0x278, %eax, %edx, %edi
	calc_f2_post	%eax, %ecx, %ebx, %edi
.endm

.macro	calc159
	add		0x27c(%r15), %esi
	add		%eax, %esi
	rorx		$0x1b, %edi, %r12d
	precalc39	%ymm12, 0x60, 0x260
	add		%r12d, %esi
.endm

	// sha1block(SHA1_CTX, buf, len)
ENTRY(_libmd_sha1block_avx2)
	push		%rbx
	push		%rbp
	push		%r12
	push		%r13
	push		%r14
	push		%r15
	sub		$1408+8, %rsp

	and		$~63, %rdx
	lea		k_xmm_ar(%rip), %r8
	mov		%rdi, %r9
	mov		%rsi, %r10
	lea		64(%rsi), %r13
	lea		64(%rsi, %rdx), %r11
	cmp		%r11, %r13
	cmovae		%r8, %r13
	vmovdqu		bswap_shufb_ctl(%rip), %ymm10

	mov		(%r9), %ecx
	mov		4(%r9), %esi
	mov		8(%r9), %edi
	mov		12(%r9), %eax
	mov		16(%r9), %edx
	mov		%rsp, %r14
	lea		2*4*80+32(%rsp), %r15
	precalc						// precalc WK for first 2 blocks
	xchg		%r14, %r15

	// this is unrolled
.Loop:	cmp		%r8, %r10			// we use the value of R8 (set below)
							// as a signal of the last block
	jne		.Lbegin
	add		$1408+8, %rsp
	pop		%r15
	pop		%r14
	pop		%r13
	pop		%r12
	pop		%rbp
	pop		%rbx
	vzeroupper
	ret

.Lbegin:
	calc0
	calc1
	calc2
	calc3
	calc4
	calc5
	calc6
	calc7
	calc8
	calc9
	calc10
	calc11
	calc12
	calc13
	calc14
	calc15
	calc16
	calc17
	calc18
	calc19
	calc20
	calc21
	calc22
	calc23
	calc24
	calc25
	calc26
	calc27
	calc28
	calc29
	calc30
	calc31
	calc32
	calc33
	calc34
	calc35
	calc36
	calc37
	calc38
	calc39
	calc40
	calc41
	calc42
	calc43
	calc44
	calc45
	calc46
	calc47
	calc48
	calc49
	calc50
	calc51
	calc52
	calc53
	calc54
	calc55
	calc56
	calc57
	calc58
	calc59

	add		$128, %r10		// move to the next even-64-byte block
	cmp		%r11, %r10		// is the current block the last one?
	cmovae		%r8, %r10		// signal the last iteration smartly

	calc60
	calc61
	calc62
	calc63
	calc64
	calc65
	calc66
	calc67
	calc68
	calc69
	calc70
	calc71
	calc72
	calc73
	calc74
	calc75
	calc76
	calc77
	calc78
	calc79

	update_hash	%eax, %edx, %ebx, %esi, %edi
	cmp		%r8, %r10		// is the current block the last one?
	je		.Loop
	mov		%edx, %ecx

	calc80
	calc81
	calc82
	calc83
	calc84
	calc85
	calc86
	calc87
	calc88
	calc89
	calc90
	calc91
	calc92
	calc93
	calc94
	calc95
	calc96
	calc97
	calc98
	calc99
	calc100
	calc101
	calc102
	calc103
	calc104
	calc105
	calc106
	calc107
	calc108
	calc109
	calc110
	calc111
	calc112
	calc113
	calc114
	calc115
	calc116
	calc117
	calc118
	calc119
	calc120
	calc121
	calc122
	calc123
	calc124
	calc125
	calc126
	calc127
	calc128
	calc129
	calc130
	calc131
	calc132
	calc133
	calc134
	calc135
	calc136
	calc137
	calc138
	calc139

	add		$128, %r13		// move to the next even-64-byte block
	cmp		%r11, %r13		// is the current block the last one?
	cmovae		%r8, %r10

	calc140
	calc141
	calc142
	calc143
	calc144
	calc145
	calc146
	calc147
	calc148
	calc149
	calc150
	calc151
	calc152
	calc153
	calc154
	calc155
	calc156
	calc157
	calc158
	calc159

	update_hash	%esi, %edi, %edx, %ecx, %ebx
	mov		%esi, %r12d		// reset state for AVX2 reg permutation
	mov		%edi, %esi
	mov		%edx, %edi
	mov		%ebx, %edx
	mov		%ecx, %eax
	mov		%r12d, %ecx
	xchg		%r14, %r15
	jmp		.Loop
END(_libmd_sha1block_avx2)

	.section	.rodata
	.balign		32
k_xmm_ar:
	.fill		8, 4, 0x5a827999
	.fill		8, 4, 0x6ed9eba1
	.fill		8, 4, 0x8f1bbcdc
	.fill		8, 4, 0xca62c1d6
	.size		k_xmm_ar, .-k_xmm_ar

bswap_shufb_ctl:
	.4byte		0x00010203
	.4byte		0x04050607
	.4byte		0x08090a0b
	.4byte		0x0c0d0e0f
	.4byte		0x00010203
	.4byte		0x04050607
	.4byte		0x08090a0b
	.4byte		0x0c0d0e0f
	.size		bswap_shufb_ctl, .-bswap_shufb_ctl

	/*
	 * SHA1 implementation using the Intel SHA extensions (SHANI).
	 *
	 * Imlemented according to the Intel white paper
	 *
	 * S. Gulley, V. Gopal, K. Yap, W. Feghali, J. Guilford,
	 * G. Wolrich: "Intel SHA Extensions: new instruction supporting
	 * the Secure Hash Algorithm on Intel® architecture processors",
	 * July 2013.
	 */
	// sha1block(SHA1_CTX, buf, len)
ENTRY(_libmd_sha1block_shani)
	and		$~63, %rdx		// round length to block-size multiple
	lea		(%rsi, %rdx, 1), %rcx	// end pointer
	test		%rdx, %rdx		// nothing to do?
	je		1f			// if so, terminate immediately

	movdqu		(%rdi), %xmm6		// h0, h1, h2, h3
	pxor		%xmm7, %xmm7
	pshufd		$0x1b, %xmm6, %xmm6	// h3, h2, h1, h0
	pinsrd		$3, 16(%rdi), %xmm7	// h4 in the highest word of xmm7
	movdqu		shuf_mask(%rip), %xmm4

	// main loop
0:	movdqa		%xmm6, %xmm8		// stash ABCD
	movdqa		%xmm7, %xmm9		// stash E

	// rounds 0--3
	movdqu		0*16(%rsi), %xmm0	// load first message block
	pshufb		%xmm4, %xmm0		// and byte-swap
	paddd		%xmm0, %xmm7		// E += w[0]
	movdqa		%xmm6, %xmm5		// E' = A
	sha1rnds4	$0, %xmm7, %xmm6	// perform rounds 0--3

	// rounds 4--7
	movdqu		1*16(%rsi), %xmm1
	pshufb		%xmm4, %xmm1
	sha1nexte	%xmm1, %xmm5
	movdqa		%xmm6, %xmm7
	sha1rnds4	$0, %xmm5, %xmm6
	sha1msg1	%xmm1, %xmm0

	// rounds 8--11
	movdqu		2*16(%rsi), %xmm2
	pshufb		%xmm4, %xmm2
	sha1nexte	%xmm2, %xmm7
	movdqa		%xmm6, %xmm5
	sha1rnds4	$0, %xmm7, %xmm6
	sha1msg1	%xmm2, %xmm1
	pxor		%xmm2, %xmm0

.macro	midround	msg3, msg0, msg1, msg2, e1, e0, k
	sha1nexte	\msg3, \e1
	movdqa		%xmm6, \e0
	sha1msg2	\msg3, \msg0
	sha1rnds4	$\k, \e1, %xmm6
	sha1msg1	\msg3, \msg2
	pxor		\msg3, \msg1
.endm

	movdqu		3*16(%rsi), %xmm3	// load third message block
	pshufb		%xmm4, %xmm3

	add		$4*16, %rsi

	midround	%xmm3, %xmm0, %xmm1, %xmm2, %xmm5, %xmm7, 0	// 12--15
	midround	%xmm0, %xmm1, %xmm2, %xmm3, %xmm7, %xmm5, 0	// 16--19
	midround	%xmm1, %xmm2, %xmm3, %xmm0, %xmm5, %xmm7, 1	// 20--23
	midround	%xmm2, %xmm3, %xmm0, %xmm1, %xmm7, %xmm5, 1	// 24--27
	midround	%xmm3, %xmm0, %xmm1, %xmm2, %xmm5, %xmm7, 1	// 28--31
	midround	%xmm0, %xmm1, %xmm2, %xmm3, %xmm7, %xmm5, 1	// 32--35
	midround	%xmm1, %xmm2, %xmm3, %xmm0, %xmm5, %xmm7, 1	// 36--39
	midround	%xmm2, %xmm3, %xmm0, %xmm1, %xmm7, %xmm5, 2	// 40--43
	midround	%xmm3, %xmm0, %xmm1, %xmm2, %xmm5, %xmm7, 2	// 44--47
	midround	%xmm0, %xmm1, %xmm2, %xmm3, %xmm7, %xmm5, 2	// 48--51
	midround	%xmm1, %xmm2, %xmm3, %xmm0, %xmm5, %xmm7, 2	// 52--55
	midround	%xmm2, %xmm3, %xmm0, %xmm1, %xmm7, %xmm5, 2	// 56--59
	midround	%xmm3, %xmm0, %xmm1, %xmm2, %xmm5, %xmm7, 3	// 60--63
	midround	%xmm0, %xmm1, %xmm2, %xmm3, %xmm7, %xmm5, 3	// 64--67

	// rounds 68--71
	sha1nexte	%xmm1, %xmm5
	movdqa		%xmm6, %xmm7
	sha1msg2	%xmm1, %xmm2
	sha1rnds4	$3, %xmm5, %xmm6
	pxor		%xmm1, %xmm3

	// rounds 72--75
	sha1nexte	%xmm2, %xmm7
	movdqa		%xmm6, %xmm5
	sha1msg2	%xmm2, %xmm3
	sha1rnds4	$3, %xmm7, %xmm6

	// rounds 76--79
	sha1nexte	%xmm3, %xmm5
	movdqa		%xmm6, %xmm7
	sha1rnds4	$3, %xmm5, %xmm6

	sha1nexte	%xmm9, %xmm7		// add saved E
	paddd		%xmm8, %xmm6		// add saved ABCD

	cmp		%rsi, %rcx		// end reached?
	jne		0b

	pshufd		$0x1b, %xmm6, %xmm6	// restore order of h0--h3
	movdqu		%xmm6, (%rdi)		// write h0--h3
	pextrd		$3, %xmm7, 16(%rdi)	// write h4
1:	ret
END(_libmd_sha1block_shani)

	.section	.rodata
	.balign		16
shuf_mask:
	.8byte		0x08090a0b0c0d0e0f
	.8byte		0x0001020304050607
	.size		shuf_mask, .-shuf_mask

	.section .note.GNU-stack,"",%progbits