/* VAES/AVX2 AMD64 accelerated AES for Libgcrypt
 * Copyright (C) 2021,2023 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This file is part of Libgcrypt.
 *
 * Libgcrypt is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * Libgcrypt is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

#if defined(__x86_64__)
#include <config.h>
#if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \
    defined(ENABLE_AESNI_SUPPORT) && defined(ENABLE_AVX2_SUPPORT) && \
    defined(HAVE_GCC_INLINE_ASM_VAES_VPCLMUL)

#include "asm-common-amd64.h"

.text

/**********************************************************************
  helper macros
 **********************************************************************/
#define no(...) /*_*/
#define yes(...) __VA_ARGS__

#define AES_OP8(op, key, b0, b1, b2, b3, b4, b5, b6, b7) \
	op key, b0, b0; \
	op key, b1, b1; \
	op key, b2, b2; \
	op key, b3, b3; \
	op key, b4, b4; \
	op key, b5, b5; \
	op key, b6, b6; \
	op key, b7, b7;

#define VAESENC8(key, b0, b1, b2, b3, b4, b5, b6, b7) \
	AES_OP8(vaesenc, key, b0, b1, b2, b3, b4, b5, b6, b7)

#define VAESDEC8(key, b0, b1, b2, b3, b4, b5, b6, b7) \
	AES_OP8(vaesdec, key, b0, b1, b2, b3, b4, b5, b6, b7)

#define XOR8(key, b0, b1, b2, b3, b4, b5, b6, b7) \
	AES_OP8(vpxor, key, b0, b1, b2, b3, b4, b5, b6, b7)

#define AES_OP4(op, key, b0, b1, b2, b3) \
	op key, b0, b0; \
	op key, b1, b1; \
	op key, b2, b2; \
	op key, b3, b3;

#define VAESENC4(key, b0, b1, b2, b3) \
	AES_OP4(vaesenc, key, b0, b1, b2, b3)

#define VAESDEC4(key, b0, b1, b2, b3) \
	AES_OP4(vaesdec, key, b0, b1, b2, b3)

#define XOR4(key, b0, b1, b2, b3) \
	AES_OP4(vpxor, key, b0, b1, b2, b3)

#define AES_OP2(op, key, b0, b1) \
	op key, b0, b0; \
	op key, b1, b1;

#define VAESENC2(key, b0, b1) \
	AES_OP2(vaesenc, key, b0, b1)

#define VAESDEC2(key, b0, b1) \
	AES_OP2(vaesdec, key, b0, b1)

#define XOR2(key, b0, b1) \
	AES_OP2(vpxor, key, b0, b1)

/**********************************************************************
  CBC-mode decryption
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_cbc_dec_amd64,@function)
.globl _gcry_vaes_avx2_cbc_dec_amd64
.align 16
_gcry_vaes_avx2_cbc_dec_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%rsi: iv
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 */
	CFI_STARTPROC();

	/* Load IV. */
	vmovdqu (%rsi), %xmm15;

	/* Process 16 blocks per loop. */
.align 8
.Lcbc_dec_blk16:
	cmpq $16, %r8;
	jb .Lcbc_dec_blk8;

	leaq -16(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm8;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vmovdqu (4 * 16)(%rcx), %ymm2;
	vmovdqu (6 * 16)(%rcx), %ymm3;
	vmovdqu (8 * 16)(%rcx), %ymm4;
	vmovdqu (10 * 16)(%rcx), %ymm5;
	vmovdqu (12 * 16)(%rcx), %ymm6;
	vmovdqu (14 * 16)(%rcx), %ymm7;
	vinserti128 $1, %xmm0, %ymm15, %ymm9;
	vpxor %ymm8, %ymm0, %ymm0;
	vpxor %ymm8, %ymm1, %ymm1;
	vpxor %ymm8, %ymm2, %ymm2;
	vpxor %ymm8, %ymm3, %ymm3;
	vpxor %ymm8, %ymm4, %ymm4;
	vpxor %ymm8, %ymm5, %ymm5;
	vpxor %ymm8, %ymm6, %ymm6;
	vpxor %ymm8, %ymm7, %ymm7;
	vbroadcasti128 (1 * 16)(%rdi), %ymm8;
	vmovdqu (1 * 16)(%rcx), %ymm10;
	vmovdqu (3 * 16)(%rcx), %ymm11;
	vmovdqu (5 * 16)(%rcx), %ymm12;
	vmovdqu (7 * 16)(%rcx), %ymm13;
	vmovdqu (9 * 16)(%rcx), %ymm14;
	vmovdqu (15 * 16)(%rcx), %xmm15;
	leaq (16 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (2 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (3 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (4 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (5 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (6 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (7 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (8 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (9 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (10 * 16)(%rdi), %ymm8;
	cmpl $12, %r9d;
	jb .Lcbc_dec_blk16_last;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (11 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (12 * 16)(%rdi), %ymm8;
	jz .Lcbc_dec_blk16_last;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (13 * 16)(%rdi), %ymm8;
	VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (14 * 16)(%rdi), %ymm8;

	/* Last round and output handling. */
  .Lcbc_dec_blk16_last:
	vpxor %ymm8, %ymm9, %ymm9;
	vpxor %ymm8, %ymm10, %ymm10;
	vpxor %ymm8, %ymm11, %ymm11;
	vpxor %ymm8, %ymm12, %ymm12;
	vpxor %ymm8, %ymm13, %ymm13;
	vpxor %ymm8, %ymm14, %ymm14;
	vaesdeclast %ymm9, %ymm0, %ymm0;
	vaesdeclast %ymm10, %ymm1, %ymm1;
	vpxor (-5 * 16)(%rcx), %ymm8, %ymm9;
	vpxor (-3 * 16)(%rcx), %ymm8, %ymm10;
	vaesdeclast %ymm11, %ymm2, %ymm2;
	vaesdeclast %ymm12, %ymm3, %ymm3;
	vaesdeclast %ymm13, %ymm4, %ymm4;
	vaesdeclast %ymm14, %ymm5, %ymm5;
	vaesdeclast %ymm9, %ymm6, %ymm6;
	vaesdeclast %ymm10, %ymm7, %ymm7;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	vmovdqu %ymm4, (8 * 16)(%rdx);
	vmovdqu %ymm5, (10 * 16)(%rdx);
	vmovdqu %ymm6, (12 * 16)(%rdx);
	vmovdqu %ymm7, (14 * 16)(%rdx);
	leaq (16 * 16)(%rdx), %rdx;

	jmp .Lcbc_dec_blk16;

	/* Handle trailing eight blocks. */
.align 8
.Lcbc_dec_blk8:
	cmpq $8, %r8;
	jb .Lcbc_dec_blk4;

	leaq -8(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vmovdqu (4 * 16)(%rcx), %ymm2;
	vmovdqu (6 * 16)(%rcx), %ymm3;
	vinserti128 $1, %xmm0, %ymm15, %ymm10;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vpxor %ymm4, %ymm2, %ymm2;
	vpxor %ymm4, %ymm3, %ymm3;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	vmovdqu (1 * 16)(%rcx), %ymm11;
	vmovdqu (3 * 16)(%rcx), %ymm12;
	vmovdqu (5 * 16)(%rcx), %ymm13;
	vmovdqu (7 * 16)(%rcx), %xmm15;
	leaq (8 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lcbc_dec_blk8_last;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lcbc_dec_blk8_last;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lcbc_dec_blk8_last:
	vpxor %ymm4, %ymm10, %ymm10;
	vpxor %ymm4, %ymm11, %ymm11;
	vpxor %ymm4, %ymm12, %ymm12;
	vpxor %ymm4, %ymm13, %ymm13;
	vaesdeclast %ymm10, %ymm0, %ymm0;
	vaesdeclast %ymm11, %ymm1, %ymm1;
	vaesdeclast %ymm12, %ymm2, %ymm2;
	vaesdeclast %ymm13, %ymm3, %ymm3;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	leaq (8 * 16)(%rdx), %rdx;

	/* Handle trailing four blocks. */
.align 8
.Lcbc_dec_blk4:
	cmpq $4, %r8;
	jb .Lcbc_dec_blk1;

	leaq -4(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vinserti128 $1, %xmm0, %ymm15, %ymm10;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	vmovdqu (1 * 16)(%rcx), %ymm11;
	vmovdqu (3 * 16)(%rcx), %xmm15;
	leaq (4 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lcbc_dec_blk4_last;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lcbc_dec_blk4_last;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESDEC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lcbc_dec_blk4_last:
	vpxor %ymm4, %ymm10, %ymm10;
	vpxor %ymm4, %ymm11, %ymm11;
	vaesdeclast %ymm10, %ymm0, %ymm0;
	vaesdeclast %ymm11, %ymm1, %ymm1;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	leaq (4 * 16)(%rdx), %rdx;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lcbc_dec_blk1:
	cmpq $1, %r8;
	jb .Ldone_cbc_dec;

	leaq -1(%r8), %r8;

	/* Load input. */
	vmovdqu (%rcx), %xmm2;
	leaq 16(%rcx), %rcx;

	/* Xor first key. */
	vpxor (0 * 16)(%rdi), %xmm2, %xmm0;

	/* AES rounds. */
	vaesdec (1 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (2 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (3 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (4 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (5 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (6 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (7 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (8 * 16)(%rdi), %xmm0, %xmm0;
	vaesdec (9 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (10 * 16)(%rdi), %xmm1;
	cmpl $12, %r9d;
	jb .Lcbc_dec_blk1_last;
	vaesdec %xmm1, %xmm0, %xmm0;
	vaesdec (11 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (12 * 16)(%rdi), %xmm1;
	jz .Lcbc_dec_blk1_last;
	vaesdec %xmm1, %xmm0, %xmm0;
	vaesdec (13 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (14 * 16)(%rdi), %xmm1;

	/* Last round and output handling. */
  .Lcbc_dec_blk1_last:
	vpxor %xmm1, %xmm15, %xmm15;
	vaesdeclast %xmm15, %xmm0, %xmm0;
	vmovdqa %xmm2, %xmm15;
	vmovdqu %xmm0, (%rdx);
	leaq 16(%rdx), %rdx;

	jmp .Lcbc_dec_blk1;

.align 8
.Ldone_cbc_dec:
	/* Store IV. */
	vmovdqu %xmm15, (%rsi);

	vzeroall;
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_cbc_dec_amd64,.-_gcry_vaes_avx2_cbc_dec_amd64)

/**********************************************************************
  CFB-mode decryption
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_cfb_dec_amd64,@function)
.globl _gcry_vaes_avx2_cfb_dec_amd64
.align 16
_gcry_vaes_avx2_cfb_dec_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%rsi: iv
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 */
	CFI_STARTPROC();

	/* Load IV. */
	vmovdqu (%rsi), %xmm15;

	/* Process 16 blocks per loop. */
.align 8
.Lcfb_dec_blk16:
	cmpq $16, %r8;
	jb .Lcfb_dec_blk8;

	leaq -16(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm8;
	vmovdqu (0 * 16)(%rcx), %ymm9;
	vinserti128 $1, %xmm9, %ymm15, %ymm0;
	vmovdqu (1 * 16)(%rcx), %ymm1;
	vmovdqu (3 * 16)(%rcx), %ymm2;
	vmovdqu (5 * 16)(%rcx), %ymm3;
	vmovdqu (7 * 16)(%rcx), %ymm4;
	vmovdqu (9 * 16)(%rcx), %ymm5;
	vmovdqu (11 * 16)(%rcx), %ymm6;
	vmovdqu (13 * 16)(%rcx), %ymm7;
	vmovdqu (15 * 16)(%rcx), %xmm15;
	vpxor %ymm8, %ymm0, %ymm0;
	vpxor %ymm8, %ymm1, %ymm1;
	vpxor %ymm8, %ymm2, %ymm2;
	vpxor %ymm8, %ymm3, %ymm3;
	vpxor %ymm8, %ymm4, %ymm4;
	vpxor %ymm8, %ymm5, %ymm5;
	vpxor %ymm8, %ymm6, %ymm6;
	vpxor %ymm8, %ymm7, %ymm7;
	vbroadcasti128 (1 * 16)(%rdi), %ymm8;
	vmovdqu (2 * 16)(%rcx), %ymm10;
	vmovdqu (4 * 16)(%rcx), %ymm11;
	vmovdqu (6 * 16)(%rcx), %ymm12;
	vmovdqu (8 * 16)(%rcx), %ymm13;
	vmovdqu (10 * 16)(%rcx), %ymm14;

	leaq (16 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (2 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (3 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (4 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (5 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (6 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (7 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (8 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (9 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (10 * 16)(%rdi), %ymm8;
	cmpl $12, %r9d;
	jb .Lcfb_dec_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (11 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (12 * 16)(%rdi), %ymm8;
	jz .Lcfb_dec_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (13 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (14 * 16)(%rdi), %ymm8;

	/* Last round and output handling. */
  .Lcfb_dec_blk16_last:
	vpxor %ymm8, %ymm9, %ymm9;
	vpxor %ymm8, %ymm10, %ymm10;
	vpxor %ymm8, %ymm11, %ymm11;
	vpxor %ymm8, %ymm12, %ymm12;
	vpxor %ymm8, %ymm13, %ymm13;
	vpxor %ymm8, %ymm14, %ymm14;
	vaesenclast %ymm9, %ymm0, %ymm0;
	vaesenclast %ymm10, %ymm1, %ymm1;
	vpxor (-4 * 16)(%rcx), %ymm8, %ymm9;
	vpxor (-2 * 16)(%rcx), %ymm8, %ymm10;
	vaesenclast %ymm11, %ymm2, %ymm2;
	vaesenclast %ymm12, %ymm3, %ymm3;
	vaesenclast %ymm13, %ymm4, %ymm4;
	vaesenclast %ymm14, %ymm5, %ymm5;
	vaesenclast %ymm9, %ymm6, %ymm6;
	vaesenclast %ymm10, %ymm7, %ymm7;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	vmovdqu %ymm4, (8 * 16)(%rdx);
	vmovdqu %ymm5, (10 * 16)(%rdx);
	vmovdqu %ymm6, (12 * 16)(%rdx);
	vmovdqu %ymm7, (14 * 16)(%rdx);
	leaq (16 * 16)(%rdx), %rdx;

	jmp .Lcfb_dec_blk16;

	/* Handle trailing eight blocks. */
.align 8
.Lcfb_dec_blk8:
	cmpq $8, %r8;
	jb .Lcfb_dec_blk4;

	leaq -8(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm10;
	vinserti128 $1, %xmm10, %ymm15, %ymm0;
	vmovdqu (1 * 16)(%rcx), %ymm1;
	vmovdqu (3 * 16)(%rcx), %ymm2;
	vmovdqu (5 * 16)(%rcx), %ymm3;
	vmovdqu (7 * 16)(%rcx), %xmm15;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vpxor %ymm4, %ymm2, %ymm2;
	vpxor %ymm4, %ymm3, %ymm3;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	vmovdqu (2 * 16)(%rcx), %ymm11;
	vmovdqu (4 * 16)(%rcx), %ymm12;
	vmovdqu (6 * 16)(%rcx), %ymm13;

	leaq (8 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lcfb_dec_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lcfb_dec_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lcfb_dec_blk8_last:
	vpxor %ymm4, %ymm10, %ymm10;
	vpxor %ymm4, %ymm11, %ymm11;
	vpxor %ymm4, %ymm12, %ymm12;
	vpxor %ymm4, %ymm13, %ymm13;
	vaesenclast %ymm10, %ymm0, %ymm0;
	vaesenclast %ymm11, %ymm1, %ymm1;
	vaesenclast %ymm12, %ymm2, %ymm2;
	vaesenclast %ymm13, %ymm3, %ymm3;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	leaq (8 * 16)(%rdx), %rdx;

	/* Handle trailing four blocks. */
.align 8
.Lcfb_dec_blk4:
	cmpq $4, %r8;
	jb .Lcfb_dec_blk1;

	leaq -4(%r8), %r8;

	/* Load input and xor first key. Update IV. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm10;
	vinserti128 $1, %xmm10, %ymm15, %ymm0;
	vmovdqu (1 * 16)(%rcx), %ymm1;
	vmovdqu (3 * 16)(%rcx), %xmm15;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	vmovdqu (2 * 16)(%rcx), %ymm11;

	leaq (4 * 16)(%rcx), %rcx;

	/* AES rounds */
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lcfb_dec_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lcfb_dec_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lcfb_dec_blk4_last:
	vpxor %ymm4, %ymm10, %ymm10;
	vpxor %ymm4, %ymm11, %ymm11;
	vaesenclast %ymm10, %ymm0, %ymm0;
	vaesenclast %ymm11, %ymm1, %ymm1;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	leaq (4 * 16)(%rdx), %rdx;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lcfb_dec_blk1:
	cmpq $1, %r8;
	jb .Ldone_cfb_dec;

	leaq -1(%r8), %r8;

	/* Xor first key. */
	vpxor (0 * 16)(%rdi), %xmm15, %xmm0;

	/* Load input as next IV. */
	vmovdqu (%rcx), %xmm15;
	leaq 16(%rcx), %rcx;

	/* AES rounds. */
	vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (10 * 16)(%rdi), %xmm1;
	cmpl $12, %r9d;
	jb .Lcfb_dec_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (12 * 16)(%rdi), %xmm1;
	jz .Lcfb_dec_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (14 * 16)(%rdi), %xmm1;

	/* Last round and output handling. */
  .Lcfb_dec_blk1_last:
	vpxor %xmm15, %xmm1, %xmm1;
	vaesenclast %xmm1, %xmm0, %xmm0;
	vmovdqu %xmm0, (%rdx);
	leaq 16(%rdx), %rdx;

	jmp .Lcfb_dec_blk1;

.align 8
.Ldone_cfb_dec:
	/* Store IV. */
	vmovdqu %xmm15, (%rsi);

	vzeroall;
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_cfb_dec_amd64,.-_gcry_vaes_avx2_cfb_dec_amd64)

/**********************************************************************
  CTR-mode encryption
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_ctr_enc_amd64,@function)
.globl _gcry_vaes_avx2_ctr_enc_amd64
.align 16
_gcry_vaes_avx2_ctr_enc_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%rsi: counter
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 */
	CFI_STARTPROC();

	movq 8(%rsi), %r10;
	movq 0(%rsi), %r11;
	bswapq %r10;
	bswapq %r11;

	vpcmpeqd %ymm15, %ymm15, %ymm15;
	vpsrldq $8, %ymm15, %ymm15;     // 0:-1
	vpaddq %ymm15, %ymm15, %ymm14;  // 0:-2
	vbroadcasti128 .Lbswap128_mask rRIP, %ymm13;

#define inc_le128(x, minus_one, tmp) \
	vpcmpeqq minus_one, x, tmp; \
	vpsubq minus_one, x, x; \
	vpslldq $8, tmp, tmp; \
	vpsubq tmp, x, x;

#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \
	vpcmpeqq minus_one, x, tmp1; \
	vpcmpeqq minus_two, x, tmp2; \
	vpor tmp1, tmp2, tmp2; \
	vpsubq minus_two, x, x; \
	vpslldq $8, tmp2, tmp2; \
	vpsubq tmp2, x, x;

#define handle_ctr_128bit_add(nblks) \
	addq $(nblks), %r10; \
	adcq $0, %r11; \
	bswapq %r10; \
	bswapq %r11; \
	movq %r10, 8(%rsi); \
	movq %r11, 0(%rsi); \
	bswapq %r10; \
	bswapq %r11;

	/* Process 16 blocks per loop. */
.align 8
.Lctr_enc_blk16:
	cmpq $16, %r8;
	jb .Lctr_enc_blk8;

	leaq -16(%r8), %r8;

	vbroadcasti128 (%rsi), %ymm7;
	vbroadcasti128 (0 * 16)(%rdi), %ymm8;

	/* detect if carry handling is needed */
	addb $16, 15(%rsi);
	jc .Lctr_enc_blk16_handle_carry;

	leaq 16(%r10), %r10;

  .Lctr_enc_blk16_byte_bige_add:
	/* Increment counters. */
	vpaddb .Lbige_addb_0 rRIP, %ymm7, %ymm0;
	vpaddb .Lbige_addb_2 rRIP, %ymm7, %ymm1;
	vpaddb .Lbige_addb_4 rRIP, %ymm7, %ymm2;
	vpaddb .Lbige_addb_6 rRIP, %ymm7, %ymm3;
	vpaddb .Lbige_addb_8 rRIP, %ymm7, %ymm4;
	vpaddb .Lbige_addb_10 rRIP, %ymm7, %ymm5;
	vpaddb .Lbige_addb_12 rRIP, %ymm7, %ymm6;
	vpaddb .Lbige_addb_14 rRIP, %ymm7, %ymm7;

  .Lctr_enc_blk16_rounds:
	/* AES rounds */
	XOR8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (1 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (2 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (3 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (4 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (5 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (6 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (7 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (8 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (9 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (10 * 16)(%rdi), %ymm8;
	cmpl $12, %r9d;
	jb .Lctr_enc_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (11 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (12 * 16)(%rdi), %ymm8;
	jz .Lctr_enc_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (13 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (14 * 16)(%rdi), %ymm8;

	/* Last round and output handling. */
  .Lctr_enc_blk16_last:
	vpxor (0 * 16)(%rcx), %ymm8, %ymm9; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm8, %ymm10;
	vpxor (4 * 16)(%rcx), %ymm8, %ymm11;
	vpxor (6 * 16)(%rcx), %ymm8, %ymm12;
	vaesenclast %ymm9, %ymm0, %ymm0;
	vaesenclast %ymm10, %ymm1, %ymm1;
	vaesenclast %ymm11, %ymm2, %ymm2;
	vaesenclast %ymm12, %ymm3, %ymm3;
	vpxor (8 * 16)(%rcx), %ymm8, %ymm9;
	vpxor (10 * 16)(%rcx), %ymm8, %ymm10;
	vpxor (12 * 16)(%rcx), %ymm8, %ymm11;
	vpxor (14 * 16)(%rcx), %ymm8, %ymm8;
	leaq (16 * 16)(%rcx), %rcx;
	vaesenclast %ymm9, %ymm4, %ymm4;
	vaesenclast %ymm10, %ymm5, %ymm5;
	vaesenclast %ymm11, %ymm6, %ymm6;
	vaesenclast %ymm8, %ymm7, %ymm7;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	vmovdqu %ymm4, (8 * 16)(%rdx);
	vmovdqu %ymm5, (10 * 16)(%rdx);
	vmovdqu %ymm6, (12 * 16)(%rdx);
	vmovdqu %ymm7, (14 * 16)(%rdx);
	leaq (16 * 16)(%rdx), %rdx;

	jmp .Lctr_enc_blk16;

  .align 8
  .Lctr_enc_blk16_handle_only_ctr_carry:
	handle_ctr_128bit_add(16);
	jmp .Lctr_enc_blk16_byte_bige_add;

  .align 8
  .Lctr_enc_blk16_handle_carry:
	jz .Lctr_enc_blk16_handle_only_ctr_carry;
	/* Increment counters (handle carry). */
	vpshufb %xmm13, %xmm7, %xmm1; /* be => le */
	vmovdqa %xmm1, %xmm0;
	inc_le128(%xmm1, %xmm15, %xmm5);
	vinserti128 $1, %xmm1, %ymm0, %ymm7; /* ctr: +1:+0 */
	vpshufb %ymm13, %ymm7, %ymm0;
	handle_ctr_128bit_add(16);
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +3:+2 */
	vpshufb %ymm13, %ymm7, %ymm1;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +5:+4 */
	vpshufb %ymm13, %ymm7, %ymm2;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +7:+6 */
	vpshufb %ymm13, %ymm7, %ymm3;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +9:+8 */
	vpshufb %ymm13, %ymm7, %ymm4;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +11:+10 */
	vpshufb %ymm13, %ymm7, %ymm5;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +13:+12 */
	vpshufb %ymm13, %ymm7, %ymm6;
	add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +15:+14 */
	vpshufb %ymm13, %ymm7, %ymm7;

	jmp .Lctr_enc_blk16_rounds;

	/* Handle trailing eight blocks. */
.align 8
.Lctr_enc_blk8:
	cmpq $8, %r8;
	jb .Lctr_enc_blk4;

	leaq -8(%r8), %r8;

	vbroadcasti128 (%rsi), %ymm3;
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;

	/* detect if carry handling is needed */
	addb $8, 15(%rsi);
	jc .Lctr_enc_blk8_handle_carry;

	leaq 8(%r10), %r10;

  .Lctr_enc_blk8_byte_bige_add:
	/* Increment counters. */
	vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0;
	vpaddb .Lbige_addb_2 rRIP, %ymm3, %ymm1;
	vpaddb .Lbige_addb_4 rRIP, %ymm3, %ymm2;
	vpaddb .Lbige_addb_6 rRIP, %ymm3, %ymm3;

  .Lctr_enc_blk8_rounds:
	/* AES rounds */
	XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lctr_enc_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lctr_enc_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lctr_enc_blk8_last:
	vpxor (0 * 16)(%rcx), %ymm4, %ymm5; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm4, %ymm6;
	vpxor (4 * 16)(%rcx), %ymm4, %ymm7;
	vpxor (6 * 16)(%rcx), %ymm4, %ymm4;
	leaq (8 * 16)(%rcx), %rcx;
	vaesenclast %ymm5, %ymm0, %ymm0;
	vaesenclast %ymm6, %ymm1, %ymm1;
	vaesenclast %ymm7, %ymm2, %ymm2;
	vaesenclast %ymm4, %ymm3, %ymm3;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	leaq (8 * 16)(%rdx), %rdx;

	jmp .Lctr_enc_blk4;

  .align 8
  .Lctr_enc_blk8_handle_only_ctr_carry:
	handle_ctr_128bit_add(8);
	jmp .Lctr_enc_blk8_byte_bige_add;

  .align 8
  .Lctr_enc_blk8_handle_carry:
	jz .Lctr_enc_blk8_handle_only_ctr_carry;
	/* Increment counters (handle carry). */
	vpshufb %xmm13, %xmm3, %xmm1; /* be => le */
	vmovdqa %xmm1, %xmm0;
	inc_le128(%xmm1, %xmm15, %xmm5);
	vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
	vpshufb %ymm13, %ymm3, %ymm0;
	handle_ctr_128bit_add(8);
	add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */
	vpshufb %ymm13, %ymm3, %ymm1;
	add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +5:+4 */
	vpshufb %ymm13, %ymm3, %ymm2;
	add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +7:+6 */
	vpshufb %ymm13, %ymm3, %ymm3;

	jmp .Lctr_enc_blk8_rounds;

	/* Handle trailing four blocks. */
.align 8
.Lctr_enc_blk4:
	cmpq $4, %r8;
	jb .Lctr_enc_blk1;

	leaq -4(%r8), %r8;

	vbroadcasti128 (%rsi), %ymm3;
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;

	/* detect if carry handling is needed */
	addb $4, 15(%rsi);
	jc .Lctr_enc_blk4_handle_carry;

	leaq 4(%r10), %r10;

  .Lctr_enc_blk4_byte_bige_add:
	/* Increment counters. */
	vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0;
	vpaddb .Lbige_addb_2 rRIP, %ymm3, %ymm1;

  .Lctr_enc_blk4_rounds:
	/* AES rounds */
	XOR2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lctr_enc_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lctr_enc_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lctr_enc_blk4_last:
	vpxor (0 * 16)(%rcx), %ymm4, %ymm5; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm4, %ymm6;
	leaq (4 * 16)(%rcx), %rcx;
	vaesenclast %ymm5, %ymm0, %ymm0;
	vaesenclast %ymm6, %ymm1, %ymm1;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	leaq (4 * 16)(%rdx), %rdx;

	jmp .Lctr_enc_blk1;

  .align 8
  .Lctr_enc_blk4_handle_only_ctr_carry:
	handle_ctr_128bit_add(4);
	jmp .Lctr_enc_blk4_byte_bige_add;

  .align 8
  .Lctr_enc_blk4_handle_carry:
	jz .Lctr_enc_blk4_handle_only_ctr_carry;
	/* Increment counters (handle carry). */
	vpshufb %xmm13, %xmm3, %xmm1; /* be => le */
	vmovdqa %xmm1, %xmm0;
	inc_le128(%xmm1, %xmm15, %xmm5);
	vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
	vpshufb %ymm13, %ymm3, %ymm0;
	handle_ctr_128bit_add(4);
	add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */
	vpshufb %ymm13, %ymm3, %ymm1;

	jmp .Lctr_enc_blk4_rounds;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lctr_enc_blk1:
	cmpq $1, %r8;
	jb .Ldone_ctr_enc;

	leaq -1(%r8), %r8;

	/* Load and increament counter. */
	vmovdqu (%rsi), %xmm0;
	handle_ctr_128bit_add(1);

	/* AES rounds. */
	vpxor (0 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (10 * 16)(%rdi), %xmm1;
	cmpl $12, %r9d;
	jb .Lctr_enc_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (12 * 16)(%rdi), %xmm1;
	jz .Lctr_enc_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (14 * 16)(%rdi), %xmm1;

	/* Last round and output handling. */
  .Lctr_enc_blk1_last:
	vpxor (%rcx), %xmm1, %xmm1; /* Xor src to last round key. */
	leaq 16(%rcx), %rcx;
	vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */
	vmovdqu %xmm0, (%rdx);
	leaq 16(%rdx), %rdx;

	jmp .Lctr_enc_blk1;

.align 8
.Ldone_ctr_enc:
	vzeroall;
	xorl %r10d, %r10d;
	xorl %r11d, %r11d;
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_ctr_enc_amd64,.-_gcry_vaes_avx2_ctr_enc_amd64)

/**********************************************************************
  Little-endian 32-bit CTR-mode encryption (GCM-SIV)
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_ctr32le_enc_amd64,@function)
.globl _gcry_vaes_avx2_ctr32le_enc_amd64
.align 16
_gcry_vaes_avx2_ctr32le_enc_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%rsi: counter
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 */
	CFI_STARTPROC();

	vbroadcasti128 (%rsi), %ymm15; // CTR

	/* Process 16 blocks per loop. */
.align 8
.Lctr32le_enc_blk16:
	cmpq $16, %r8;
	jb .Lctr32le_enc_blk8;

	leaq -16(%r8), %r8;

	vbroadcasti128 (0 * 16)(%rdi), %ymm8;

	/* Increment counters. */
	vpaddd .Lle_addd_0 rRIP, %ymm15, %ymm0;
	vpaddd .Lle_addd_2 rRIP, %ymm15, %ymm1;
	vpaddd .Lle_addd_4 rRIP, %ymm15, %ymm2;
	vpaddd .Lle_addd_6 rRIP, %ymm15, %ymm3;
	vpaddd .Lle_addd_8 rRIP, %ymm15, %ymm4;
	vpaddd .Lle_addd_10 rRIP, %ymm15, %ymm5;
	vpaddd .Lle_addd_12 rRIP, %ymm15, %ymm6;
	vpaddd .Lle_addd_14 rRIP, %ymm15, %ymm7;

	vpaddd .Lle_addd_16_2 rRIP, %ymm15, %ymm15;

	/* AES rounds */
	XOR8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (1 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (2 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (3 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (4 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (5 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (6 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (7 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (8 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (9 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (10 * 16)(%rdi), %ymm8;
	cmpl $12, %r9d;
	jb .Lctr32le_enc_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (11 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (12 * 16)(%rdi), %ymm8;
	jz .Lctr32le_enc_blk16_last;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (13 * 16)(%rdi), %ymm8;
	VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
	vbroadcasti128 (14 * 16)(%rdi), %ymm8;

	/* Last round and output handling. */
  .Lctr32le_enc_blk16_last:
	vpxor (0 * 16)(%rcx), %ymm8, %ymm9; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm8, %ymm10;
	vpxor (4 * 16)(%rcx), %ymm8, %ymm11;
	vpxor (6 * 16)(%rcx), %ymm8, %ymm12;
	vaesenclast %ymm9, %ymm0, %ymm0;
	vaesenclast %ymm10, %ymm1, %ymm1;
	vaesenclast %ymm11, %ymm2, %ymm2;
	vaesenclast %ymm12, %ymm3, %ymm3;
	vpxor (8 * 16)(%rcx), %ymm8, %ymm9;
	vpxor (10 * 16)(%rcx), %ymm8, %ymm10;
	vpxor (12 * 16)(%rcx), %ymm8, %ymm11;
	vpxor (14 * 16)(%rcx), %ymm8, %ymm8;
	leaq (16 * 16)(%rcx), %rcx;
	vaesenclast %ymm9, %ymm4, %ymm4;
	vaesenclast %ymm10, %ymm5, %ymm5;
	vaesenclast %ymm11, %ymm6, %ymm6;
	vaesenclast %ymm8, %ymm7, %ymm7;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	vmovdqu %ymm4, (8 * 16)(%rdx);
	vmovdqu %ymm5, (10 * 16)(%rdx);
	vmovdqu %ymm6, (12 * 16)(%rdx);
	vmovdqu %ymm7, (14 * 16)(%rdx);
	leaq (16 * 16)(%rdx), %rdx;

	jmp .Lctr32le_enc_blk16;

	/* Handle trailing eight blocks. */
.align 8
.Lctr32le_enc_blk8:
	cmpq $8, %r8;
	jb .Lctr32le_enc_blk4;

	leaq -8(%r8), %r8;

	vbroadcasti128 (0 * 16)(%rdi), %ymm4;

	/* Increment counters. */
	vpaddd .Lle_addd_0 rRIP, %ymm15, %ymm0;
	vpaddd .Lle_addd_2 rRIP, %ymm15, %ymm1;
	vpaddd .Lle_addd_4 rRIP, %ymm15, %ymm2;
	vpaddd .Lle_addd_6 rRIP, %ymm15, %ymm3;

	vpaddd .Lle_addd_8_2 rRIP, %ymm15, %ymm15;

	/* AES rounds */
	XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lctr32le_enc_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lctr32le_enc_blk8_last;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lctr32le_enc_blk8_last:
	vpxor (0 * 16)(%rcx), %ymm4, %ymm5; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm4, %ymm6;
	vpxor (4 * 16)(%rcx), %ymm4, %ymm7;
	vpxor (6 * 16)(%rcx), %ymm4, %ymm4;
	leaq (8 * 16)(%rcx), %rcx;
	vaesenclast %ymm5, %ymm0, %ymm0;
	vaesenclast %ymm6, %ymm1, %ymm1;
	vaesenclast %ymm7, %ymm2, %ymm2;
	vaesenclast %ymm4, %ymm3, %ymm3;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	leaq (8 * 16)(%rdx), %rdx;

	/* Handle trailing four blocks. */
.align 8
.Lctr32le_enc_blk4:
	cmpq $4, %r8;
	jb .Lctr32le_enc_blk1;

	leaq -4(%r8), %r8;

	vbroadcasti128 (0 * 16)(%rdi), %ymm4;

	/* Increment counters. */
	vpaddd .Lle_addd_0 rRIP, %ymm15, %ymm0;
	vpaddd .Lle_addd_2 rRIP, %ymm15, %ymm1;

	vpaddd .Lle_addd_4_2 rRIP, %ymm15, %ymm15;

	/* AES rounds */
	XOR2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (2 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (3 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (4 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (5 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (6 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (7 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (8 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (9 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (10 * 16)(%rdi), %ymm4;
	cmpl $12, %r9d;
	jb .Lctr32le_enc_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (11 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (12 * 16)(%rdi), %ymm4;
	jz .Lctr32le_enc_blk4_last;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (13 * 16)(%rdi), %ymm4;
	VAESENC2(%ymm4, %ymm0, %ymm1);
	vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	/* Last round and output handling. */
  .Lctr32le_enc_blk4_last:
	vpxor (0 * 16)(%rcx), %ymm4, %ymm5; /* Xor src to last round key. */
	vpxor (2 * 16)(%rcx), %ymm4, %ymm6;
	leaq (4 * 16)(%rcx), %rcx;
	vaesenclast %ymm5, %ymm0, %ymm0;
	vaesenclast %ymm6, %ymm1, %ymm1;
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	leaq (4 * 16)(%rdx), %rdx;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lctr32le_enc_blk1:
	cmpq $1, %r8;
	jb .Ldone_ctr32le_enc;

	leaq -1(%r8), %r8;

	/* Load and increament counter. */
	vmovdqu %xmm15, %xmm0;
	vpaddd .Lle_addd_1 rRIP, %xmm15, %xmm15;

	/* AES rounds. */
	vpxor (0 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
	vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (10 * 16)(%rdi), %xmm1;
	cmpl $12, %r9d;
	jb .Lctr32le_enc_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (12 * 16)(%rdi), %xmm1;
	jz .Lctr32le_enc_blk1_last;
	vaesenc %xmm1, %xmm0, %xmm0;
	vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
	vmovdqa (14 * 16)(%rdi), %xmm1;

	/* Last round and output handling. */
  .Lctr32le_enc_blk1_last:
	vpxor (%rcx), %xmm1, %xmm1; /* Xor src to last round key. */
	leaq 16(%rcx), %rcx;
	vaesenclast %xmm1, %xmm0, %xmm0; /* Last round and xor with xmm1. */
	vmovdqu %xmm0, (%rdx);
	leaq 16(%rdx), %rdx;

	jmp .Lctr32le_enc_blk1;

.align 8
.Ldone_ctr32le_enc:
	vmovdqu %xmm15, (%rsi);
	vzeroall;
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_ctr32le_enc_amd64,.-_gcry_vaes_avx2_ctr32le_enc_amd64)

/**********************************************************************
  OCB-mode encryption/decryption/authentication
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_ocb_crypt_amd64,@function)
.globl _gcry_vaes_avx2_ocb_crypt_amd64
.align 16
_gcry_vaes_avx2_ocb_crypt_amd64:
	/* input:
	 *	%rdi:     round keys
	 *	%esi:     nblk
	 *	%rdx:     dst
	 *	%rcx:     src
	 *	%r8:      nblocks
	 *	%r9:      nrounds
	 *	16(%rbp): offset
	 *	24(%rbp): checksum
	 *	32(%rbp): L-array
	 *	40(%rbp): decrypt/encrypt/auth (%r15d)
	 */
	CFI_STARTPROC();

#define STACK_REGS_POS (16 * 16 + 4 * 16 + 2 * 16)
#define STACK_ALLOC    (STACK_REGS_POS + 5 * 8)
#define OFFSET_PTR_Q   16(%rbp)
#define CHECKSUM_PTR_Q 24(%rbp)
#define L_ARRAY_PTR_L  32(%rbp)
#define OPER_MODE_L    40(%rbp)

	pushq %rbp;
	CFI_PUSH(%rbp);
	movq %rsp, %rbp;
	CFI_DEF_CFA_REGISTER(%rbp);

	subq $STACK_ALLOC, %rsp;
	andq $~63, %rsp;

	movq %r12, (STACK_REGS_POS + 0 * 8)(%rsp);
	CFI_REG_ON_STACK(r12, STACK_REGS_POS + 0 * 8);
	movq %r13, (STACK_REGS_POS + 1 * 8)(%rsp);
	CFI_REG_ON_STACK(r13, STACK_REGS_POS + 1 * 8);
	movq %r14, (STACK_REGS_POS + 2 * 8)(%rsp);
	CFI_REG_ON_STACK(r14, STACK_REGS_POS + 2 * 8);
	movq %r15, (STACK_REGS_POS + 3 * 8)(%rsp);
	CFI_REG_ON_STACK(r15, STACK_REGS_POS + 3 * 8);
	movq %rbx, (STACK_REGS_POS + 4 * 8)(%rsp);
	CFI_REG_ON_STACK(rbx, STACK_REGS_POS + 4 * 8);

	movl OPER_MODE_L, %r15d; /* decrypt/encrypt/auth-mode. */
	movq OFFSET_PTR_Q, %r14; /* offset ptr. */
	movq CHECKSUM_PTR_Q, %rbx; /* checksum ptr. */

	leal (, %r9d, 4), %eax;
	vmovdqu (%r14), %xmm15; /* Load offset. */
	movq L_ARRAY_PTR_L, %r14; /* L-array ptr. */
	vmovdqa (0 * 16)(%rdi), %xmm0; /* first key */
	vpxor %xmm14, %xmm14, %xmm14;
	vpxor %xmm13, %xmm13, %xmm13;
	vpxor (%rdi, %rax, 4), %xmm0, %xmm0; /* first key ^ last key */
	vpxor (0 * 16)(%rdi), %xmm15, %xmm15; /* offset ^ first key */
	vmovdqa %xmm0, (14 * 16)(%rsp);
	vmovdqa %xmm0, (15 * 16)(%rsp);

.align 8
.Lhandle_unaligned_ocb:
	/* Get number of blocks to align nblk to 16 (and L-array optimization). */
	movl %esi, %r10d;
	negl %r10d;
	andl $15, %r10d;
	cmpq %r8, %r10;
	cmovaq %r8, %r10;
	cmpq $1, %r10;
	jb .Lunaligned_ocb_done;

	/* Number of blocks after alignment. */
	movq %r8, %r11;
	subq %r10, %r11;

	/* If number after alignment is less than 16, skip aligned handling
	 * completely. */
	cmp $16, %r11;
	cmovbq %r8, %r10;

	/* Unaligned: Process eight blocks per loop. */
.align 8
.Locb_unaligned_blk8:
	cmpq $8, %r10;
	jb .Locb_unaligned_blk4;

	leaq -8(%r8), %r8;
	leaq -8(%r10), %r10;

	leal 1(%esi), %r11d;
	leal 2(%esi), %r12d;
	leal 3(%esi), %r13d;
	leal 4(%esi), %eax;
	tzcntl %r11d, %r11d;
	tzcntl %r12d, %r12d;
	tzcntl %r13d, %r13d;
	tzcntl %eax, %eax;
	shll $4, %r11d;
	shll $4, %r12d;
	shll $4, %r13d;
	shll $4, %eax;
	vpxor (%r14, %r11), %xmm15, %xmm5;
	vpxor (%r14, %r12), %xmm5, %xmm6;
	vpxor (%r14, %r13), %xmm6, %xmm7;
	vpxor (%r14, %rax), %xmm7, %xmm8;

	leal 5(%esi), %r11d;
	leal 6(%esi), %r12d;
	leal 7(%esi), %r13d;
	leal 8(%esi), %esi;
	tzcntl %r11d, %r11d;
	tzcntl %r12d, %r12d;
	tzcntl %r13d, %r13d;
	tzcntl %esi, %eax;
	shll $4, %r11d;
	shll $4, %r12d;
	shll $4, %r13d;
	shll $4, %eax;
	vpxor (%r14, %r11), %xmm8, %xmm9;
	vpxor (%r14, %r12), %xmm9, %xmm10;
	vpxor (%r14, %r13), %xmm10, %xmm11;
	vpxor (%r14, %rax), %xmm11, %xmm15;

	vinserti128 $1, %xmm6, %ymm5, %ymm5;
	vinserti128 $1, %xmm8, %ymm7, %ymm6;
	vinserti128 $1, %xmm10, %ymm9, %ymm7;
	vinserti128 $1, %xmm15, %ymm11, %ymm8;

	cmpl $1, %r15d;
	jb .Locb_unaligned_blk8_dec;
	ja .Locb_unaligned_blk8_auth;
		vmovdqu (0 * 16)(%rcx), %ymm0;
		vmovdqu (2 * 16)(%rcx), %ymm1;
		vmovdqu (4 * 16)(%rcx), %ymm2;
		vmovdqu (6 * 16)(%rcx), %ymm3;
		leaq (8 * 16)(%rcx), %rcx;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;
		vpxor %ymm2, %ymm14, %ymm14;
		vpxor %ymm3, %ymm13, %ymm13;
		vpxor %ymm5, %ymm0, %ymm0;
		vpxor %ymm6, %ymm1, %ymm1;
		vpxor %ymm7, %ymm2, %ymm2;
		vpxor %ymm8, %ymm3, %ymm3;

		vmovdqa (14 * 16)(%rsp), %ymm9;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk8_enc_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		jz .Locb_unaligned_blk8_enc_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);

		/* Last round and output handling. */
	.Locb_unaligned_blk8_enc_last:
		vpxor %ymm5, %ymm9, %ymm5; /* Xor src to last round key. */
		vpxor %ymm6, %ymm9, %ymm6;
		vpxor %ymm7, %ymm9, %ymm7;
		vpxor %ymm8, %ymm9, %ymm4;
		vaesenclast %ymm5, %ymm0, %ymm0;
		vaesenclast %ymm6, %ymm1, %ymm1;
		vaesenclast %ymm7, %ymm2, %ymm2;
		vaesenclast %ymm4, %ymm3, %ymm3;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;

		jmp .Locb_unaligned_blk8;

	.align 8
	.Locb_unaligned_blk8_auth:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm7, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm8, %ymm3;
		leaq (8 * 16)(%rcx), %rcx;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk8_auth_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Locb_unaligned_blk8_auth_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

		/* Last round and output handling. */
	.Locb_unaligned_blk8_auth_last:
		vaesenclast %ymm4, %ymm0, %ymm0;
		vaesenclast %ymm4, %ymm1, %ymm1;
		vaesenclast %ymm4, %ymm2, %ymm2;
		vaesenclast %ymm4, %ymm3, %ymm3;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;
		vpxor %ymm2, %ymm14, %ymm14;
		vpxor %ymm3, %ymm13, %ymm13;

		jmp .Locb_unaligned_blk8;

	.align 8
	.Locb_unaligned_blk8_dec:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm7, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm8, %ymm3;
		leaq (8 * 16)(%rcx), %rcx;

		vmovdqa (14 * 16)(%rsp), %ymm9;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk8_dec_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		jz .Locb_unaligned_blk8_dec_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);

		/* Last round and output handling. */
	.Locb_unaligned_blk8_dec_last:
		vpxor %ymm5, %ymm9, %ymm5; /* Xor src to last round key. */
		vpxor %ymm6, %ymm9, %ymm6;
		vpxor %ymm7, %ymm9, %ymm7;
		vpxor %ymm8, %ymm9, %ymm4;
		vaesdeclast %ymm5, %ymm0, %ymm0;
		vaesdeclast %ymm6, %ymm1, %ymm1;
		vaesdeclast %ymm7, %ymm2, %ymm2;
		vaesdeclast %ymm4, %ymm3, %ymm3;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;
		vpxor %ymm2, %ymm14, %ymm14;
		vpxor %ymm3, %ymm13, %ymm13;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;

		jmp .Locb_unaligned_blk8;

	/* Unaligned: Process four blocks. */
.align 8
.Locb_unaligned_blk4:
	cmpq $4, %r10;
	jb .Locb_unaligned_blk1;

	leaq -4(%r8), %r8;
	leaq -4(%r10), %r10;

	leal 1(%esi), %r11d;
	leal 2(%esi), %r12d;
	leal 3(%esi), %r13d;
	leal 4(%esi), %esi;
	tzcntl %r11d, %r11d;
	tzcntl %r12d, %r12d;
	tzcntl %r13d, %r13d;
	tzcntl %esi, %eax;
	shll $4, %r11d;
	shll $4, %r12d;
	shll $4, %r13d;
	shll $4, %eax;

	vpxor (%r14, %r11), %xmm15, %xmm5;
	vpxor (%r14, %r12), %xmm5, %xmm6;
	vinserti128 $1, %xmm6, %ymm5, %ymm5;
	vpxor (%r14, %r13), %xmm6, %xmm7;
	vpxor (%r14, %rax), %xmm7, %xmm15;
	vinserti128 $1, %xmm15, %ymm7, %ymm6;

	cmpl $1, %r15d;
	jb .Locb_unaligned_blk4_dec;
	ja .Locb_unaligned_blk4_auth;
		vmovdqu (0 * 16)(%rcx), %ymm0;
		vmovdqu (2 * 16)(%rcx), %ymm1;
		leaq (4 * 16)(%rcx), %rcx;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;
		vpxor %ymm5, %ymm0, %ymm0;
		vpxor %ymm6, %ymm1, %ymm1;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk4_enc_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		jz .Locb_unaligned_blk4_enc_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);

	      /* Last round and output handling. */
	.Locb_unaligned_blk4_enc_last:
		vmovdqa (14 * 16)(%rsp), %ymm8;
		vpxor %ymm5, %ymm8, %ymm5; /* Xor src to last round key. */
		vpxor %ymm6, %ymm8, %ymm6;
		vaesenclast %ymm5, %ymm0, %ymm0;
		vaesenclast %ymm6, %ymm1, %ymm1;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;

		jmp .Locb_unaligned_blk1;

	.align 8
	.Locb_unaligned_blk4_auth:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		leaq (4 * 16)(%rcx), %rcx;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk4_auth_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Locb_unaligned_blk4_auth_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

	      /* Last round and output handling. */
	.Locb_unaligned_blk4_auth_last:
		vaesenclast %ymm4, %ymm0, %ymm0;
		vaesenclast %ymm4, %ymm1, %ymm1;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;

		jmp .Locb_unaligned_blk1;

	.align 8
	.Locb_unaligned_blk4_dec:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		leaq (4 * 16)(%rcx), %rcx;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk4_dec_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		jz .Locb_unaligned_blk4_dec_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);

	      /* Last round and output handling. */
	.Locb_unaligned_blk4_dec_last:
		vmovdqa (14 * 16)(%rsp), %ymm8;
		vpxor %ymm5, %ymm8, %ymm5; /* Xor src to last round key. */
		vpxor %ymm6, %ymm8, %ymm6;
		vaesdeclast %ymm5, %ymm0, %ymm0;
		vaesdeclast %ymm6, %ymm1, %ymm1;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %ymm1, %ymm13, %ymm13;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;

	/* Unaligned: Process one block per loop. */
.align 8
.Locb_unaligned_blk1:
	cmpq $1, %r10;
	jb .Lunaligned_ocb_done;

	leaq -1(%r8), %r8;
	leaq -1(%r10), %r10;

	leal 1(%esi), %esi;
	tzcntl %esi, %r11d;
	shll $4, %r11d;
	vpxor (%r14, %r11), %xmm15, %xmm15;

	cmpl $1, %r15d;
	jb .Locb_unaligned_blk1_dec;
	ja .Locb_unaligned_blk1_auth;
		vmovdqu (%rcx), %xmm0;
		vpxor %ymm0, %ymm14, %ymm14;
		vpxor %xmm15, %xmm0, %xmm0;
		leaq 16(%rcx), %rcx;

		/* AES rounds. */
		vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk1_enc_last;
		vaesenc (10 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
		jz .Locb_unaligned_blk1_enc_last;
		vaesenc (12 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;

		/* Last round and output handling. */
	.Locb_unaligned_blk1_enc_last:
		vpxor (14 * 16)(%rsp), %xmm15, %xmm1;
		vaesenclast %xmm1, %xmm0, %xmm0;
		vmovdqu %xmm0, (%rdx);
		leaq 16(%rdx), %rdx;

		jmp .Locb_unaligned_blk1;

	.align 8
	.Locb_unaligned_blk1_auth:
		vpxor (%rcx), %xmm15, %xmm0;
		leaq 16(%rcx), %rcx;

		/* AES rounds. */
		vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (10 * 16)(%rdi), %xmm1;
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk1_auth_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (12 * 16)(%rdi), %xmm1;
		jz .Locb_unaligned_blk1_auth_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (14 * 16)(%rdi), %xmm1;

		/* Last round and output handling. */
	.Locb_unaligned_blk1_auth_last:
		vaesenclast %xmm1, %xmm0, %xmm0;
		vpxor %ymm0, %ymm14, %ymm14;

		jmp .Locb_unaligned_blk1;

	.align 8
	.Locb_unaligned_blk1_dec:
		vpxor (%rcx), %xmm15, %xmm0;
		leaq 16(%rcx), %rcx;

		/* AES rounds. */
		vaesdec (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (9 * 16)(%rdi), %xmm0, %xmm0;
		cmpl $12, %r9d;
		jb .Locb_unaligned_blk1_dec_last;
		vaesdec (10 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (11 * 16)(%rdi), %xmm0, %xmm0;
		jz .Locb_unaligned_blk1_dec_last;
		vaesdec (12 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (13 * 16)(%rdi), %xmm0, %xmm0;

		/* Last round and output handling. */
	.Locb_unaligned_blk1_dec_last:
		vpxor (14 * 16)(%rsp), %xmm15, %xmm1;
		vaesdeclast %xmm1, %xmm0, %xmm0;
		vpxor %ymm0, %ymm14, %ymm14;
		vmovdqu %xmm0, (%rdx);
		leaq 16(%rdx), %rdx;

		jmp .Locb_unaligned_blk1;

.align 8
.Lunaligned_ocb_done:
	cmpq $1, %r8;
	jb .Ldone_ocb;

	/* Short buffers do not benefit from L-array optimization. */
	movq %r8, %r10;
	cmpq $16, %r8;
	jb .Locb_unaligned_blk8;

	vinserti128 $1, %xmm15, %ymm15, %ymm15;

	/* Prepare L-array optimization.
	 * Since nblk is aligned to 16, offsets will have following
	 * construction:
	 *  - block1 = ntz{0} = offset ^ L[0]
	 *  - block2 = ntz{1} = offset ^ L[0] ^ L[1]
	 *  - block3 = ntz{0} = offset ^ L[1]
	 *  - block4 = ntz{2} = offset ^ L[1] ^ L[2]
	 *  - block5 = ntz{0} = offset ^ L[0] ^ L[1] ^ L[2]
	 *  - block6 = ntz{1} = offset ^ L[0] ^ L[2]
	 *  - block7 = ntz{0} = offset ^ L[2]
	 *  - block8 = ntz{3} = offset ^ L[2] ^ L[3]
	 *  - block9 = ntz{0} = offset ^ L[0] ^ L[2] ^ L[3]
	 *  - block10 = ntz{1} = offset ^ L[0] ^ L[1] ^ L[2] ^ L[3]
	 *  - block11 = ntz{0} = offset ^ L[1] ^ L[2] ^ L[3]
	 *  - block12 = ntz{2} = offset ^ L[1] ^ L[3]
	 *  - block13 = ntz{0} = offset ^ L[0] ^ L[1] ^ L[3]
	 *  - block14 = ntz{1} = offset ^ L[0] ^ L[3]
	 *  - block15 = ntz{0} = offset ^ L[3]
	 *  - block16 = ntz{x} = offset ^ L[3] ^ L[ntz{x}]
	 */
	vmovdqu (0 * 16)(%r14), %xmm0;
	vmovdqu (1 * 16)(%r14), %xmm1;
	vmovdqu (2 * 16)(%r14), %xmm2;
	vmovdqu (3 * 16)(%r14), %xmm3;
	vpxor %ymm13, %ymm14, %ymm14;
	vmovdqa %ymm14, (20 * 16)(%rsp);
	vpxor %xmm0, %xmm1, %xmm4; /* L[0] ^ L[1] */
	vpxor %xmm0, %xmm2, %xmm5; /* L[0] ^ L[2] */
	vpxor %xmm0, %xmm3, %xmm6; /* L[0] ^ L[3] */
	vpxor %xmm1, %xmm2, %xmm7; /* L[1] ^ L[2] */
	vpxor %xmm1, %xmm3, %xmm8; /* L[1] ^ L[3] */
	vpxor %xmm2, %xmm3, %xmm9; /* L[2] ^ L[3] */
	vpxor %xmm4, %xmm2, %xmm10; /* L[0] ^ L[1] ^ L[2] */
	vpxor %xmm5, %xmm3, %xmm11; /* L[0] ^ L[2] ^ L[3] */
	vpxor %xmm7, %xmm3, %xmm12; /* L[1] ^ L[2] ^ L[3] */
	vpxor %xmm0, %xmm8, %xmm13; /* L[0] ^ L[1] ^ L[3] */
	vpxor %xmm4, %xmm9, %xmm14; /* L[0] ^ L[1] ^ L[2] ^ L[3] */
	vinserti128 $1, %xmm4, %ymm0, %ymm0;
	vinserti128 $1, %xmm7, %ymm1, %ymm1;
	vinserti128 $1, %xmm5, %ymm10, %ymm10;
	vinserti128 $1, %xmm9, %ymm2, %ymm2;
	vinserti128 $1, %xmm14, %ymm11, %ymm11;
	vinserti128 $1, %xmm8, %ymm12, %ymm12;
	vinserti128 $1, %xmm6, %ymm13, %ymm13;
	vmovdqa %ymm0,  (0 * 16)(%rsp);
	vmovdqa %ymm1,  (2 * 16)(%rsp);
	vmovdqa %ymm10, (4 * 16)(%rsp);
	vmovdqa %ymm2,  (6 * 16)(%rsp);
	vmovdqa %ymm11, (8 * 16)(%rsp);
	vmovdqa %ymm12, (10 * 16)(%rsp);
	vmovdqa %ymm13, (12 * 16)(%rsp);

	/* Aligned: Process 16 blocks per loop. */
.align 8
.Locb_aligned_blk16:
	cmpq $16, %r8;
	jb .Locb_aligned_blk8;

	leaq -16(%r8), %r8;

	leal 16(%esi), %esi;
	tzcntl %esi, %eax;
	shll $4, %eax;

	vpxor (0 * 16)(%rsp), %ymm15, %ymm8;
	vpxor (2 * 16)(%rsp), %ymm15, %ymm9;
	vpxor (4 * 16)(%rsp), %ymm15, %ymm10;
	vpxor (6 * 16)(%rsp), %ymm15, %ymm11;
	vpxor (8 * 16)(%rsp), %ymm15, %ymm12;

	vpxor (3 * 16)(%r14), %xmm15, %xmm13; /* offset ^ first key ^ L[3] */
	vpxor (%r14, %rax), %xmm13, %xmm14; /* offset ^ first key ^ L[3] ^ L[ntz{nblk+16}] */
	vinserti128 $1, %xmm14, %ymm13, %ymm14;

	cmpl $1, %r15d;
	jb .Locb_aligned_blk16_dec;
	ja .Locb_aligned_blk16_auth;
		vmovdqu (0 * 16)(%rcx), %ymm0;
		vmovdqu (2 * 16)(%rcx), %ymm1;
		vmovdqu (4 * 16)(%rcx), %ymm2;
		vmovdqu (6 * 16)(%rcx), %ymm3;
		vpxor (8 * 16)(%rcx), %ymm0, %ymm4;
		vpxor (10 * 16)(%rcx), %ymm1, %ymm5;
		vpxor (12 * 16)(%rcx), %ymm2, %ymm6;
		vpxor (14 * 16)(%rcx), %ymm3, %ymm7;
		vpxor %ymm4, %ymm5, %ymm5;
		vpxor %ymm6, %ymm7, %ymm7;
		vpxor %ymm5, %ymm7, %ymm7;
		vpxor (20 * 16)(%rsp), %ymm7, %ymm7;
		vmovdqa %ymm7, (20 * 16)(%rsp);

		vpxor (10 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (14 * 16)(%rcx), %ymm14, %ymm7;

		vpxor %ymm8, %ymm0, %ymm0;
		vpxor %ymm9, %ymm1, %ymm1;
		vpxor %ymm10, %ymm2, %ymm2;
		vpxor %ymm11, %ymm3, %ymm3;
		vpxor (8 * 16)(%rcx), %ymm12, %ymm4;
		vpxor (10 * 16)(%rcx), %ymm13, %ymm5;
		vmovdqa %ymm13, (16 * 16)(%rsp);
		vpxor (12 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (12 * 16)(%rcx), %ymm13, %ymm6;
		vmovdqa %ymm13, (18 * 16)(%rsp);

		leaq (16 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (2 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (3 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (4 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (5 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (6 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (7 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (8 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (9 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		cmpl $12, %r9d;
		jb .Locb_aligned_blk16_enc_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (11 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		jz .Locb_aligned_blk16_enc_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (13 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);

		/* Last round and output handling. */
	.Locb_aligned_blk16_enc_last:
		vmovdqa (14 * 16)(%rsp), %ymm13;
		vpxor %ymm8, %ymm13, %ymm8;
		vpxor %ymm9, %ymm13, %ymm9;
		vpxor %ymm10, %ymm13, %ymm10;
		vpxor %ymm11, %ymm13, %ymm11;
		vaesenclast %ymm8, %ymm0, %ymm0;
		vaesenclast %ymm9, %ymm1, %ymm1;
		vaesenclast %ymm10, %ymm2, %ymm2;
		vaesenclast %ymm11, %ymm3, %ymm3;
		vpxor %ymm12, %ymm13, %ymm12;
		vpxor (16 * 16)(%rsp), %ymm13, %ymm8;
		vpxor (18 * 16)(%rsp), %ymm13, %ymm9;
		vpxor %ymm14, %ymm13, %ymm13;
		vaesenclast %ymm12, %ymm4, %ymm4;
		vaesenclast %ymm8, %ymm5, %ymm5;
		vaesenclast %ymm9, %ymm6, %ymm6;
		vaesenclast %ymm13, %ymm7, %ymm7;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		vmovdqu %ymm4, (8 * 16)(%rdx);
		vmovdqu %ymm5, (10 * 16)(%rdx);
		vmovdqu %ymm6, (12 * 16)(%rdx);
		vmovdqu %ymm7, (14 * 16)(%rdx);
		leaq (16 * 16)(%rdx), %rdx;

		jmp .Locb_aligned_blk16;

	.align 8
	.Locb_aligned_blk16_auth:
		vpxor (10 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (14 * 16)(%rcx), %ymm14, %ymm7;

		vpxor (0 * 16)(%rcx), %ymm8, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm9, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm10, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm11, %ymm3;
		vpxor (8 * 16)(%rcx), %ymm12, %ymm4;
		vpxor (10 * 16)(%rcx), %ymm13, %ymm5;
		vmovdqa %ymm13, (16 * 16)(%rsp);
		vpxor (12 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (12 * 16)(%rcx), %ymm13, %ymm6;
		vmovdqa %ymm13, (18 * 16)(%rsp);

		leaq (16 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (2 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (3 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (4 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (5 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (6 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (7 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (8 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (9 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (10 * 16)(%rdi), %ymm13;
		cmpl $12, %r9d;
		jb .Locb_aligned_blk16_auth_last;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (11 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (12 * 16)(%rdi), %ymm13;
		jz .Locb_aligned_blk16_auth_last;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (13 * 16)(%rdi), %ymm13;
		VAESENC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (14 * 16)(%rdi), %ymm13;

		/* Last round and output handling. */
	.Locb_aligned_blk16_auth_last:
		vaesenclast %ymm13, %ymm0, %ymm0;
		vaesenclast %ymm13, %ymm1, %ymm1;
		vaesenclast %ymm13, %ymm2, %ymm2;
		vaesenclast %ymm13, %ymm3, %ymm3;
		vaesenclast %ymm13, %ymm4, %ymm4;
		vaesenclast %ymm13, %ymm5, %ymm5;
		vaesenclast %ymm13, %ymm6, %ymm6;
		vaesenclast %ymm13, %ymm7, %ymm7;

		vpxor %ymm1, %ymm0, %ymm0;
		vpxor %ymm3, %ymm2, %ymm2;
		vpxor %ymm5, %ymm4, %ymm4;
		vpxor %ymm7, %ymm6, %ymm6;
		vpxor %ymm2, %ymm0, %ymm0;
		vpxor %ymm6, %ymm4, %ymm4;
		vpxor %ymm4, %ymm0, %ymm0;
		vpxor (20 * 16)(%rsp), %ymm0, %ymm0;
		vmovdqa %ymm0, (20 * 16)(%rsp);

		jmp .Locb_aligned_blk16;

	.align 8
	.Locb_aligned_blk16_dec:
		vpxor (10 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (14 * 16)(%rcx), %ymm14, %ymm7;

		vpxor (0 * 16)(%rcx), %ymm8, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm9, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm10, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm11, %ymm3;
		vpxor (8 * 16)(%rcx), %ymm12, %ymm4;
		vpxor (10 * 16)(%rcx), %ymm13, %ymm5;
		vmovdqa %ymm13, (16 * 16)(%rsp);
		vpxor (12 * 16)(%rsp), %ymm15, %ymm13;
		vpxor (12 * 16)(%rcx), %ymm13, %ymm6;
		vmovdqa %ymm13, (18 * 16)(%rsp);

		leaq (16 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (2 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (3 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (4 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (5 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (6 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (7 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (8 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (9 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		cmpl $12, %r9d;
		jb .Locb_aligned_blk16_dec_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (11 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		jz .Locb_aligned_blk16_dec_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (13 * 16)(%rdi), %ymm13;
		VAESDEC8(%ymm13, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);

		/* Last round and output handling. */
	.Locb_aligned_blk16_dec_last:
		vmovdqa (14 * 16)(%rsp), %ymm13;
		vpxor %ymm8, %ymm13, %ymm8;
		vpxor %ymm9, %ymm13, %ymm9;
		vpxor %ymm10, %ymm13, %ymm10;
		vpxor %ymm11, %ymm13, %ymm11;
		vaesdeclast %ymm8, %ymm0, %ymm0;
		vaesdeclast %ymm9, %ymm1, %ymm1;
		vaesdeclast %ymm10, %ymm2, %ymm2;
		vaesdeclast %ymm11, %ymm3, %ymm3;
		vpxor %ymm12, %ymm13, %ymm12;
		vpxor (16 * 16)(%rsp), %ymm13, %ymm8;
		vpxor (18 * 16)(%rsp), %ymm13, %ymm9;
		vpxor %ymm14, %ymm13, %ymm13;
		vaesdeclast %ymm12, %ymm4, %ymm4;
		vaesdeclast %ymm8, %ymm5, %ymm5;
		vaesdeclast %ymm9, %ymm6, %ymm6;
		vaesdeclast %ymm13, %ymm7, %ymm7;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		vpxor %ymm1, %ymm0, %ymm0;
		vpxor %ymm3, %ymm2, %ymm2;
		vmovdqu %ymm4, (8 * 16)(%rdx);
		vmovdqu %ymm5, (10 * 16)(%rdx);
		vmovdqu %ymm6, (12 * 16)(%rdx);
		vmovdqu %ymm7, (14 * 16)(%rdx);
		vpxor %ymm5, %ymm4, %ymm4;
		vpxor %ymm7, %ymm6, %ymm6;
		leaq (16 * 16)(%rdx), %rdx;

		vpxor %ymm4, %ymm0, %ymm0;
		vpxor %ymm6, %ymm2, %ymm2;
		vpxor %ymm2, %ymm0, %ymm0;
		vpxor (20 * 16)(%rsp), %ymm0, %ymm0;
		vmovdqa %ymm0, (20 * 16)(%rsp);

		jmp .Locb_aligned_blk16;

	/* Aligned: Process trailing eight blocks. */
.align 8
.Locb_aligned_blk8:
	cmpq $8, %r8;
	jb .Locb_aligned_done;

	leaq -8(%r8), %r8;

	leal 8(%esi), %esi;

	vpxor (0 * 16)(%rsp), %ymm15, %ymm5;
	vpxor (2 * 16)(%rsp), %ymm15, %ymm6;
	vpxor (4 * 16)(%rsp), %ymm15, %ymm7;
	vpxor (6 * 16)(%rsp), %ymm15, %ymm14;

	cmpl $1, %r15d;
	jb .Locb_aligned_blk8_dec;
	ja .Locb_aligned_blk8_auth;
		vmovdqu (0 * 16)(%rcx), %ymm0;
		vmovdqu (2 * 16)(%rcx), %ymm1;
		vmovdqu (4 * 16)(%rcx), %ymm2;
		vmovdqu (6 * 16)(%rcx), %ymm3;
		vpxor %ymm2, %ymm0, %ymm10;
		vpxor %ymm3, %ymm1, %ymm11;
		vpxor %ymm11, %ymm10, %ymm10;
		vpxor (20 * 16)(%rsp), %ymm10, %ymm10;
		vmovdqa %ymm10, (20 * 16)(%rsp);

		vpxor %ymm5, %ymm0, %ymm0;
		vpxor %ymm6, %ymm1, %ymm1;
		vpxor %ymm7, %ymm2, %ymm2;
		vpxor %ymm14, %ymm3, %ymm3;
		leaq (8 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		vmovdqa (14 * 16)(%rsp), %ymm8;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		cmpl $12, %r9d;
		jb .Locb_aligned_blk8_enc_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		jz .Locb_aligned_blk8_enc_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);

		/* Last round and output handling. */
	.Locb_aligned_blk8_enc_last:
		vpxor %ymm5, %ymm8, %ymm5;
		vpxor %ymm6, %ymm8, %ymm6;
		vpxor %ymm7, %ymm8, %ymm7;
		vpxor %ymm14, %ymm8, %ymm4;
		vaesenclast %ymm5, %ymm0, %ymm0;
		vaesenclast %ymm6, %ymm1, %ymm1;
		vaesenclast %ymm7, %ymm2, %ymm2;
		vaesenclast %ymm4, %ymm3, %ymm3;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;

		jmp .Locb_aligned_done;

	.align 8
	.Locb_aligned_blk8_auth:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm7, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm14, %ymm3;
		leaq (8 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Locb_aligned_blk8_auth_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Locb_aligned_blk8_auth_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

		/* Last round and output handling. */
	.Locb_aligned_blk8_auth_last:
		vaesenclast %ymm4, %ymm0, %ymm0;
		vaesenclast %ymm4, %ymm1, %ymm1;
		vaesenclast %ymm4, %ymm2, %ymm2;
		vaesenclast %ymm4, %ymm3, %ymm3;

		vpxor %ymm1, %ymm0, %ymm0;
		vpxor %ymm3, %ymm2, %ymm2;
		vpxor %ymm2, %ymm0, %ymm0;
		vpxor (20 * 16)(%rsp), %ymm0, %ymm0;
		vmovdqa %ymm0, (20 * 16)(%rsp);

		jmp .Locb_aligned_done;

	.align 8
	.Locb_aligned_blk8_dec:
		vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
		vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
		vpxor (4 * 16)(%rcx), %ymm7, %ymm2;
		vpxor (6 * 16)(%rcx), %ymm14, %ymm3;
		leaq (8 * 16)(%rcx), %rcx;

		vperm2i128 $0x11, %ymm14, %ymm14, %ymm15;

		vmovdqa (14 * 16)(%rsp), %ymm8;

		/* AES rounds */
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		cmpl $12, %r9d;
		jb .Locb_aligned_blk8_dec_last;
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		jz .Locb_aligned_blk8_dec_last;
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

		/* Last round and output handling. */
	.Locb_aligned_blk8_dec_last:
		vpxor %ymm5, %ymm8, %ymm5;
		vpxor %ymm6, %ymm8, %ymm6;
		vpxor %ymm7, %ymm8, %ymm7;
		vpxor %ymm14, %ymm8, %ymm4;
		vaesdeclast %ymm5, %ymm0, %ymm0;
		vaesdeclast %ymm6, %ymm1, %ymm1;
		vaesdeclast %ymm7, %ymm2, %ymm2;
		vaesdeclast %ymm4, %ymm3, %ymm3;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;

		vpxor %ymm1, %ymm0, %ymm0;
		vpxor %ymm3, %ymm2, %ymm2;
		vpxor %ymm2, %ymm0, %ymm0;
		vpxor (20 * 16)(%rsp), %ymm0, %ymm0;
		vmovdqa %ymm0, (20 * 16)(%rsp);

.align 8
.Locb_aligned_done:
	vmovdqa (20 * 16)(%rsp), %ymm14;
	vpxor %xmm13, %xmm13, %xmm13;

	/* Burn stack. */
	vmovdqa %ymm13, (0 * 16)(%rsp);
	vmovdqa %ymm13, (2 * 16)(%rsp);
	vmovdqa %ymm13, (4 * 16)(%rsp);
	vmovdqa %ymm13, (6 * 16)(%rsp);
	vmovdqa %ymm13, (8 * 16)(%rsp);
	vmovdqa %ymm13, (10 * 16)(%rsp);
	vmovdqa %ymm13, (12 * 16)(%rsp);
	vmovdqa %ymm13, (16 * 16)(%rsp);
	vmovdqa %ymm13, (18 * 16)(%rsp);
	vmovdqa %ymm13, (20 * 16)(%rsp);

	/* Handle tailing 1…7 blocks in nblk-unaligned loop. */
	movq %r8, %r10;
	cmpq $1, %r8;
	jnb .Locb_unaligned_blk8;

.align 8
.Ldone_ocb:
	vpxor %ymm13, %ymm14, %ymm14;
	vextracti128 $1, %ymm14, %xmm13;
	vpxor (%rbx), %xmm14, %xmm14;
	vpxor %xmm13, %xmm14, %xmm14;
	vmovdqu %xmm14, (%rbx);

	movq OFFSET_PTR_Q, %r14; /* offset ptr. */
	vpxor (0 * 16)(%rdi), %xmm15, %xmm15; /* offset ^ first key ^ first key */
	vmovdqu %xmm15, (%r14); /* Store offset. */

	/* Burn stack. */
	vpxor %ymm0, %ymm0, %ymm0;
	vmovdqa %ymm0, (14 * 16)(%rsp);

	vzeroall;

	movq (STACK_REGS_POS + 0 * 8)(%rsp), %r12;
	CFI_RESTORE(%r12);
	movq (STACK_REGS_POS + 1 * 8)(%rsp), %r13;
	CFI_RESTORE(%r13);
	movq (STACK_REGS_POS + 2 * 8)(%rsp), %r14;
	CFI_RESTORE(%r14);
	movq (STACK_REGS_POS + 3 * 8)(%rsp), %r15;
	CFI_RESTORE(%r15);
	movq (STACK_REGS_POS + 4 * 8)(%rsp), %rbx;
	CFI_RESTORE(%rbx);

	xorl %eax, %eax;

	leave;
	CFI_LEAVE();
	ret_spec_stop

#undef STACK_REGS_POS
#undef STACK_ALLOC

	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_ocb_crypt_amd64,.-_gcry_vaes_avx2_ocb_crypt_amd64)

/**********************************************************************
  XTS-mode encryption
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_xts_crypt_amd64,@function)
.globl _gcry_vaes_avx2_xts_crypt_amd64
.align 16
_gcry_vaes_avx2_xts_crypt_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%rsi: tweak
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 *	8(%rsp): encrypt
	 */
	CFI_STARTPROC();

	movl 8(%rsp), %eax;

#define tweak_clmul(shift, out, tweak, hi_tweak, tmp1, tmp2) \
	vpsrld $(32-(shift)), hi_tweak, tmp2; \
	vpsllq $(shift), tweak, out; \
	vpclmulqdq $0, .Lxts_gfmul_clmul rRIP, tmp2, tmp1; \
	vpunpckhqdq tmp2, tmp1, tmp1; \
	vpxor tmp1, out, out;

	/* Prepare tweak. */
	vmovdqu (%rsi), %xmm15;
	vpshufb .Lxts_high_bit_shuf rRIP, %xmm15, %xmm13;
	tweak_clmul(1, %xmm11, %xmm15, %xmm13, %xmm0, %xmm1);
	vinserti128 $1, %xmm11, %ymm15, %ymm15; /* tweak:tweak1 */
	vpshufb .Lxts_high_bit_shuf rRIP, %ymm15, %ymm13;

	cmpq $8, %r8;
	jb .Lxts_crypt_blk4;

	/* Process eight blocks per loop. */
	leaq -8(%r8), %r8;

	vmovdqa %ymm15, %ymm5;
	tweak_clmul(2, %ymm6, %ymm15, %ymm13, %ymm0, %ymm1);
	tweak_clmul(4, %ymm7, %ymm15, %ymm13, %ymm0, %ymm1);
	tweak_clmul(6, %ymm8, %ymm15, %ymm13, %ymm0, %ymm1);
	tweak_clmul(8, %ymm15, %ymm15, %ymm13, %ymm0, %ymm1);
	vpshufb .Lxts_high_bit_shuf rRIP, %ymm15, %ymm13;

	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
	vpxor (2 * 16)(%rcx), %ymm6, %ymm1;
	vpxor (4 * 16)(%rcx), %ymm7, %ymm2;
	vpxor (6 * 16)(%rcx), %ymm8, %ymm3;

	leaq (8 * 16)(%rcx), %rcx;

.align 8
.Lxts_crypt_blk8_loop:
	cmpq $8, %r8;
	jb .Lxts_crypt_blk8_tail;
		leaq -8(%r8), %r8;

		testl %eax, %eax;
		jz .Lxts_dec_blk8;
			/* AES rounds */
			XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (1 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (2 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (3 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (4 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			  vmovdqa %ymm15, %ymm9;
			  tweak_clmul(2, %ymm10, %ymm15, %ymm13, %ymm12, %ymm14);
			  tweak_clmul(4, %ymm11, %ymm15, %ymm13, %ymm12, %ymm14);
			vbroadcasti128 (5 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (6 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (7 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (8 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (9 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (10 * 16)(%rdi), %ymm4;
			cmpl $12, %r9d;
			jb .Lxts_enc_blk8_last;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (11 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (12 * 16)(%rdi), %ymm4;
			jz .Lxts_enc_blk8_last;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (13 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (14 * 16)(%rdi), %ymm4;

			/* Last round and output handling. */
		.Lxts_enc_blk8_last:
			vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
			vpxor %ymm4, %ymm6, %ymm6;
			vpxor %ymm4, %ymm7, %ymm7;
			vpxor %ymm4, %ymm8, %ymm4;
			  tweak_clmul(6, %ymm8, %ymm15, %ymm13, %ymm12, %ymm14);
			  tweak_clmul(8, %ymm15, %ymm15, %ymm13, %ymm12, %ymm14);
			vpshufb .Lxts_high_bit_shuf rRIP, %ymm15, %ymm13;
			vaesenclast %ymm5, %ymm0, %ymm0;
			vaesenclast %ymm6, %ymm1, %ymm1;
			vaesenclast %ymm7, %ymm2, %ymm2;
			vaesenclast %ymm4, %ymm3, %ymm3;

			vmovdqu %ymm0, (0 * 16)(%rdx);
			vmovdqu %ymm1, (2 * 16)(%rdx);
			vmovdqu %ymm2, (4 * 16)(%rdx);
			vmovdqu %ymm3, (6 * 16)(%rdx);
			leaq (8 * 16)(%rdx), %rdx;

			vbroadcasti128 (0 * 16)(%rdi), %ymm4;
			vpxor (0 * 16)(%rcx), %ymm9, %ymm0;
			vpxor (2 * 16)(%rcx), %ymm10, %ymm1;
			vpxor (4 * 16)(%rcx), %ymm11, %ymm2;
			vpxor (6 * 16)(%rcx), %ymm8, %ymm3;

			  vmovdqa %ymm9, %ymm5;
			  vmovdqa %ymm10, %ymm6;
			  vmovdqa %ymm11, %ymm7;

			leaq (8 * 16)(%rcx), %rcx;

			jmp .Lxts_crypt_blk8_loop;

		.align 8
		.Lxts_dec_blk8:
			/* AES rounds */
			XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (1 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (2 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (3 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (4 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			  vmovdqa %ymm15, %ymm9;
			  tweak_clmul(2, %ymm10, %ymm15, %ymm13, %ymm12, %ymm14);
			  tweak_clmul(4, %ymm11, %ymm15, %ymm13, %ymm12, %ymm14);
			vbroadcasti128 (5 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (6 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (7 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (8 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (9 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (10 * 16)(%rdi), %ymm4;
			cmpl $12, %r9d;
			jb .Lxts_dec_blk8_last;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (11 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (12 * 16)(%rdi), %ymm4;
			jz .Lxts_dec_blk8_last;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (13 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (14 * 16)(%rdi), %ymm4;

			/* Last round and output handling. */
		.Lxts_dec_blk8_last:
			vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
			vpxor %ymm4, %ymm6, %ymm6;
			vpxor %ymm4, %ymm7, %ymm7;
			vpxor %ymm4, %ymm8, %ymm4;
			  tweak_clmul(6, %ymm8, %ymm15, %ymm13, %ymm12, %ymm14);
			  tweak_clmul(8, %ymm15, %ymm15, %ymm13, %ymm12, %ymm14);
			vpshufb .Lxts_high_bit_shuf rRIP, %ymm15, %ymm13;
			vaesdeclast %ymm5, %ymm0, %ymm0;
			vaesdeclast %ymm6, %ymm1, %ymm1;
			vaesdeclast %ymm7, %ymm2, %ymm2;
			vaesdeclast %ymm4, %ymm3, %ymm3;

			vmovdqu %ymm0, (0 * 16)(%rdx);
			vmovdqu %ymm1, (2 * 16)(%rdx);
			vmovdqu %ymm2, (4 * 16)(%rdx);
			vmovdqu %ymm3, (6 * 16)(%rdx);
			leaq (8 * 16)(%rdx), %rdx;

			vbroadcasti128 (0 * 16)(%rdi), %ymm4;
			vpxor (0 * 16)(%rcx), %ymm9, %ymm0;
			vpxor (2 * 16)(%rcx), %ymm10, %ymm1;
			vpxor (4 * 16)(%rcx), %ymm11, %ymm2;
			vpxor (6 * 16)(%rcx), %ymm8, %ymm3;

			  vmovdqa %ymm9, %ymm5;
			  vmovdqa %ymm10, %ymm6;
			  vmovdqa %ymm11, %ymm7;

			leaq (8 * 16)(%rcx), %rcx;

			jmp .Lxts_crypt_blk8_loop;

	.align 8
	.Lxts_crypt_blk8_tail:
		testl %eax, %eax;
		jz .Lxts_dec_tail_blk8;
			/* AES rounds */
			XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (1 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (2 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (3 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (4 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (5 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (6 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (7 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (8 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (9 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (10 * 16)(%rdi), %ymm4;
			cmpl $12, %r9d;
			jb .Lxts_enc_blk8_tail_last;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (11 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (12 * 16)(%rdi), %ymm4;
			jz .Lxts_enc_blk8_tail_last;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (13 * 16)(%rdi), %ymm4;
			VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (14 * 16)(%rdi), %ymm4;

			/* Last round and output handling. */
		.Lxts_enc_blk8_tail_last:
			vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
			vpxor %ymm4, %ymm6, %ymm6;
			vpxor %ymm4, %ymm7, %ymm7;
			vpxor %ymm4, %ymm8, %ymm4;
			vaesenclast %ymm5, %ymm0, %ymm0;
			vaesenclast %ymm6, %ymm1, %ymm1;
			vaesenclast %ymm7, %ymm2, %ymm2;
			vaesenclast %ymm4, %ymm3, %ymm3;
			vmovdqu %ymm0, (0 * 16)(%rdx);
			vmovdqu %ymm1, (2 * 16)(%rdx);
			vmovdqu %ymm2, (4 * 16)(%rdx);
			vmovdqu %ymm3, (6 * 16)(%rdx);
			leaq (8 * 16)(%rdx), %rdx;

			jmp .Lxts_crypt_blk4;

		.align 8
		.Lxts_dec_tail_blk8:
			/* AES rounds */
			XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (1 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (2 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (3 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (4 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (5 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (6 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (7 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (8 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (9 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (10 * 16)(%rdi), %ymm4;
			cmpl $12, %r9d;
			jb .Lxts_dec_blk8_tail_last;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (11 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (12 * 16)(%rdi), %ymm4;
			jz .Lxts_dec_blk8_tail_last;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (13 * 16)(%rdi), %ymm4;
			VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
			vbroadcasti128 (14 * 16)(%rdi), %ymm4;

			/* Last round and output handling. */
		.Lxts_dec_blk8_tail_last:
			vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
			vpxor %ymm4, %ymm6, %ymm6;
			vpxor %ymm4, %ymm7, %ymm7;
			vpxor %ymm4, %ymm8, %ymm4;
			vaesdeclast %ymm5, %ymm0, %ymm0;
			vaesdeclast %ymm6, %ymm1, %ymm1;
			vaesdeclast %ymm7, %ymm2, %ymm2;
			vaesdeclast %ymm4, %ymm3, %ymm3;
			vmovdqu %ymm0, (0 * 16)(%rdx);
			vmovdqu %ymm1, (2 * 16)(%rdx);
			vmovdqu %ymm2, (4 * 16)(%rdx);
			vmovdqu %ymm3, (6 * 16)(%rdx);
			leaq (8 * 16)(%rdx), %rdx;

	/* Handle trailing four blocks. */
.align 8
.Lxts_crypt_blk4:
	/* Try exit early as typically input length is large power of 2. */
	cmpq $1, %r8;
	jb .Ldone_xts_crypt;
	cmpq $4, %r8;
	jb .Lxts_crypt_blk1;

	leaq -4(%r8), %r8;

	vmovdqa %ymm15, %ymm5;
	tweak_clmul(2, %ymm6, %ymm15, %ymm13, %ymm0, %ymm1);
	tweak_clmul(4, %ymm15, %ymm15, %ymm13, %ymm0, %ymm1);
	vpshufb .Lxts_high_bit_shuf rRIP, %ymm15, %ymm13;

	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vpxor (0 * 16)(%rcx), %ymm5, %ymm0;
	vpxor (2 * 16)(%rcx), %ymm6, %ymm1;

	leaq (4 * 16)(%rcx), %rcx;

	testl %eax, %eax;
	jz .Lxts_dec_blk4;
		/* AES rounds */
		XOR2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lxts_enc_blk4_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lxts_enc_blk4_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

		/* Last round and output handling. */
	.Lxts_enc_blk4_last:
		vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
		vpxor %ymm4, %ymm6, %ymm6;
		vaesenclast %ymm5, %ymm0, %ymm0;
		vaesenclast %ymm6, %ymm1, %ymm1;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;

		jmp .Lxts_crypt_blk1;

	.align 8
	.Lxts_dec_blk4:
		/* AES rounds */
		XOR2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (1 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lxts_dec_blk4_last;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lxts_dec_blk4_last;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;

		/* Last round and output handling. */
	.Lxts_dec_blk4_last:
		vpxor %ymm4, %ymm5, %ymm5; /* Xor tweak to last round key. */
		vpxor %ymm4, %ymm6, %ymm6;
		vaesdeclast %ymm5, %ymm0, %ymm0;
		vaesdeclast %ymm6, %ymm1, %ymm1;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lxts_crypt_blk1:
	cmpq $1, %r8;
	jb .Ldone_xts_crypt;

	leaq -1(%r8), %r8;

	vpxor (%rcx), %xmm15, %xmm0;
	vmovdqa %xmm15, %xmm5;
	tweak_clmul(1, %xmm15, %xmm15, %xmm13, %xmm2, %xmm3);
	vpshufb .Lxts_high_bit_shuf rRIP, %xmm15, %xmm13;

	leaq 16(%rcx), %rcx;

	testl %eax, %eax;
	jz .Lxts_dec_blk1;
		/* AES rounds. */
		vpxor (0 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (10 * 16)(%rdi), %xmm1;
		cmpl $12, %r9d;
		jb .Lxts_enc_blk1_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (12 * 16)(%rdi), %xmm1;
		jz .Lxts_enc_blk1_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (14 * 16)(%rdi), %xmm1;

		/* Last round and output handling. */
	.Lxts_enc_blk1_last:
		vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */
		vaesenclast %xmm5, %xmm0, %xmm0;
		vmovdqu %xmm0, (%rdx);
		leaq 16(%rdx), %rdx;

		jmp .Lxts_crypt_blk1;

	.align 8
	.Lxts_dec_blk1:
		/* AES rounds. */
		vpxor (0 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (9 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (10 * 16)(%rdi), %xmm1;
		cmpl $12, %r9d;
		jb .Lxts_dec_blk1_last;
		vaesdec %xmm1, %xmm0, %xmm0;
		vaesdec (11 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (12 * 16)(%rdi), %xmm1;
		jz .Lxts_dec_blk1_last;
		vaesdec %xmm1, %xmm0, %xmm0;
		vaesdec (13 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (14 * 16)(%rdi), %xmm1;

		/* Last round and output handling. */
	.Lxts_dec_blk1_last:
		vpxor %xmm1, %xmm5, %xmm5; /* Xor tweak to last round key. */
		vaesdeclast %xmm5, %xmm0, %xmm0;
		vmovdqu %xmm0, (%rdx);
		leaq 16(%rdx), %rdx;

		jmp .Lxts_crypt_blk1;

.align 8
.Ldone_xts_crypt:
	/* Store IV. */
	vmovdqu %xmm15, (%rsi);

	vzeroall;

	xorl %eax, %eax
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_xts_crypt_amd64,.-_gcry_vaes_avx2_xts_crypt_amd64)

/**********************************************************************
  ECB-mode encryption
 **********************************************************************/
ELF(.type _gcry_vaes_avx2_ecb_crypt_amd64,@function)
.globl _gcry_vaes_avx2_ecb_crypt_amd64
.align 16
_gcry_vaes_avx2_ecb_crypt_amd64:
	/* input:
	 *	%rdi: round keys
	 *	%esi: encrypt
	 *	%rdx: dst
	 *	%rcx: src
	 *	%r8:  nblocks
	 *	%r9:  nrounds
	 */
	CFI_STARTPROC();

	/* Process 16 blocks per loop. */
.align 8
.Lecb_blk16:
	cmpq $16, %r8;
	jb .Lecb_blk8;

	leaq -16(%r8), %r8;

	/* Load input and xor first key. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm8;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vmovdqu (4 * 16)(%rcx), %ymm2;
	vmovdqu (6 * 16)(%rcx), %ymm3;
	vmovdqu (8 * 16)(%rcx), %ymm4;
	vmovdqu (10 * 16)(%rcx), %ymm5;
	vmovdqu (12 * 16)(%rcx), %ymm6;
	vmovdqu (14 * 16)(%rcx), %ymm7;
	vpxor %ymm8, %ymm0, %ymm0;
	vpxor %ymm8, %ymm1, %ymm1;
	vpxor %ymm8, %ymm2, %ymm2;
	vpxor %ymm8, %ymm3, %ymm3;
	vpxor %ymm8, %ymm4, %ymm4;
	vpxor %ymm8, %ymm5, %ymm5;
	vpxor %ymm8, %ymm6, %ymm6;
	vpxor %ymm8, %ymm7, %ymm7;
	vbroadcasti128 (1 * 16)(%rdi), %ymm8;
	leaq (16 * 16)(%rcx), %rcx;

	testl %esi, %esi;
	jz .Lecb_dec_blk16;
		/* AES rounds */
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (2 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (3 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (4 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (5 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (6 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (7 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (8 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (9 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (10 * 16)(%rdi), %ymm8;
		cmpl $12, %r9d;
		jb .Lecb_enc_blk16_last;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (11 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (12 * 16)(%rdi), %ymm8;
		jz .Lecb_enc_blk16_last;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (13 * 16)(%rdi), %ymm8;
		VAESENC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (14 * 16)(%rdi), %ymm8;
	  .Lecb_enc_blk16_last:
		vaesenclast %ymm8, %ymm0, %ymm0;
		vaesenclast %ymm8, %ymm1, %ymm1;
		vaesenclast %ymm8, %ymm2, %ymm2;
		vaesenclast %ymm8, %ymm3, %ymm3;
		vaesenclast %ymm8, %ymm4, %ymm4;
		vaesenclast %ymm8, %ymm5, %ymm5;
		vaesenclast %ymm8, %ymm6, %ymm6;
		vaesenclast %ymm8, %ymm7, %ymm7;
		jmp .Lecb_blk16_end;

	  .align 8
	  .Lecb_dec_blk16:
		/* AES rounds */
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (2 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (3 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (4 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (5 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (6 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (7 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (8 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (9 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (10 * 16)(%rdi), %ymm8;
		cmpl $12, %r9d;
		jb .Lecb_dec_blk16_last;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (11 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (12 * 16)(%rdi), %ymm8;
		jz .Lecb_dec_blk16_last;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (13 * 16)(%rdi), %ymm8;
		VAESDEC8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
		vbroadcasti128 (14 * 16)(%rdi), %ymm8;
	  .Lecb_dec_blk16_last:
		vaesdeclast %ymm8, %ymm0, %ymm0;
		vaesdeclast %ymm8, %ymm1, %ymm1;
		vaesdeclast %ymm8, %ymm2, %ymm2;
		vaesdeclast %ymm8, %ymm3, %ymm3;
		vaesdeclast %ymm8, %ymm4, %ymm4;
		vaesdeclast %ymm8, %ymm5, %ymm5;
		vaesdeclast %ymm8, %ymm6, %ymm6;
		vaesdeclast %ymm8, %ymm7, %ymm7;
		jmp .Lecb_blk16_end;

  .align 8
  .Lecb_blk16_end:
	vmovdqu %ymm0, (0 * 16)(%rdx);
	vmovdqu %ymm1, (2 * 16)(%rdx);
	vmovdqu %ymm2, (4 * 16)(%rdx);
	vmovdqu %ymm3, (6 * 16)(%rdx);
	vmovdqu %ymm4, (8 * 16)(%rdx);
	vmovdqu %ymm5, (10 * 16)(%rdx);
	vmovdqu %ymm6, (12 * 16)(%rdx);
	vmovdqu %ymm7, (14 * 16)(%rdx);
	leaq (16 * 16)(%rdx), %rdx;

	jmp .Lecb_blk16;

	/* Handle trailing eight blocks. */
.align 8
.Lecb_blk8:
	cmpq $8, %r8;
	jmp .Lecb_blk4;

	leaq -8(%r8), %r8;

	/* Load input and xor first key. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vmovdqu (4 * 16)(%rcx), %ymm2;
	vmovdqu (6 * 16)(%rcx), %ymm3;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vpxor %ymm4, %ymm2, %ymm2;
	vpxor %ymm4, %ymm3, %ymm3;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	leaq (8 * 16)(%rcx), %rcx;

	testl %esi, %esi;
	jz .Lecb_dec_blk8;
		/* AES rounds */
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lecb_enc_blk8_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lecb_enc_blk8_last;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;
	  .Lecb_enc_blk8_last:
		vaesenclast %ymm4, %ymm0, %ymm0;
		vaesenclast %ymm4, %ymm1, %ymm1;
		vaesenclast %ymm4, %ymm2, %ymm2;
		vaesenclast %ymm4, %ymm3, %ymm3;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;
		jmp .Lecb_blk4;

	  .align 8
	  .Lecb_dec_blk8:
		/* AES rounds */
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lecb_dec_blk8_last;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lecb_dec_blk8_last;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;
	  .Lecb_dec_blk8_last:
		vaesdeclast %ymm4, %ymm0, %ymm0;
		vaesdeclast %ymm4, %ymm1, %ymm1;
		vaesdeclast %ymm4, %ymm2, %ymm2;
		vaesdeclast %ymm4, %ymm3, %ymm3;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		vmovdqu %ymm2, (4 * 16)(%rdx);
		vmovdqu %ymm3, (6 * 16)(%rdx);
		leaq (8 * 16)(%rdx), %rdx;

	/* Handle trailing four blocks. */
.align 8
.Lecb_blk4:
	cmpq $4, %r8;
	jb .Lecb_blk1;

	leaq -4(%r8), %r8;

	/* Load input and xor first key. */
	vbroadcasti128 (0 * 16)(%rdi), %ymm4;
	vmovdqu (0 * 16)(%rcx), %ymm0;
	vmovdqu (2 * 16)(%rcx), %ymm1;
	vpxor %ymm4, %ymm0, %ymm0;
	vpxor %ymm4, %ymm1, %ymm1;
	vbroadcasti128 (1 * 16)(%rdi), %ymm4;
	leaq (4 * 16)(%rcx), %rcx;

	testl %esi, %esi;
	jz .Lecb_dec_blk4;
		/* AES rounds */
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lecb_enc_blk4_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lecb_enc_blk4_last;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESENC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;
	  .Lecb_enc_blk4_last:
		vaesenclast %ymm4, %ymm0, %ymm0;
		vaesenclast %ymm4, %ymm1, %ymm1;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;
		jmp .Lecb_blk1;

	  .align 8
	  .Lecb_dec_blk4:
		/* AES rounds */
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (2 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (3 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (4 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (5 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (6 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (7 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (8 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (9 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (10 * 16)(%rdi), %ymm4;
		cmpl $12, %r9d;
		jb .Lecb_dec_blk4_last;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (11 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (12 * 16)(%rdi), %ymm4;
		jz .Lecb_dec_blk4_last;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (13 * 16)(%rdi), %ymm4;
		VAESDEC2(%ymm4, %ymm0, %ymm1);
		vbroadcasti128 (14 * 16)(%rdi), %ymm4;
	  .Lecb_dec_blk4_last:
		vaesdeclast %ymm4, %ymm0, %ymm0;
		vaesdeclast %ymm4, %ymm1, %ymm1;
		vmovdqu %ymm0, (0 * 16)(%rdx);
		vmovdqu %ymm1, (2 * 16)(%rdx);
		leaq (4 * 16)(%rdx), %rdx;

	/* Process trailing one to three blocks, one per loop. */
.align 8
.Lecb_blk1:
	cmpq $1, %r8;
	jb .Ldone_ecb;

	leaq -1(%r8), %r8;

	/* Load input. */
	vmovdqu (%rcx), %xmm2;
	leaq 16(%rcx), %rcx;

	/* Xor first key. */
	vpxor (0 * 16)(%rdi), %xmm2, %xmm0;

	testl %esi, %esi;
	jz .Lecb_dec_blk1;
		/* AES rounds. */
		vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesenc (9 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (10 * 16)(%rdi), %xmm1;
		cmpl $12, %r9d;
		jb .Lecb_enc_blk1_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (11 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (12 * 16)(%rdi), %xmm1;
		jz .Lecb_enc_blk1_last;
		vaesenc %xmm1, %xmm0, %xmm0;
		vaesenc (13 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (14 * 16)(%rdi), %xmm1;
	  .Lecb_enc_blk1_last:
		vaesenclast %xmm1, %xmm0, %xmm0;
		jmp .Lecb_blk1_end;

	  .align 8
	  .Lecb_dec_blk1:
		/* AES rounds. */
		vaesdec (1 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (2 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (3 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (4 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (5 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (6 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (7 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (8 * 16)(%rdi), %xmm0, %xmm0;
		vaesdec (9 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (10 * 16)(%rdi), %xmm1;
		cmpl $12, %r9d;
		jb .Lecb_dec_blk1_last;
		vaesdec %xmm1, %xmm0, %xmm0;
		vaesdec (11 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (12 * 16)(%rdi), %xmm1;
		jz .Lecb_dec_blk1_last;
		vaesdec %xmm1, %xmm0, %xmm0;
		vaesdec (13 * 16)(%rdi), %xmm0, %xmm0;
		vmovdqa (14 * 16)(%rdi), %xmm1;
	  .Lecb_dec_blk1_last:
		vaesdeclast %xmm1, %xmm0, %xmm0;
		jmp .Lecb_blk1_end;

  .align 8
  .Lecb_blk1_end:
	vmovdqu %xmm0, (%rdx);
	leaq 16(%rdx), %rdx;

	jmp .Lecb_blk1;

.align 8
.Ldone_ecb:
	vzeroall;
	ret_spec_stop
	CFI_ENDPROC();
ELF(.size _gcry_vaes_avx2_ecb_crypt_amd64,.-_gcry_vaes_avx2_ecb_crypt_amd64)

/**********************************************************************
  constants
 **********************************************************************/
SECTION_RODATA

ELF(.type _gcry_vaes_consts,@object)
_gcry_vaes_consts:
.align 32
.Lbige_addb_0:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lbige_addb_1:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
.Lbige_addb_2:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
.Lbige_addb_3:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
.Lbige_addb_4:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
.Lbige_addb_5:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
.Lbige_addb_6:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
.Lbige_addb_7:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
.Lbige_addb_8:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
.Lbige_addb_9:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
.Lbige_addb_10:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
.Lbige_addb_11:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
.Lbige_addb_12:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
.Lbige_addb_13:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
.Lbige_addb_14:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
.Lbige_addb_15:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15

.Lle_addd_0:
	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_1:
	.byte 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_2:
	.byte 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_3:
	.byte 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_4:
	.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_5:
	.byte 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_6:
	.byte 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_7:
	.byte 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_8:
	.byte 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_9:
	.byte 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_10:
	.byte 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_11:
	.byte 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_12:
	.byte 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_13:
	.byte 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_14:
	.byte 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_15:
	.byte 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0

.Lle_addd_4_2:
	.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	.byte 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_8_2:
	.byte 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	.byte 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
.Lle_addd_16_2:
	.byte 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	.byte 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0

.Lxts_gfmul_clmul:
	.long 0x00, 0x87, 0x00, 0x00
	.long 0x00, 0x87, 0x00, 0x00
.Lxts_high_bit_shuf:
	.byte -1, -1, -1, -1, 12, 13, 14, 15
	.byte 4, 5, 6, 7, -1, -1, -1, -1
	.byte -1, -1, -1, -1, 12, 13, 14, 15
	.byte 4, 5, 6, 7, -1, -1, -1, -1
.Lbswap128_mask:
	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
ELF(.size _gcry_vaes_consts,.-_gcry_vaes_consts)

#endif /* HAVE_GCC_INLINE_ASM_VAES */
#endif /* __x86_64__ */
