/* camellia-simd128.h - Camellia cipher SIMD128 intrinsics implementation
 * Copyright (C) 2023,2025 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This file is part of Libgcrypt.
 *
 * Libgcrypt is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * Libgcrypt is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

/*
 * SSE/AVX/NEON implementation of Camellia cipher, using AES-NI/ARMv8-CE/
 * PPC-crypto for sbox calculations. This implementation takes 16 input blocks
 * and process them in parallel. Vectorized key setup is also available at
 * the end of file. This implementation is from
 *  - https://github.com/jkivilin/camellia-simd-aesni
 *
 * This work was originally presented in Master's Thesis,
 *   "Block Ciphers: Fast Implementations on x86-64 Architecture" (pages 42-50)
 *   http://urn.fi/URN:NBN:fi:oulu-201305311409
 */

#include <config.h>
#include "types.h"


#define ALWAYS_INLINE inline __attribute__((always_inline))
#define NO_INLINE __attribute__((noinline))
#define NO_INSTRUMENT_FUNCTION __attribute__((no_instrument_function))

#define ASM_FUNC_ATTR          NO_INSTRUMENT_FUNCTION
#define ASM_FUNC_ATTR_INLINE   ASM_FUNC_ATTR ALWAYS_INLINE
#define ASM_FUNC_ATTR_NOINLINE ASM_FUNC_ATTR NO_INLINE SIMD128_OPT_ATTR


#if defined(HAVE_GCC_INLINE_ASM_PPC_ALTIVEC) && !defined(WORDS_BIGENDIAN)

/**********************************************************************
  AT&T x86 asm to intrinsics conversion macros (PowerPC VSX+crypto)
 **********************************************************************/
#include "simd-common-ppc.h"
#include <altivec.h>

typedef vector signed char int8x16_t;
typedef vector unsigned char uint8x16_t;
typedef vector unsigned short uint16x8_t;
typedef vector unsigned int uint32x4_t;
typedef vector unsigned long long uint64x2_t;
typedef uint64x2_t __m128i;

#ifdef __clang__
/* clang has mismatching prototype for vec_sbox_be. */
static ASM_FUNC_ATTR_INLINE uint8x16_t
asm_sbox_be(uint8x16_t b)
{
  uint8x16_t o;
  __asm__ ("vsbox %0, %1\n\t" : "=v" (o) : "v" (b));
  return o;
}
#undef vec_sbox_be
#define vec_sbox_be asm_sbox_be
#endif

#define vec_bswap(a)            ((__m128i)vec_reve((uint8x16_t)a))

#define vpand128(a, b, o)       (o = vec_and(b, a))
#define vpandn128(a, b, o)      (o = vec_andc(a, b))
#define vpxor128(a, b, o)       (o = vec_xor(b, a))
#define vpor128(a, b, o)        (o = vec_or(b, a))

#define vpsrlb128(s, a, o)      ({ o = (__m128i)((uint8x16_t)a >> s); })
#define vpsllb128(s, a, o)      ({ o = (__m128i)((uint8x16_t)a << s); })
#define vpsrlw128(s, a, o)      ({ o = (__m128i)((uint16x8_t)a >> s); })
#define vpsllw128(s, a, o)      ({ o = (__m128i)((uint16x8_t)a << s); })
#define vpsrld128(s, a, o)      ({ o = (__m128i)((uint32x4_t)a >> s); })
#define vpslld128(s, a, o)      ({ o = (__m128i)((uint32x4_t)a << s); })
#define vpsrlq128(s, a, o)      ({ o = (__m128i)((uint64x2_t)a >> s); })
#define vpsllq128(s, a, o)      ({ o = (__m128i)((uint64x2_t)a << s); })
#define vpsrldq128(s, a, o)     ({ uint64x2_t __tmp = { 0, 0 }; \
				  o = (__m128i)vec_sld((uint8x16_t)__tmp, \
						       (uint8x16_t)a, (16 - (s)) & 15);})
#define vpslldq128(s, a, o)     ({ uint64x2_t __tmp = { 0, 0 }; \
				  o = (__m128i)vec_sld((uint8x16_t)a, \
						       (uint8x16_t)__tmp, (s) & 15);})

#define if_vpsrlb128(...)       __VA_ARGS__
#define if_not_vpsrlb128(...)   /*_*/
#define vpsrl_byte_128(s, a, o) vpsrlb128(s, a, o)
#define vpsll_byte_128(s, a, o) vpsllb128(s, a, o)

#define if_vprolb128(...)       __VA_ARGS__
#define if_not_vprolb128(...)   /*_*/
#define vprolb128(s, a, o, tmp) ({ vpsllb128((s), a, tmp); \
				   vpsrlb128((8-(s)), a, o); \
				   vpxor128(tmp, o, o); })

#define vpaddb128(a, b, o)      (o = (__m128i)vec_add((uint8x16_t)b, (uint8x16_t)a))

#define vpcmpgtb128(a, b, o)    (o = (__m128i)vec_cmpgt((int8x16_t)b, (int8x16_t)a))
#define vpabsb128(a, o)         (o = (__m128i)vec_abs((int8x16_t)a))

#define vpshufd128_0x4e(a, o)   (o = (__m128i)vec_reve((uint64x2_t)a))
#define vpshufd128_0x1b(a, o)   (o = (__m128i)vec_reve((uint32x4_t)a))

#define vpshufb128(m, a, o) \
	({ uint64x2_t __tmpz = { 0, 0 }; \
	   o = (__m128i)vec_perm((uint8x16_t)a, (uint8x16_t)__tmpz, (uint8x16_t)m); })

#define vpunpckhdq128(a, b, o)  (o = (__m128i)vec_mergel((uint32x4_t)b, (uint32x4_t)a))
#define vpunpckldq128(a, b, o)  (o = (__m128i)vec_mergeh((uint32x4_t)b, (uint32x4_t)a))
#define vpunpckhqdq128(a, b, o) (o = (__m128i)vec_mergel((uint64x2_t)b, (uint64x2_t)a))
#define vpunpcklqdq128(a, b, o) (o = (__m128i)vec_mergeh((uint64x2_t)b, (uint64x2_t)a))

#define vmovdqa128(a, o)        (o = a)
#define vmovd128(a, o)          ({ uint32x4_t __tmp = { (a), 0, 0, 0 }; \
				   o = (__m128i)(__tmp); })
#define vmovq128(a, o)          ({ uint64x2_t __tmp = { (a), 0 }; \
				   o = (__m128i)(__tmp); })

#define vmovd128_amemld(z, a, o) ({ \
				   const uint32_t *__tmp_ptr = (const void *)(a); \
				   uint32x4_t __tmp = { __tmp_ptr[z], 0, 0, 0 }; \
				   o = (__m128i)(__tmp); })
#define vmovq128_amemld(a, o)   ({ uint64x2_t __tmp = { *(const uint64_t *)(a), 0 }; \
				   o = (__m128i)(__tmp); })

#define vmovdqa128_memld(a, o)  (o = *(const __m128i *)(a))
#define vmovdqa128_memst(a, o)  (*(__m128i *)(o) = (a))
#define vpshufb128_amemld(m, a, o) vpshufb128(*(const __m128i *)(m), a, o)

/* Following operations may have unaligned memory input */
#define vmovdqu128_memld(a, o)  (o = (__m128i)vec_xl(0, (const uint8_t *)(a)))
#define vpxor128_memld(a, b, o) vpxor128(b, (__m128i)vec_xl(0, (const uint8_t *)(a)), o)
#define vmovq128_memld(a, o)    ({ uint64x2_t __tmp = { *(const uint64_unaligned_t *)(a), 0 }; \
				   o = (__m128i)(__tmp); })

/* Following operations may have unaligned memory output */
#define vmovdqu128_memst(a, o)  vec_xst((uint8x16_t)(a), 0, (uint8_t *)(o))
#define vmovq128_memst(a, o)    (((uint64_unaligned_t *)(o))[0] = ((__m128i)(a))[0])

/* PowerPC AES encrypt last round => ShiftRows + SubBytes + XOR round key  */
static const uint8x16_t shift_row __attribute__((unused)) =
  { 0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, 1, 6, 11 };
#define vaesenclast128(a, b, o) \
	({ uint64x2_t __tmp = (__m128i)vec_sbox_be((uint8x16_t)(b)); \
	   vpshufb128(shift_row, __tmp, __tmp); \
	   vpxor128(a, __tmp, o); })

/* Macros for exposing SubBytes from PowerPC crypto instructions. */
#define aes_subbytes(a, o) \
	(o = (__m128i)vec_sbox_be((uint8x16_t)(a)))
#define aes_subbytes_and_shuf_and_xor(zero, a, o) \
        vaesenclast128((zero), (a), (o))
/*#define aes_load_inv_shufmask(shufmask_reg) \
	load_frequent_const(inv_shift_row, (shufmask_reg))*/
#define aes_inv_shuf(shufmask_reg, a, o) \
	vpshufb128(shufmask_reg, (a), (o))
#define if_aes_subbytes(...) __VA_ARGS__
#define if_not_aes_subbytes(...) /*_*/

#define memory_barrier_with_vec(a) __asm__("" : "+wa"(a) :: "memory")

#endif /* __powerpc__ */

#ifdef __ARM_NEON

/**********************************************************************
  AT&T x86 asm to intrinsics conversion macros (ARMv8-CE)
 **********************************************************************/
#include "simd-common-aarch64.h"
#include <arm_neon.h>

#define __m128i uint64x2_t

#define vpand128(a, b, o)       (o = vandq_u64(b, a))
#define vpandn128(a, b, o)      (o = vbicq_u64(a, b))
#define vpxor128(a, b, o)       (o = veorq_u64(b, a))
#define vpor128(a, b, o)        (o = vorrq_u64(b, a))

#define vpsrlb128(s, a, o)      (o = (__m128i)vshrq_n_u8((uint8x16_t)a, s))
#define vpsllb128(s, a, o)      (o = (__m128i)vshlq_n_u8((uint8x16_t)a, s))
#define vpsrlw128(s, a, o)      (o = (__m128i)vshrq_n_u16((uint16x8_t)a, s))
#define vpsllw128(s, a, o)      (o = (__m128i)vshlq_n_u16((uint16x8_t)a, s))
#define vpsrld128(s, a, o)      (o = (__m128i)vshrq_n_u32((uint32x4_t)a, s))
#define vpslld128(s, a, o)      (o = (__m128i)vshlq_n_u32((uint32x4_t)a, s))
#define vpsrlq128(s, a, o)      (o = (__m128i)vshrq_n_u64(a, s))
#define vpsllq128(s, a, o)      (o = (__m128i)vshlq_n_u64(a, s))
#define vpsrldq128(s, a, o)     ({ uint64x2_t __tmp = { 0, 0 }; \
				o = (__m128i)vextq_u8((uint8x16_t)a, \
						      (uint8x16_t)__tmp, (s) & 15);})
#define vpslldq128(s, a, o)     ({ uint64x2_t __tmp = { 0, 0 }; \
				o = (__m128i)vextq_u8((uint8x16_t)__tmp, \
						      (uint8x16_t)a, (16 - (s)) & 15);})

#define if_vpsrlb128(...)       __VA_ARGS__
#define if_not_vpsrlb128(...)   /*_*/
#define vpsrl_byte_128(s, a, o) vpsrlb128(s, a, o)
#define vpsll_byte_128(s, a, o) vpsllb128(s, a, o)

#define if_vprolb128(...)       __VA_ARGS__
#define if_not_vprolb128(...)   /*_*/
#define vprolb128(s, a, o, t)   ({ t = (__m128i)vshlq_n_u8((uint8x16_t)a, s); \
				   o = (__m128i)vsriq_n_u8((uint8x16_t)t, \
							   (uint8x16_t)a, \
							   8-(s)); })

#define vpaddb128(a, b, o)      (o = (__m128i)vaddq_u8((uint8x16_t)b, (uint8x16_t)a))

#define vpcmpgtb128(a, b, o)    (o = (__m128i)vcgtq_s8((int8x16_t)b, (int8x16_t)a))
#define vpabsb128(a, o)         (o = (__m128i)vabsq_s8((int8x16_t)a))

#define vpshufd128_0x4e(a, o)   (o = (__m128i)vextq_u8((uint8x16_t)a, (uint8x16_t)a, 8))
#define vpshufd128_0x1b(a, o)   (o = (__m128i)vrev64q_u32((uint32x4_t)vextq_u8((uint8x16_t)a, (uint8x16_t)a, 8)))
#define vpshufb128(m, a, o)     (o = (__m128i)vqtbl1q_u8((uint8x16_t)a, (uint8x16_t)m))

#define vpunpckhdq128(a, b, o)  (o = (__m128i)vzip2q_u32((uint32x4_t)b, (uint32x4_t)a))
#define vpunpckldq128(a, b, o)  (o = (__m128i)vzip1q_u32((uint32x4_t)b, (uint32x4_t)a))
#define vpunpckhqdq128(a, b, o) (o = (__m128i)vzip2q_u64(b, a))
#define vpunpcklqdq128(a, b, o) (o = (__m128i)vzip1q_u64(b, a))

/* CE AES encrypt last round => ShiftRows + SubBytes + XOR round key  */
#define vaesenclast128(a, b, o) (o = (__m128i)vaeseq_u8((uint8x16_t)b, (uint8x16_t)a))

#define vmovdqa128(a, o)        (o = a)
#define vmovd128(a, o)          ({ uint32x4_t __tmp = { a, 0, 0, 0 }; o = (__m128i)__tmp; })
#define vmovq128(a, o)          ({ uint64x2_t __tmp = { a, 0 }; o = (__m128i)__tmp; })

#define vmovd128_amemld(z, a, o) ({ \
				   const uint32_t *__tmp_ptr = (const void *)(a); \
				   uint32x4_t __tmp = { __tmp_ptr[z], 0, 0, 0 }; \
				   o = (__m128i)(__tmp); })
#define vmovq128_amemld(a, o)   ({ uint64x1_t __tmp = vld1_u64((const uint64_t *)(a)); \
				   o = (__m128i)vcombine_u64(__tmp, vcreate_u64(0)); })

#define vmovdqa128_memld(a, o)  (o = (*(const __m128i *)(a)))
#define vmovdqa128_memst(a, o)  (*(__m128i *)(o) = (a))
#define vpshufb128_amemld(m, a, o) vpshufb128(*(const __m128i *)(m), a, o)

/* Following operations may have unaligned memory input */
#define vmovdqu128_memld(a, o)  (o = (__m128i)vld1q_u8((const uint8_t *)(a)))
#define vpxor128_memld(a, b, o) vpxor128(b, (__m128i)vld1q_u8((const uint8_t *)(a)), o)
#define vmovq128_memld(a, o)    ({ uint8x8_t __tmp = vld1_u8((const uint8_t *)(a)); \
				   o = (__m128i)vcombine_u8(__tmp, vcreate_u8(0)); })

/* Following operations may have unaligned memory output */
#define vmovdqu128_memst(a, o)  vst1q_u8((uint8_t *)(o), (uint8x16_t)a)
#define vmovq128_memst(a, o)    (((uint64_unaligned_t *)(o))[0] = (a)[0])

/* Macros for exposing SubBytes from Crypto-Extension instruction set. */
#define aes_subbytes_and_shuf_and_xor(zero, a, o) \
        vaesenclast128(zero, a, o)
#define aes_load_inv_shufmask(shufmask_reg) \
	load_frequent_const(inv_shift_row, shufmask_reg)
#define aes_inv_shuf(shufmask_reg, a, o) \
	vpshufb128(shufmask_reg, a, o)
#define if_aes_subbytes(...) /*_*/
#define if_not_aes_subbytes(...) __VA_ARGS__

#define memory_barrier_with_vec(a) __asm__("" : "+w"(a) :: "memory")

#endif /* __ARM_NEON */

#if defined(__x86_64__) || defined(__i386__)

/**********************************************************************
  AT&T x86 asm to intrinsics conversion macros
 **********************************************************************/
#include <x86intrin.h>

#define vpand128(a, b, o)       (o = _mm_and_si128(b, a))
#define vpandn128(a, b, o)      (o = _mm_andnot_si128(b, a))
#define vpxor128(a, b, o)       (o = _mm_xor_si128(b, a))
#define vpor128(a, b, o)        (o = _mm_or_si128(b, a))

#define vpsrlw128(s, a, o)      (o = _mm_srli_epi16(a, s))
#define vpsllw128(s, a, o)      (o = _mm_slli_epi16(a, s))
#define vpsrld128(s, a, o)      (o = _mm_srli_epi32(a, s))
#define vpslld128(s, a, o)      (o = _mm_slli_epi32(a, s))
#define vpsrlq128(s, a, o)      (o = _mm_srli_epi64(a, s))
#define vpsllq128(s, a, o)      (o = _mm_slli_epi64(a, s))
#define vpsrldq128(s, a, o)     (o = _mm_srli_si128(a, s))
#define vpslldq128(s, a, o)     (o = _mm_slli_si128(a, s))

#define if_vpsrlb128(...)       /*_*/
#define if_not_vpsrlb128(...)   __VA_ARGS__
#define vpsrl_byte_128(s, a, o) vpsrld128(s, a, o)
#define vpsll_byte_128(s, a, o) vpslld128(s, a, o)

#define if_vprolb128(...)       /*_*/
#define if_not_vprolb128(...)   __VA_ARGS__

#define vpaddb128(a, b, o)      (o = _mm_add_epi8(b, a))

#define vpcmpgtb128(a, b, o)    (o = _mm_cmpgt_epi8(b, a))
#define vpabsb128(a, o)         (o = _mm_abs_epi8(a))

#define vpshufd128_0x1b(a, o)   (o = _mm_shuffle_epi32(a, 0x1b))
#define vpshufd128_0x4e(a, o)   (o = _mm_shuffle_epi32(a, 0x4e))
#define vpshufb128(m, a, o)     (o = _mm_shuffle_epi8(a, m))

#define vpunpckhdq128(a, b, o)  (o = _mm_unpackhi_epi32(b, a))
#define vpunpckldq128(a, b, o)  (o = _mm_unpacklo_epi32(b, a))
#define vpunpckhqdq128(a, b, o) (o = _mm_unpackhi_epi64(b, a))
#define vpunpcklqdq128(a, b, o) (o = _mm_unpacklo_epi64(b, a))

/* AES-NI encrypt last round => ShiftRows + SubBytes + XOR round key  */
#define vaesenclast128(a, b, o) (o = _mm_aesenclast_si128(b, a))

#define vmovdqa128(a, o)        (o = a)
#define vmovd128(a, o)          (o = _mm_set_epi32(0, 0, 0, a))
#define vmovq128(a, o)          (o = _mm_set_epi64x(0, a))

#define vmovd128_amemld(z, a, o) ({ \
				   const uint32_t *__tmp_ptr = (const void *)(a); \
				   o = (__m128i)_mm_loadu_si32(__tmp_ptr + (z)); })
#define vmovq128_amemld(a, o)   (o = (__m128i)_mm_loadu_si64(a))

#define vmovdqa128_memld(a, o)  (o = (*(const __m128i *)(a)))
#define vmovdqa128_memst(a, o)  (*(__m128i *)(o) = (a))
#define vpshufb128_amemld(m, a, o) vpshufb128(*(const __m128i *)(m), a, o)

/* Following operations may have unaligned memory input */
#define vmovdqu128_memld(a, o)  (o = _mm_loadu_si128((const __m128i *)(a)))
#define vpxor128_memld(a, b, o) \
	vpxor128(b, _mm_loadu_si128((const __m128i *)(a)), o)
#define vmovq128_memld(a, o)    vmovq128_amemld(a, o)

/* Following operations may have unaligned memory output */
#define vmovdqu128_memst(a, o)  _mm_storeu_si128((__m128i *)(o), a)
#define vmovq128_memst(a, o)    _mm_storel_epi64((__m128i *)(o), a)

/* Macros for exposing SubBytes from AES-NI instruction set. */
#define aes_subbytes_and_shuf_and_xor(zero, a, o) \
	vaesenclast128(zero, a, o)
#define aes_load_inv_shufmask(shufmask_reg) \
	load_frequent_const(inv_shift_row, shufmask_reg)
#define aes_inv_shuf(shufmask_reg, a, o) \
	vpshufb128(shufmask_reg, a, o)
#define if_aes_subbytes(...) /*_*/
#define if_not_aes_subbytes(...) __VA_ARGS__

#define memory_barrier_with_vec(a) __asm__("" : "+x"(a) :: "memory")

#endif /* defined(__x86_64__) || defined(__i386__) */

/**********************************************************************
  helper macros
 **********************************************************************/
#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
	vpand128(x, mask4bit, tmp0); \
	if_vpsrlb128(vpsrlb128(4, x, x)); \
	if_not_vpsrlb128(vpandn128(x, mask4bit, x)); \
	if_not_vpsrlb128(vpsrld128(4, x, x)); \
	\
	vpshufb128(tmp0, lo_t, tmp0); \
	vpshufb128(x, hi_t, x); \
	vpxor128(tmp0, x, x);

#define filter_8bit_3op(out, in, lo_t, hi_t, mask4bit, tmp0) \
	vmovdqa128(in, out); \
	filter_8bit(out, lo_t, hi_t, mask4bit, tmp0);

#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
	vpunpckhdq128(x1, x0, t2); \
	vpunpckldq128(x1, x0, x0); \
	\
	vpunpckldq128(x3, x2, t1); \
	vpunpckhdq128(x3, x2, x2); \
	\
	vpunpckhqdq128(t1, x0, x1); \
	vpunpcklqdq128(t1, x0, x0); \
	\
	vpunpckhqdq128(x2, t2, x3); \
	vpunpcklqdq128(x2, t2, x2);

#define load_zero(o) vmovq128(0, o)

#define load_frequent_const(constant, o) vmovdqa128(constant ## _stack, o)

#define prepare_frequent_const(constant) \
	vmovdqa128_memld(&(constant), constant ## _stack); \
	memory_barrier_with_vec(constant ## _stack)

#define prepare_frequent_constants() \
	prepare_frequent_const(inv_shift_row); \
	prepare_frequent_const(pack_bswap); \
	prepare_frequent_const(shufb_16x16b); \
	prepare_frequent_const(mask_0f); \
	prepare_frequent_const(pre_tf_lo_s1); \
	prepare_frequent_const(pre_tf_hi_s1); \
	prepare_frequent_const(pre_tf_lo_s4); \
	prepare_frequent_const(pre_tf_hi_s4); \
	prepare_frequent_const(post_tf_lo_s1); \
	prepare_frequent_const(post_tf_hi_s1); \
	prepare_frequent_const(post_tf_lo_s3); \
	prepare_frequent_const(post_tf_hi_s3); \
	prepare_frequent_const(post_tf_lo_s2); \
	prepare_frequent_const(post_tf_hi_s2)

#define frequent_constants_declare \
	__m128i inv_shift_row_stack; \
	__m128i pack_bswap_stack; \
	__m128i shufb_16x16b_stack; \
	__m128i mask_0f_stack; \
	__m128i pre_tf_lo_s1_stack; \
	__m128i pre_tf_hi_s1_stack; \
	__m128i pre_tf_lo_s4_stack; \
	__m128i pre_tf_hi_s4_stack; \
	__m128i post_tf_lo_s1_stack; \
	__m128i post_tf_hi_s1_stack; \
	__m128i post_tf_lo_s3_stack; \
	__m128i post_tf_hi_s3_stack; \
	__m128i post_tf_lo_s2_stack; \
	__m128i post_tf_hi_s2_stack

/**********************************************************************
  16-way camellia macros
 **********************************************************************/

/*
 * IN:
 *   x0..x7: byte-sliced AB state
 *   mem_cd: register pointer storing CD state
 *   key: index for key material
 * OUT:
 *   x0..x7: new byte-sliced CD state
 */
#define roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \
		  t7, mem_cd, key) \
	/* \
	 * S-function with AES subbytes \
	 */ \
	if_not_aes_subbytes(aes_load_inv_shufmask(t4);) \
	load_frequent_const(mask_0f, t7); \
	load_frequent_const(pre_tf_lo_s1, t0); \
	load_frequent_const(pre_tf_hi_s1, t1); \
	\
	/* AES inverse shift rows */ \
	if_not_aes_subbytes( \
	  aes_inv_shuf(t4, x0, x0); \
	  aes_inv_shuf(t4, x7, x7); \
	  aes_inv_shuf(t4, x1, x1); \
	  aes_inv_shuf(t4, x4, x4); \
	  aes_inv_shuf(t4, x2, x2); \
	  aes_inv_shuf(t4, x5, x5); \
	  aes_inv_shuf(t4, x3, x3); \
	  aes_inv_shuf(t4, x6, x6); \
	) \
	\
	/* prefilter sboxes 1, 2 and 3 */ \
	load_frequent_const(pre_tf_lo_s4, t2); \
	load_frequent_const(pre_tf_hi_s4, t3); \
	filter_8bit(x0, t0, t1, t7, t6); \
	filter_8bit(x7, t0, t1, t7, t6); \
	filter_8bit(x1, t0, t1, t7, t6); \
	filter_8bit(x4, t0, t1, t7, t6); \
	filter_8bit(x2, t0, t1, t7, t6); \
	filter_8bit(x5, t0, t1, t7, t6); \
	\
	/* prefilter sbox 4 */ \
	if_not_aes_subbytes(load_zero(t4);) \
	filter_8bit(x3, t2, t3, t7, t6); \
	filter_8bit(x6, t2, t3, t7, t6); \
	\
	/* AES subbytes + AES shift rows */ \
	load_frequent_const(post_tf_lo_s1, t0); \
	load_frequent_const(post_tf_hi_s1, t1); \
	if_not_aes_subbytes( \
	  aes_subbytes_and_shuf_and_xor(t4, x0, x0); \
	  aes_subbytes_and_shuf_and_xor(t4, x7, x7); \
	  aes_subbytes_and_shuf_and_xor(t4, x1, x1); \
	  aes_subbytes_and_shuf_and_xor(t4, x4, x4); \
	  aes_subbytes_and_shuf_and_xor(t4, x2, x2); \
	  aes_subbytes_and_shuf_and_xor(t4, x5, x5); \
	  aes_subbytes_and_shuf_and_xor(t4, x3, x3); \
	  aes_subbytes_and_shuf_and_xor(t4, x6, x6); \
	) \
	if_aes_subbytes( \
	  aes_subbytes(x0, x0); \
	  aes_subbytes(x7, x7); \
	  aes_subbytes(x1, x1); \
	  aes_subbytes(x4, x4); \
	  aes_subbytes(x2, x2); \
	  aes_subbytes(x5, x5); \
	  aes_subbytes(x3, x3); \
	  aes_subbytes(x6, x6); \
	) \
	\
	/* postfilter sboxes 1 and 4 */ \
	load_frequent_const(post_tf_lo_s3, t2); \
	load_frequent_const(post_tf_hi_s3, t3); \
	filter_8bit(x0, t0, t1, t7, t6); \
	filter_8bit(x7, t0, t1, t7, t6); \
	filter_8bit(x3, t0, t1, t7, t6); \
	filter_8bit(x6, t0, t1, t7, t6); \
	\
	/* postfilter sbox 3 */ \
	load_frequent_const(post_tf_lo_s2, t4); \
	load_frequent_const(post_tf_hi_s2, t5); \
	filter_8bit(x2, t2, t3, t7, t6); \
	filter_8bit(x5, t2, t3, t7, t6); \
	\
	vmovq128_amemld(&(key), t0); \
	\
	/* postfilter sbox 2 */ \
	filter_8bit(x1, t4, t5, t7, t2); \
	filter_8bit(x4, t4, t5, t7, t2); \
	\
	/* P-function */ \
	vpxor128(x5, x0, x0); \
	vpxor128(x6, x1, x1); \
	vpxor128(x7, x2, x2); \
	vpxor128(x4, x3, x3); \
	\
	vpxor128(x2, x4, x4); \
	vpxor128(x3, x5, x5); \
	vpxor128(x0, x6, x6); \
	vpxor128(x1, x7, x7); \
	\
	vpxor128(x7, x0, x0); \
	vpxor128(x4, x1, x1); \
	vpxor128(x5, x2, x2); \
	vpxor128(x6, x3, x3); \
	\
	vpxor128(x3, x4, x4); \
	vpxor128(x0, x5, x5); \
	vpxor128(x1, x6, x6); \
	vpxor128(x2, x7, x7); /* note: high and low parts swapped */ \
	\
	/* Add key material and result to CD (x becomes new CD) */ \
	\
	vpshufb128(bcast[7], t0, t7); \
	vpshufb128(bcast[6], t0, t6); \
	vpshufb128(bcast[5], t0, t5); \
	vpshufb128(bcast[4], t0, t4); \
	vpshufb128(bcast[3], t0, t3); \
	vpshufb128(bcast[2], t0, t2); \
	vpshufb128(bcast[1], t0, t1); \
	\
	vpxor128(t3, x4, x4); \
	vpxor128(mem_cd[0], x4, x4); \
	\
	load_zero(t3); \
	vpshufb128(t3, t0, t0); \
	\
	vpxor128(t2, x5, x5); \
	vpxor128(mem_cd[1], x5, x5); \
	\
	vpxor128(t1, x6, x6); \
	vpxor128(mem_cd[2], x6, x6); \
	\
	vpxor128(t0, x7, x7); \
	vpxor128(mem_cd[3], x7, x7); \
	\
	vpxor128(t7, x0, x0); \
	vpxor128(mem_cd[4], x0, x0); \
	\
	vpxor128(t6, x1, x1); \
	vpxor128(mem_cd[5], x1, x1); \
	\
	vpxor128(t5, x2, x2); \
	vpxor128(mem_cd[6], x2, x2); \
	\
	vpxor128(t4, x3, x3); \
	vpxor128(mem_cd[7], x3, x3);

/*
 * IN/OUT:
 *  x0..x7: byte-sliced AB state preloaded
 *  mem_ab: byte-sliced AB state in memory
 *  mem_cb: byte-sliced CD state in memory
 */
#define two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, i, dir, store_ab) \
	roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		  y6, y7, mem_cd, ctx->key_table[(i)]); \
	\
	vmovdqa128(x4, mem_cd[0]); \
	vmovdqa128(x5, mem_cd[1]); \
	vmovdqa128(x6, mem_cd[2]); \
	vmovdqa128(x7, mem_cd[3]); \
	vmovdqa128(x0, mem_cd[4]); \
	vmovdqa128(x1, mem_cd[5]); \
	vmovdqa128(x2, mem_cd[6]); \
	vmovdqa128(x3, mem_cd[7]); \
	\
	roundsm16(x4, x5, x6, x7, x0, x1, x2, x3, y0, y1, y2, y3, y4, y5, \
		  y6, y7, mem_ab, ctx->key_table[(i) + (dir)]); \
	\
	store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab);

#define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */

#define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \
	/* Store new AB state */ \
	vmovdqa128(x0, mem_ab[0]); \
	vmovdqa128(x1, mem_ab[1]); \
	vmovdqa128(x2, mem_ab[2]); \
	vmovdqa128(x3, mem_ab[3]); \
	vmovdqa128(x4, mem_ab[4]); \
	vmovdqa128(x5, mem_ab[5]); \
	vmovdqa128(x6, mem_ab[6]); \
	vmovdqa128(x7, mem_ab[7]);

#define enc_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, i) \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store);

#define dec_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, i) \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \
	two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store);

/*
 * IN:
 *  v0..3: byte-sliced 32-bit integers
 * OUT:
 *  v0..3: (IN <<< 1)
 */
#define rol32_1_16(v0, v1, v2, v3, t0, t1, t2, zero) \
	if_vpsrlb128(vpsrlb128(7, v0, t0)); \
	if_not_vpsrlb128(vpcmpgtb128(v0, zero, t0)); \
	vpaddb128(v0, v0, v0); \
	if_not_vpsrlb128(vpabsb128(t0, t0)); \
	\
	if_vpsrlb128(vpsrlb128(7, v1, t1)); \
	if_not_vpsrlb128(vpcmpgtb128(v1, zero, t1)); \
	vpaddb128(v1, v1, v1); \
	if_not_vpsrlb128(vpabsb128(t1, t1)); \
	\
	if_vpsrlb128(vpsrlb128(7, v2, t2)); \
	if_not_vpsrlb128(vpcmpgtb128(v2, zero, t2)); \
	vpaddb128(v2, v2, v2); \
	if_not_vpsrlb128(vpabsb128(t2, t2)); \
	\
	vpor128(t0, v1, v1); \
	\
	if_vpsrlb128(vpsrlb128(7, v3, t0)); \
	if_not_vpsrlb128(vpcmpgtb128(v3, zero, t0)); \
	vpaddb128(v3, v3, v3); \
	if_not_vpsrlb128(vpabsb128(t0, t0)); \
	\
	vpor128(t1, v2, v2); \
	vpor128(t2, v3, v3); \
	vpor128(t0, v0, v0);

/*
 * IN:
 *   r: byte-sliced AB state in memory
 *   l: byte-sliced CD state in memory
 * OUT:
 *   x0..x7: new byte-sliced CD state
 */
#define fls16(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \
	      tt1, tt2, tt3, kl, kr) \
	/* \
	 * t0 = kll; \
	 * t0 &= ll; \
	 * lr ^= rol32(t0, 1); \
	 */ \
	load_zero(tt0); \
	vmovd128_amemld(0, kl, t0); \
	vpshufb128(tt0, t0, t3); \
	vpshufb128(bcast[1], t0, t2); \
	vpshufb128(bcast[2], t0, t1); \
	vpshufb128(bcast[3], t0, t0); \
	\
	vpand128(l0, t0, t0); \
	vpand128(l1, t1, t1); \
	vpand128(l2, t2, t2); \
	vpand128(l3, t3, t3); \
	\
	rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
	\
	vpxor128(l4, t0, l4); \
	vmovdqa128(l4, l[4]); \
	vpxor128(l5, t1, l5); \
	vmovdqa128(l5, l[5]); \
	vpxor128(l6, t2, l6); \
	vmovdqa128(l6, l[6]); \
	vpxor128(l7, t3, l7); \
	vmovdqa128(l7, l[7]); \
	\
	/* \
	 * t2 = krr; \
	 * t2 |= rr; \
	 * rl ^= t2; \
	 */ \
	\
	vmovd128_amemld(1, kr, t0); \
	vpshufb128(tt0, t0, t3); \
	vpshufb128(bcast[1], t0, t2); \
	vpshufb128(bcast[2], t0, t1); \
	vpshufb128(bcast[3], t0, t0); \
	\
	vpor128(r[4], t0, t0); \
	vpor128(r[5], t1, t1); \
	vpor128(r[6], t2, t2); \
	vpor128(r[7], t3, t3); \
	\
	vpxor128(r[0], t0, t0); \
	vpxor128(r[1], t1, t1); \
	vpxor128(r[2], t2, t2); \
	vpxor128(r[3], t3, t3); \
	vmovdqa128(t0, r[0]); \
	vmovdqa128(t1, r[1]); \
	vmovdqa128(t2, r[2]); \
	vmovdqa128(t3, r[3]); \
	\
	/* \
	 * t2 = krl; \
	 * t2 &= rl; \
	 * rr ^= rol32(t2, 1); \
	 */ \
	vmovd128_amemld(0, kr, t0); \
	vpshufb128(tt0, t0, t3); \
	vpshufb128(bcast[1], t0, t2); \
	vpshufb128(bcast[2], t0, t1); \
	vpshufb128(bcast[3], t0, t0); \
	\
	vpand128(r[0], t0, t0); \
	vpand128(r[1], t1, t1); \
	vpand128(r[2], t2, t2); \
	vpand128(r[3], t3, t3); \
	\
	rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
	\
	vpxor128(r[4], t0, t0); \
	vpxor128(r[5], t1, t1); \
	vpxor128(r[6], t2, t2); \
	vpxor128(r[7], t3, t3); \
	vmovdqa128(t0, r[4]); \
	vmovdqa128(t1, r[5]); \
	vmovdqa128(t2, r[6]); \
	vmovdqa128(t3, r[7]); \
	\
	/* \
	 * t0 = klr; \
	 * t0 |= lr; \
	 * ll ^= t0; \
	 */ \
	\
	vmovd128_amemld(1, kl, t0); \
	vpshufb128(tt0, t0, t3); \
	vpshufb128(bcast[1], t0, t2); \
	vpshufb128(bcast[2], t0, t1); \
	vpshufb128(bcast[3], t0, t0); \
	\
	vpor128(l4, t0, t0); \
	vpor128(l5, t1, t1); \
	vpor128(l6, t2, t2); \
	vpor128(l7, t3, t3); \
	\
	vpxor128(l0, t0, l0); \
	vmovdqa128(l0, l[0]); \
	vpxor128(l1, t1, l1); \
	vmovdqa128(l1, l[1]); \
	vpxor128(l2, t2, l2); \
	vmovdqa128(l2, l[2]); \
	vpxor128(l3, t3, l3); \
	vmovdqa128(l3, l[3]);

#define byteslice_16x16b_fast(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, \
			      a3, b3, c3, d3, st0, st1) \
	vmovdqa128(d2, st0); \
	vmovdqa128(d3, st1); \
	transpose_4x4(a0, a1, a2, a3, d2, d3); \
	transpose_4x4(b0, b1, b2, b3, d2, d3); \
	vmovdqa128(st0, d2); \
	vmovdqa128(st1, d3); \
	\
	vmovdqa128(a0, st0); \
	vmovdqa128(a1, st1); \
	transpose_4x4(c0, c1, c2, c3, a0, a1); \
	transpose_4x4(d0, d1, d2, d3, a0, a1); \
	\
	vmovdqa128(shufb_16x16b_stack, a0); \
	vmovdqa128(st1, a1); \
	vpshufb128(a0, a2, a2); \
	vpshufb128(a0, a3, a3); \
	vpshufb128(a0, b0, b0); \
	vpshufb128(a0, b1, b1); \
	vpshufb128(a0, b2, b2); \
	vpshufb128(a0, b3, b3); \
	vpshufb128(a0, a1, a1); \
	vpshufb128(a0, c0, c0); \
	vpshufb128(a0, c1, c1); \
	vpshufb128(a0, c2, c2); \
	vpshufb128(a0, c3, c3); \
	vpshufb128(a0, d0, d0); \
	vpshufb128(a0, d1, d1); \
	vpshufb128(a0, d2, d2); \
	vpshufb128(a0, d3, d3); \
	vmovdqa128(d3, st1); \
	vmovdqa128(st0, d3); \
	vpshufb128(a0, d3, a0); \
	vmovdqa128(d2, st0); \
	\
	transpose_4x4(a0, b0, c0, d0, d2, d3); \
	transpose_4x4(a1, b1, c1, d1, d2, d3); \
	vmovdqa128(st0, d2); \
	vmovdqa128(st1, d3); \
	\
	vmovdqa128(b0, st0); \
	vmovdqa128(b1, st1); \
	transpose_4x4(a2, b2, c2, d2, b0, b1); \
	transpose_4x4(a3, b3, c3, d3, b0, b1); \
	vmovdqa128(st0, b0); \
	vmovdqa128(st1, b1); \
	/* does not adjust output bytes inside vectors */

/* load blocks to registers and apply pre-whitening */
#define inpack16_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		     y6, y7, rio, key) \
	vmovq128_amemld(&(key), x0); \
	vpshufb128(pack_bswap_stack, x0, x0); \
	\
	vpxor128_memld((rio) + 0 * 16, x0, y7); \
	vpxor128_memld((rio) + 1 * 16, x0, y6); \
	vpxor128_memld((rio) + 2 * 16, x0, y5); \
	vpxor128_memld((rio) + 3 * 16, x0, y4); \
	vpxor128_memld((rio) + 4 * 16, x0, y3); \
	vpxor128_memld((rio) + 5 * 16, x0, y2); \
	vpxor128_memld((rio) + 6 * 16, x0, y1); \
	vpxor128_memld((rio) + 7 * 16, x0, y0); \
	vpxor128_memld((rio) + 8 * 16, x0, x7); \
	vpxor128_memld((rio) + 9 * 16, x0, x6); \
	vpxor128_memld((rio) + 10 * 16, x0, x5); \
	vpxor128_memld((rio) + 11 * 16, x0, x4); \
	vpxor128_memld((rio) + 12 * 16, x0, x3); \
	vpxor128_memld((rio) + 13 * 16, x0, x2); \
	vpxor128_memld((rio) + 14 * 16, x0, x1); \
	vpxor128_memld((rio) + 15 * 16, x0, x0);

/* byteslice pre-whitened blocks and store to temporary memory */
#define inpack16_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		      y6, y7, mem_ab, mem_cd) \
	byteslice_16x16b_fast(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, \
			      y4, y5, y6, y7, mem_ab[0], mem_cd[0]); \
	\
	vmovdqa128(x0, mem_ab[0]); \
	vmovdqa128(x1, mem_ab[1]); \
	vmovdqa128(x2, mem_ab[2]); \
	vmovdqa128(x3, mem_ab[3]); \
	vmovdqa128(x4, mem_ab[4]); \
	vmovdqa128(x5, mem_ab[5]); \
	vmovdqa128(x6, mem_ab[6]); \
	vmovdqa128(x7, mem_ab[7]); \
	vmovdqa128(y0, mem_cd[0]); \
	vmovdqa128(y1, mem_cd[1]); \
	vmovdqa128(y2, mem_cd[2]); \
	vmovdqa128(y3, mem_cd[3]); \
	vmovdqa128(y4, mem_cd[4]); \
	vmovdqa128(y5, mem_cd[5]); \
	vmovdqa128(y6, mem_cd[6]); \
	vmovdqa128(y7, mem_cd[7]);

/* de-byteslice, apply post-whitening and store blocks */
#define outunpack16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \
		    y5, y6, y7, key, stack_tmp0, stack_tmp1) \
	byteslice_16x16b_fast(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, \
			      y3, y7, x3, x7, stack_tmp0, stack_tmp1); \
	\
	vmovdqa128(x0, stack_tmp0); \
	\
	vmovq128_amemld(&(key), x0); \
	vpshufb128(pack_bswap_stack, x0, x0); \
	\
	vpxor128(x0, y7, y7); \
	vpxor128(x0, y6, y6); \
	vpxor128(x0, y5, y5); \
	vpxor128(x0, y4, y4); \
	vpxor128(x0, y3, y3); \
	vpxor128(x0, y2, y2); \
	vpxor128(x0, y1, y1); \
	vpxor128(x0, y0, y0); \
	vpxor128(x0, x7, x7); \
	vpxor128(x0, x6, x6); \
	vpxor128(x0, x5, x5); \
	vpxor128(x0, x4, x4); \
	vpxor128(x0, x3, x3); \
	vpxor128(x0, x2, x2); \
	vpxor128(x0, x1, x1); \
	vpxor128(stack_tmp0, x0, x0);

#define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
		     y6, y7, rio) \
	vmovdqu128_memst(x0, (rio) + 0 * 16); \
	vmovdqu128_memst(x1, (rio) + 1 * 16); \
	vmovdqu128_memst(x2, (rio) + 2 * 16); \
	vmovdqu128_memst(x3, (rio) + 3 * 16); \
	vmovdqu128_memst(x4, (rio) + 4 * 16); \
	vmovdqu128_memst(x5, (rio) + 5 * 16); \
	vmovdqu128_memst(x6, (rio) + 6 * 16); \
	vmovdqu128_memst(x7, (rio) + 7 * 16); \
	vmovdqu128_memst(y0, (rio) + 8 * 16); \
	vmovdqu128_memst(y1, (rio) + 9 * 16); \
	vmovdqu128_memst(y2, (rio) + 10 * 16); \
	vmovdqu128_memst(y3, (rio) + 11 * 16); \
	vmovdqu128_memst(y4, (rio) + 12 * 16); \
	vmovdqu128_memst(y5, (rio) + 13 * 16); \
	vmovdqu128_memst(y6, (rio) + 14 * 16); \
	vmovdqu128_memst(y7, (rio) + 15 * 16);

/**********************************************************************
  macros for defining constant vectors
 **********************************************************************/
#define SWAP_LE64(x) (x)

#define M128I_BYTE(a0, a1, a2, a3, a4, a5, a6, a7, b0, b1, b2, b3, b4, b5, b6, b7) \
	{ \
	  SWAP_LE64((((a0) & 0xffULL) << 0) | \
		    (((a1) & 0xffULL) << 8) | \
		    (((a2) & 0xffULL) << 16) | \
		    (((a3) & 0xffULL) << 24) | \
		    (((a4) & 0xffULL) << 32) | \
		    (((a5) & 0xffULL) << 40) | \
		    (((a6) & 0xffULL) << 48) | \
		    (((a7) & 0xffULL) << 56)), \
	  SWAP_LE64((((b0) & 0xffULL) << 0) | \
		    (((b1) & 0xffULL) << 8) | \
		    (((b2) & 0xffULL) << 16) | \
		    (((b3) & 0xffULL) << 24) | \
		    (((b4) & 0xffULL) << 32) | \
		    (((b5) & 0xffULL) << 40) | \
		    (((b6) & 0xffULL) << 48) | \
		    (((b7) & 0xffULL) << 56)) \
	}

#define M128I_U32(a0, a1, b0, b1) \
	{ \
	  SWAP_LE64((((a0) & 0xffffffffULL) << 0) | \
		    (((a1) & 0xffffffffULL) << 32)), \
	  SWAP_LE64((((b0) & 0xffffffffULL) << 0) | \
		    (((b1) & 0xffffffffULL) << 32)) \
	}

#define M128I_REP16(x) { (0x0101010101010101ULL * (x)), (0x0101010101010101ULL * (x)) }

#define SHUFB_BYTES(idx) \
	(((0 + (idx)) << 0)  | ((4 + (idx)) << 8) | \
	 ((8 + (idx)) << 16) | ((12 + (idx)) << 24))

typedef u64 uint64_unaligned_t __attribute__((aligned(1), may_alias));

static const __m128i shufb_16x16b =
  M128I_U32(SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3));

static const __m128i pack_bswap =
  M128I_U32(0x00010203, 0x04050607, 0x0f0f0f0f, 0x0f0f0f0f);

static const __m128i bcast[8] =
{
  M128I_REP16(0), M128I_REP16(1), M128I_REP16(2), M128I_REP16(3),
  M128I_REP16(4), M128I_REP16(5), M128I_REP16(6), M128I_REP16(7)
};

/*
 * pre-SubByte transform
 *
 * pre-lookup for sbox1, sbox2, sbox3:
 *   swap_bitendianness(
 *       isom_map_camellia_to_aes(
 *           camellia_f(
 *               swap_bitendianess(in)
 *           )
 *       )
 *   )
 *
 * (note: '⊕ 0xc5' inside camellia_f())
 */
static const __m128i pre_tf_lo_s1 =
  M128I_BYTE(0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86,
	     0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88);

static const __m128i pre_tf_hi_s1 =
  M128I_BYTE(0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a,
	     0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23);

/*
 * pre-SubByte transform
 *
 * pre-lookup for sbox4:
 *   swap_bitendianness(
 *       isom_map_camellia_to_aes(
 *           camellia_f(
 *               swap_bitendianess(in <<< 1)
 *           )
 *       )
 *   )
 *
 * (note: '⊕ 0xc5' inside camellia_f())
 */
static const __m128i pre_tf_lo_s4 =
  M128I_BYTE(0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25,
	     0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74);

static const __m128i pre_tf_hi_s4 =
  M128I_BYTE(0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72,
	     0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf);

/*
 * post-SubByte transform
 *
 * post-lookup for sbox1, sbox4:
 *  swap_bitendianness(
 *      camellia_h(
 *          isom_map_aes_to_camellia(
 *              swap_bitendianness(
 *                  aes_inverse_affine_transform(in)
 *              )
 *          )
 *      )
 *  )
 *
 * (note: '⊕ 0x6e' inside camellia_h())
 */
static const __m128i post_tf_lo_s1 =
  M128I_BYTE(0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31,
	     0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1);

static const __m128i post_tf_hi_s1 =
  M128I_BYTE(0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8,
	     0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c);

/*
 * post-SubByte transform
 *
 * post-lookup for sbox2:
 *  swap_bitendianness(
 *      camellia_h(
 *          isom_map_aes_to_camellia(
 *              swap_bitendianness(
 *                  aes_inverse_affine_transform(in)
 *              )
 *          )
 *      )
 *  ) <<< 1
 *
 * (note: '⊕ 0x6e' inside camellia_h())
 */
static const __m128i post_tf_lo_s2 =
  M128I_BYTE(0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62,
	     0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3);

static const __m128i post_tf_hi_s2 =
  M128I_BYTE(0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51,
	     0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18);

/*
 * post-SubByte transform
 *
 * post-lookup for sbox3:
 *  swap_bitendianness(
 *      camellia_h(
 *          isom_map_aes_to_camellia(
 *              swap_bitendianness(
 *                  aes_inverse_affine_transform(in)
 *              )
 *          )
 *      )
 *  ) >>> 1
 *
 * (note: '⊕ 0x6e' inside camellia_h())
 */
static const __m128i post_tf_lo_s3 =
  M128I_BYTE(0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98,
	     0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8);

static const __m128i post_tf_hi_s3 =
  M128I_BYTE(0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54,
	     0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06);

/* For isolating SubBytes from AESENCLAST, inverse shift row */
static const __m128i inv_shift_row =
  M128I_BYTE(0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b,
	     0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03);

/* 4-bit mask */
static const __m128i mask_0f =
  M128I_U32(0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f, 0x0f0f0f0f);

/* Encrypts 16 input block from IN and writes result to OUT. IN and OUT may
 * unaligned pointers. */
void ASM_FUNC_ATTR_NOINLINE
FUNC_ENC_BLK16(const void *key_table, void *vout, const void *vin,
	       int key_length)
{
  const struct enc_ctx_s
  {
    const u64 *key_table;
    int key_length;
  } sctx =
    {
      .key_table = (const u64 *)key_table,
      .key_length = key_length
    };
  const struct enc_ctx_s *ctx = &sctx;
  char *out = vout;
  const char *in = vin;
  __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
  __m128i ab[8];
  __m128i cd[8];
  __m128i tmp0, tmp1;
  unsigned int lastk, k;
  frequent_constants_declare;

  prepare_frequent_constants();

  if (ctx->key_length > 16)
    lastk = 32;
  else
    lastk = 24;

  inpack16_pre(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
	       x15, in, ctx->key_table[0]);

  inpack16_post(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
		x15, ab, cd);

  k = 0;
  while (1)
    {
      enc_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
		  x15, ab, cd, k);

      if (k == lastk - 8)
	break;

      fls16(ab, x0, x1, x2, x3, x4, x5, x6, x7, cd, x8, x9, x10, x11, x12, x13, x14,
	    x15, &ctx->key_table[k + 8], &ctx->key_table[k + 9]);

      k += 8;
    }

  /* load CD for output */
  vmovdqa128(cd[0], x8);
  vmovdqa128(cd[1], x9);
  vmovdqa128(cd[2], x10);
  vmovdqa128(cd[3], x11);
  vmovdqa128(cd[4], x12);
  vmovdqa128(cd[5], x13);
  vmovdqa128(cd[6], x14);
  vmovdqa128(cd[7], x15);

  outunpack16(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
	      x15, ctx->key_table[lastk], tmp0, tmp1);

  write_output(x7, x6, x5, x4, x3, x2, x1, x0, x15, x14, x13, x12, x11, x10, x9,
	       x8, out);

  clear_vec_regs();
}

/* Decrypts 16 input block from IN and writes result to OUT. IN and OUT may
 * unaligned pointers. */
void ASM_FUNC_ATTR_NOINLINE
FUNC_DEC_BLK16(const void *key_table, void *vout, const void *vin,
	       int key_length)
{
  const struct dec_ctx_s
  {
    const u64 *key_table;
    int key_length;
  } sctx =
    {
      .key_table = (const u64 *)key_table,
      .key_length = key_length
    };
  const struct dec_ctx_s *ctx = &sctx;
  char *out = vout;
  const char *in = vin;
  __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
  __m128i ab[8];
  __m128i cd[8];
  __m128i tmp0, tmp1;
  unsigned int firstk, k;
  frequent_constants_declare;

  prepare_frequent_constants();

  if (ctx->key_length > 16)
    firstk = 32;
  else
    firstk = 24;

  inpack16_pre(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
	       x15, in, ctx->key_table[firstk]);

  inpack16_post(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
		x15, ab, cd);

  k = firstk - 8;
  while (1)
    {
      dec_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13,
		  x14, x15, ab, cd, k);

      if (k == 0)
	break;

      fls16(ab, x0, x1, x2, x3, x4, x5, x6, x7, cd, x8, x9, x10, x11, x12, x13,
	    x14, x15, &ctx->key_table[k + 1], &ctx->key_table[k]);

      k -= 8;
    }

  /* load CD for output */
  vmovdqa128(cd[0], x8);
  vmovdqa128(cd[1], x9);
  vmovdqa128(cd[2], x10);
  vmovdqa128(cd[3], x11);
  vmovdqa128(cd[4], x12);
  vmovdqa128(cd[5], x13);
  vmovdqa128(cd[6], x14);
  vmovdqa128(cd[7], x15);

  outunpack16(x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14,
	      x15, ctx->key_table[0], tmp0, tmp1);

  write_output(x7, x6, x5, x4, x3, x2, x1, x0, x15, x14, x13, x12, x11, x10, x9,
	       x8, out);

  clear_vec_regs();
}

/********* Key setup **********************************************************/

/* Camellia F-function, 1-way SIMD128. */
#define camellia_f_core(ab, x, t0, t1, t2, t3, t4, inv_shift_row_n_s2n3_shuffle, \
			_0f0f0f0fmask, pre_s1lo_mask, pre_s1hi_mask, \
			sp1mask, sp2mask, sp3mask, sp4mask, fn_out_xor, \
			out_xor_dst) \
	/* \
	 * S-function with AES subbytes \
	 */ \
	\
	vmovdqa128_memld(&pre_tf_lo_s4, t0); \
	vmovdqa128_memld(&pre_tf_hi_s4, t1); \
	if_not_aes_subbytes(load_zero(t3)); \
	\
	/* prefilter sboxes s1,s2,s3 */ \
	filter_8bit_3op(t4, ab, pre_s1lo_mask, pre_s1hi_mask, _0f0f0f0fmask, t2); \
	\
	/* prefilter sbox s4 */ \
	filter_8bit_3op(x, ab, t0, t1, _0f0f0f0fmask, t2); \
	\
	vmovdqa128_memld(&post_tf_lo_s1, t0); \
	vmovdqa128_memld(&post_tf_hi_s1, t1); \
	\
	if_not_aes_subbytes(/* AES subbytes + AES shift rows */); \
	if_not_aes_subbytes(aes_subbytes_and_shuf_and_xor(t3, t4, t4)); \
	if_not_aes_subbytes(aes_subbytes_and_shuf_and_xor(t3, x, x)); \
	\
	if_aes_subbytes(/* AES subbytes */); \
	if_aes_subbytes(aes_subbytes(t4, t4)); \
	if_aes_subbytes(aes_subbytes(x, x)); \
	\
	/* postfilter sboxes s1,s2,s3 */ \
	filter_8bit(t4, t0, t1, _0f0f0f0fmask, t2); \
	\
	/* postfilter sbox s4 */ \
	filter_8bit(x, t0, t1, _0f0f0f0fmask, t2); \
	\
	/* output rotation for sbox2 (<<< 1) */ \
	/* output rotation for sbox3 (>>> 1) */ \
	/* permutation */ \
	if_vprolb128(vpshufb128(sp2mask, t4, t0)); \
	if_vprolb128(vpshufb128(sp3mask, t4, t1)); \
	if_vprolb128(vpshufb128(sp1mask, t4, t4)); \
	if_vprolb128(vpshufb128(sp4mask, x, x)); \
	if_vprolb128(vprolb128(1, t0, t0, t2)); \
	if_vprolb128(vprolb128(7, t1, t1, t3)); \
	if_not_vprolb128(aes_inv_shuf(inv_shift_row_n_s2n3_shuffle, t4, t1)); \
	if_not_vprolb128(vpshufb128(sp1mask, t4, t4)); \
	if_not_vprolb128(vpshufb128(sp4mask, x, x)); \
	if_not_vprolb128(vpaddb128(t1, t1, t2)); \
	if_not_vprolb128(vpsrl_byte_128(7, t1, t0)); \
	if_not_vprolb128(vpsll_byte_128(7, t1, t3)); \
	if_not_vprolb128(vpor128(t0, t2, t0)); \
	if_not_vprolb128(vpsrl_byte_128(1, t1, t1)); \
	if_not_vprolb128(vpshufb128(sp2mask, t0, t0)); \
	if_not_vprolb128(vpor128(t1, t3, t1)); \
	if_not_vprolb128(vpshufb128(sp3mask, t1, t1)); \
	\
	vpxor128(x, t4, t4); \
	vpxor128(t4, t0, t0); \
	vpxor128(t1, t0, t0); \
	vpsrldq128(8, t0, x); \
	fn_out_xor(t0, x, out_xor_dst);

#define camellia_f_xor_x(t0, x, _) \
	vpxor128(t0, x, x);

/*
 * IN:
 *  ab: 64-bit AB state
 *  cd: 64-bit CD state
 */
#define camellia_f(ab, x, t0, t1, t2, t3, t4, inv_shift_row, \
		   _0f0f0f0fmask, pre_s1lo_mask, pre_s1hi_mask, key) \
	({ \
	  __m128i sp1mask, sp2mask, sp3mask, sp4mask; \
	  vmovq128_amemld(&(key), t0); \
	  vmovdqa128_memld(&sp1110111010011110mask, sp1mask); \
	  vmovdqa128_memld(&sp0222022222000222mask, sp2mask); \
	  vmovdqa128_memld(&sp3033303303303033mask, sp3mask); \
	  vmovdqa128_memld(&sp0044440444044404mask, sp4mask); \
	  vpxor128(ab, t0, x); \
	  camellia_f_core(x, x, t0, t1, t2, t3, t4, inv_shift_row, \
			  _0f0f0f0fmask, pre_s1lo_mask, pre_s1hi_mask, \
			  sp1mask, sp2mask, sp3mask, sp4mask, \
			  camellia_f_xor_x, _); \
	})

#define vec_rol128(in, out, nrol, t0) \
	vpshufd128_0x4e(in, out); \
	vpsllq128((nrol), in, t0); \
	vpsrlq128((64-(nrol)), out, out); \
	vpaddb128(t0, out, out);

#define vec_ror128(in, out, nror, t0) \
	vpshufd128_0x4e(in, out); \
	vpsrlq128((nror), in, t0); \
	vpsllq128((64-(nror)), out, out); \
	vpaddb128(t0, out, out);

#define U64_BYTE(a0, a1, a2, a3, b0, b1, b2, b3) \
	( \
	  SWAP_LE64((((a0) & 0xffULL) << 0) | \
		    (((a1) & 0xffULL) << 8) | \
		    (((a2) & 0xffULL) << 16) | \
		    (((a3) & 0xffULL) << 24) | \
		    (((b0) & 0xffULL) << 32) | \
		    (((b1) & 0xffULL) << 40) | \
		    (((b2) & 0xffULL) << 48) | \
		    (((b3) & 0xffULL) << 56)) \
	)

#define U64_U32(a0, b0) \
	( \
	  SWAP_LE64((((a0) & 0xffffffffULL) << 0) | \
		    (((b0) & 0xffffffffULL) << 32)) \
	)

static const __m128i bswap128_mask =
  M128I_BYTE(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);

if_not_vprolb128(
  static const __m128i inv_shift_row_and_unpcklbw =
    M128I_BYTE(0x00, 0xff, 0x0d, 0xff, 0x0a, 0xff, 0x07, 0xff,
	       0x04, 0xff, 0x01, 0xff, 0x0e, 0xff, 0x0b, 0xff);
)

static const __m128i sp0044440444044404mask =
  if_aes_subbytes(M128I_U32(0xffff0404, 0x0404ff04, 0x0101ff01, 0x0101ff01))
  if_not_aes_subbytes(M128I_U32(0xffff0404, 0x0404ff04, 0x0d0dff0d, 0x0d0dff0d));

static const __m128i sp1110111010011110mask =
  if_aes_subbytes(M128I_U32(0x000000ff, 0x000000ff, 0x07ffff07, 0x070707ff))
  if_not_aes_subbytes(M128I_U32(0x000000ff, 0x000000ff, 0x0bffff0b, 0x0b0b0bff));

static const __m128i sp0222022222000222mask =
  if_aes_subbytes(if_vprolb128(M128I_U32(0xff030303, 0xff030303, 0x0606ffff, 0xff060606)))
  if_aes_subbytes(if_not_vprolb128(M128I_U32(0xff0e0e0e, 0xff0e0e0e, 0x0c0cffff, 0xff0c0c0c)))
  if_not_aes_subbytes(if_vprolb128(M128I_U32(0xff070707, 0xff070707, 0x0e0effff, 0xff0e0e0e)))
  if_not_aes_subbytes(if_not_vprolb128(M128I_U32(0xff060606, 0xff060606, 0x0c0cffff, 0xff0c0c0c)));

static const __m128i sp3033303303303033mask =
  if_aes_subbytes(if_vprolb128(M128I_U32(0x02ff0202, 0x02ff0202, 0xff0505ff, 0x05ff0505)))
  if_aes_subbytes(if_not_vprolb128(M128I_U32(0x04ff0404, 0x04ff0404, 0xff0202ff, 0x0202ff02)))
  if_not_aes_subbytes(if_vprolb128(M128I_U32(0x0aff0a0a, 0x0aff0a0a, 0xff0101ff, 0x01ff0101)))
  if_not_aes_subbytes(if_not_vprolb128(M128I_U32(0x04ff0404, 0x04ff0404, 0xff0a0aff, 0x0aff0a0a)));

static const u64 sigma1 =
  U64_U32(0x3BCC908B, 0xA09E667F);

static const u64 sigma2 =
  U64_U32(0x4CAA73B2, 0xB67AE858);

static const u64 sigma3 =
  U64_U32(0xE94F82BE, 0xC6EF372F);

static const u64 sigma4 =
  U64_U32(0xF1D36F1C, 0x54FF53A5);

static const u64 sigma5 =
  U64_U32(0xDE682D1D, 0x10E527FA);

static const u64 sigma6 =
  U64_U32(0xB3E6C1FD, 0xB05688C2);

#define cmll_sub(n, ctx) &ctx->key_table[n]

static ASM_FUNC_ATTR_INLINE void
camellia_setup128(void *key_table, __m128i x0)
{
  struct setup128_ctx_s
  {
    u64 *key_table;
  } sctx = { .key_table = (u64 *)key_table };
  struct setup128_ctx_s *ctx = &sctx;

  /* input:
   *   ctx: subkey storage at key_table(CTX)
   *   x0: key
   */

  __m128i x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
  __m128i tmp0;

#define KL128 x0
#define KA128 x2

  vpshufb128_amemld(&bswap128_mask, KL128, KL128);

  if_not_vprolb128(vmovdqa128_memld(&inv_shift_row_and_unpcklbw, x11));
  vmovdqa128_memld(&mask_0f, x13);
  vmovdqa128_memld(&pre_tf_lo_s1, x14);
  vmovdqa128_memld(&pre_tf_hi_s1, x15);

  /*
   * Generate KA
   */
  vpsrldq128(8, KL128, x2);
  vmovdqa128(KL128, x3);
  vpslldq128(8, x3, x3);
  vpsrldq128(8, x3, x3);

  camellia_f(x2, x4, x1,
	     x5, x6, x7, x8,
	     x11, x13, x14, x15, sigma1);
  vpxor128(x4, x3, x3);
  camellia_f(x3, x2, x1,
	     x5, x6, x7, x8,
	     x11, x13, x14, x15, sigma2);
  camellia_f(x2, x3, x1,
	     x5, x6, x7, x8,
	     x11, x13, x14, x15, sigma3);
  vpxor128(x4, x3, x3);
  camellia_f(x3, x4, x1,
	     x5, x6, x7, x8,
	     x11, x13, x14, x15, sigma4);

  vpslldq128(8, x3, x3);
  vpxor128(x4, x2, x2);
  vpsrldq128(8, x3, x3);
  vpslldq128(8, x2, KA128);
  vpor128(x3, KA128, KA128);

  /*
   * Generate subkeys
   */
  vmovdqu128_memst(KA128, cmll_sub(24, ctx));
  vec_rol128(KL128, x3, 15, x15);
  vec_rol128(KA128, x4, 15, x15);
  vec_rol128(KA128, x5, 30, x15);
  vec_rol128(KL128, x6, 45, x15);
  vec_rol128(KA128, x7, 45, x15);
  vec_rol128(KL128, x8, 60, x15);
  vec_rol128(KA128, x9, 60, x15);
  vec_ror128(KL128, x10, 128-77, x15);

  /* absorb kw2 to other subkeys */
  vpslldq128(8, KL128, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, KA128, KA128);
  vpxor128(x15, x3, x3);
  vpxor128(x15, x4, x4);

  /* subl(1) ^= subr(1) & ~subr(9); */
  vpandn128(x15, x5, x13);
  vpslldq128(12, x13, x13);
  vpsrldq128(8, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(1) & subl(9), subr(1) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x5, x14);
  vpslld128(1, x14, x11);
  vpsrld128(31, x14, x14);
  vpaddb128(x11, x14, x14);
  vpslldq128(8, x14, x14);
  vpsrldq128(12, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x6, x6);
  vpxor128(x15, x8, x8);
  vpxor128(x15, x9, x9);

  /* subl(1) ^= subr(1) & ~subr(17); */
  vpandn128(x15, x10, x13);
  vpslldq128(12, x13, x13);
  vpsrldq128(8, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(1) & subl(17), subr(1) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x10, x14);
  vpslld128(1, x14, x11);
  vpsrld128(31, x14, x14);
  vpaddb128(x11, x14, x14);
  vpslldq128(8, x14, x14);
  vpsrldq128(12, x14, x14);
  vpxor128(x14, x15, x15);

  vpshufd128_0x1b(KL128, KL128);
  vpshufd128_0x1b(KA128, KA128);
  vpshufd128_0x1b(x3, x3);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x5, x5);
  vpshufd128_0x1b(x6, x6);
  vpshufd128_0x1b(x7, x7);
  vpshufd128_0x1b(x8, x8);
  vpshufd128_0x1b(x9, x9);
  vpshufd128_0x1b(x10, x10);

  vmovdqu128_memst(KL128, cmll_sub(0, ctx));
  vpshufd128_0x1b(KL128, KL128);
  vmovdqu128_memst(KA128, cmll_sub(2, ctx));
  vmovdqu128_memst(x3, cmll_sub(4, ctx));
  vmovdqu128_memst(x4, cmll_sub(6, ctx));
  vmovdqu128_memst(x5, cmll_sub(8, ctx));
  vmovdqu128_memst(x6, cmll_sub(10, ctx));
  vpsrldq128(8, x8, x8);
  vmovq128_memst(x7, cmll_sub(12, ctx));
  vmovq128_memst(x8, cmll_sub(13, ctx));
  vmovdqu128_memst(x9, cmll_sub(14, ctx));
  vmovdqu128_memst(x10, cmll_sub(16, ctx));

  vmovdqu128_memld(cmll_sub(24, ctx), KA128);

  vec_ror128(KL128, x3, 128 - 94, x7);
  vec_ror128(KA128, x4, 128 - 94, x7);
  vec_ror128(KL128, x5, 128 - 111, x7);
  vec_ror128(KA128, x6, 128 - 111, x7);

  vpxor128(x15, x3, x3);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x5, x5);
  vpslldq128(8, x15, x15);
  vpxor128(x15, x6, x6);

  /* absorb kw4 to other subkeys */
  vpslldq128(8, x6, x15);
  vpxor128(x15, x5, x5);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x3, x3);

  /* subl(25) ^= subr(25) & ~subr(16); */
  vmovdqu128_memld(cmll_sub(16, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x10);
  vpandn128(x15, x10, x13);
  vpslldq128(4, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(25) & subl(16), subr(25) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x10, x14);
  vpslld128(1, x14, x11);
  vpsrld128(31, x14, x14);
  vpaddb128(x11, x14, x14);
  vpsrldq128(12, x14, x14);
  vpslldq128(8, x14, x14);
  vpxor128(x14, x15, x15);

  vpshufd128_0x1b(x3, x3);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x5, x5);
  vpshufd128_0x1b(x6, x6);

  vmovdqu128_memst(x3, cmll_sub(18, ctx));
  vmovdqu128_memst(x4, cmll_sub(20, ctx));
  vmovdqu128_memst(x5, cmll_sub(22, ctx));
  vmovdqu128_memst(x6, cmll_sub(24, ctx));

  vmovdqu128_memld(cmll_sub(14, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x3);
  vmovdqu128_memld(cmll_sub(12, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x4);
  vmovdqu128_memld(cmll_sub(10, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x5);
  vmovdqu128_memld(cmll_sub(8, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x6);

  vpxor128(x15, x3, x3);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x5, x5);

  /* subl(25) ^= subr(25) & ~subr(8); */
  vpandn128(x15, x6, x13);
  vpslldq128(4, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(25) & subl(8), subr(25) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x6, x14);
  vpslld128(1, x14, x11);
  vpsrld128(31, x14, x14);
  vpaddb128(x11, x14, x14);
  vpsrldq128(12, x14, x14);
  vpslldq128(8, x14, x14);
  vpxor128(x14, x15, x15);

  vpshufd128_0x1b(x3, x3);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x5, x5);

  vmovdqu128_memst(x3, cmll_sub(14, ctx));
  vmovdqu128_memst(x4, cmll_sub(12, ctx));
  vmovdqu128_memst(x5, cmll_sub(10, ctx));

  vmovdqu128_memld(cmll_sub(6, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x6);
  vmovdqu128_memld(cmll_sub(4, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x4);
  vmovdqu128_memld(cmll_sub(2, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x2);
  vmovdqu128_memld(cmll_sub(0, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x0);

  vpxor128(x15, x6, x6);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x2, x2);
  vpxor128(x15, x0, x0);

  vpshufd128_0x1b(x6, x6);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x2, x2);
  vpshufd128_0x1b(x0, x0);

  vpsrldq128(8, x2, x3);
  vpsrldq128(8, x4, x5);
  vpsrldq128(8, x6, x7);

  /*
   * key XOR is end of F-function.
   */
  vpxor128(x2, x0, x0);
  vpxor128(x4, x2, x2);

  vmovq128_memst(x0, cmll_sub(0, ctx));
  vmovq128_memst(x3, cmll_sub(2, ctx));
  vpxor128(x5, x3, x3);
  vpxor128(x6, x4, x4);
  vpxor128(x7, x5, x5);
  vmovq128_memst(x2, cmll_sub(3, ctx));
  vmovq128_memst(x3, cmll_sub(4, ctx));
  vmovq128_memst(x4, cmll_sub(5, ctx));
  vmovq128_memst(x5, cmll_sub(6, ctx));

  vmovq128_amemld(cmll_sub(7, ctx), x7);
  vmovq128_amemld(cmll_sub(8, ctx), x8);
  vmovq128_amemld(cmll_sub(9, ctx), x9);
  vmovq128_amemld(cmll_sub(10, ctx), x10);
  /* tl = subl(10) ^ (subr(10) & ~subr(8)); */
  vpandn128(x10, x8, x15);
  vpsrldq128(4, x15, x15);
  vpxor128(x15, x10, x0);
  /* dw = tl & subl(8), tr = subr(10) ^ CAMELLIA_RL1(dw); */
  vpand128(x8, x0, x15);
  vpslld128(1, x15, x14);
  vpsrld128(31, x15, x15);
  vpaddb128(x14, x15, x15);
  vpslldq128(12, x15, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, x0, x0);

  vpxor128(x0, x6, x6);
  vmovq128_memst(x6, cmll_sub(7, ctx));

  vmovq128_amemld(cmll_sub(11, ctx), x11);
  vmovq128_amemld(cmll_sub(12, ctx), x12);
  vmovq128_amemld(cmll_sub(13, ctx), x13);
  vmovq128_amemld(cmll_sub(14, ctx), x14);
  vmovq128_amemld(cmll_sub(15, ctx), x15);
  /* tl = subl(7) ^ (subr(7) & ~subr(9)); */
  vpandn128(x7, x9, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x7, x0);
  /* dw = tl & subl(9), tr = subr(7) ^ CAMELLIA_RL1(dw); */
  vpand128(x9, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vpxor128(x11, x0, x0);
  vpxor128(x12, x10, x10);
  vpxor128(x13, x11, x11);
  vpxor128(x14, x12, x12);
  vpxor128(x15, x13, x13);
  vmovq128_memst(x0, cmll_sub(10, ctx));
  vmovq128_memst(x10, cmll_sub(11, ctx));
  vmovq128_memst(x11, cmll_sub(12, ctx));
  vmovq128_memst(x12, cmll_sub(13, ctx));
  vmovq128_memst(x13, cmll_sub(14, ctx));

  vmovq128_amemld(cmll_sub(16, ctx), x6);
  vmovq128_amemld(cmll_sub(17, ctx), x7);
  vmovq128_amemld(cmll_sub(18, ctx), x8);
  vmovq128_amemld(cmll_sub(19, ctx), x9);
  vmovq128_amemld(cmll_sub(20, ctx), x10);
  /* tl = subl(18) ^ (subr(18) & ~subr(16)); */
  vpandn128(x8, x6, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x8, x0);
  /* dw = tl & subl(16), tr = subr(18) ^ CAMELLIA_RL1(dw); */
  vpand128(x6, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vpxor128(x14, x0, x0);
  vmovq128_memst(x0, cmll_sub(15, ctx));

  /* tl = subl(15) ^ (subr(15) & ~subr(17)); */
  vpandn128(x15, x7, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x15, x0);
  /* dw = tl & subl(17), tr = subr(15) ^ CAMELLIA_RL1(dw); */
  vpand128(x7, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vmovq128_amemld(cmll_sub(21, ctx), x1);
  vmovq128_amemld(cmll_sub(22, ctx), x2);
  vmovq128_amemld(cmll_sub(23, ctx), x3);
  vmovq128_amemld(cmll_sub(24, ctx), x4);

  vpxor128(x9, x0, x0);
  vpxor128(x10, x8, x8);
  vpxor128(x1, x9, x9);
  vpxor128(x2, x10, x10);
  vpxor128(x3, x1, x1);
  vpxor128(x4, x3, x3);

  vmovq128_memst(x0, cmll_sub(18, ctx));
  vmovq128_memst(x8, cmll_sub(19, ctx));
  vmovq128_memst(x9, cmll_sub(20, ctx));
  vmovq128_memst(x10, cmll_sub(21, ctx));
  vmovq128_memst(x1, cmll_sub(22, ctx));
  vmovq128_memst(x2, cmll_sub(23, ctx));
  vmovq128_memst(x3, cmll_sub(24, ctx));

#undef KL128
#undef KA128

  /* kw2 and kw4 are unused now. */
  load_zero(tmp0);
  vmovq128_memst(tmp0, cmll_sub(1, ctx));
  vmovq128_memst(tmp0, cmll_sub(25, ctx));

  clear_vec_regs();
}

static ASM_FUNC_ATTR_INLINE void
camellia_setup256(void *key_table, __m128i x0, __m128i x1)
{
  struct setup256_ctx_s
  {
    u64 *key_table;
  } sctx = { .key_table = (u64 *)key_table };
  struct setup256_ctx_s *ctx = &sctx;

  /* input:
   *   ctx: subkey storage at key_table(CTX)
   *   x0, x1: key
   */

  __m128i x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
  __m128i tmp0;

#define KL128 x0
#define KR128 x1
#define KA128 x2
#define KB128 x3

  vpshufb128_amemld(&bswap128_mask, KL128, KL128);
  vpshufb128_amemld(&bswap128_mask, KR128, KR128);

  if_not_vprolb128(vmovdqa128_memld(&inv_shift_row_and_unpcklbw, x11));
  vmovdqa128_memld(&mask_0f, x13);
  vmovdqa128_memld(&pre_tf_lo_s1, x14);
  vmovdqa128_memld(&pre_tf_hi_s1, x15);

  /*
   * Generate KA
   */
  vpxor128(KL128, KR128, x3);
  vpsrldq128(8, KR128, x6);
  vpsrldq128(8, x3, x2);
  vpslldq128(8, x3, x3);
  vpsrldq128(8, x3, x3);

  camellia_f(x2, x4, x5,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma1);
  vpxor128(x4, x3, x3);
  camellia_f(x3, x2, x5,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma2);
  vpxor128(x6, x2, x2);
  camellia_f(x2, x3, x5,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma3);
  vpxor128(x4, x3, x3);
  vpxor128(KR128, x3, x3);
  camellia_f(x3, x4, x5,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma4);

  vpslldq128(8, x3, x3);
  vpxor128(x4, x2, x2);
  vpsrldq128(8, x3, x3);
  vpslldq128(8, x2, KA128);
  vpor128(x3, KA128, KA128);

  /*
   * Generate KB
   */
  vpxor128(KA128, KR128, x3);
  vpsrldq128(8, x3, x4);
  vpslldq128(8, x3, x3);
  vpsrldq128(8, x3, x3);

  camellia_f(x4, x5, x6,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma5);
  vpxor128(x5, x3, x3);

  camellia_f(x3, x5, x6,
	     x7, x8, x9, x10,
	     x11, x13, x14, x15, sigma6);
  vpslldq128(8, x3, x3);
  vpxor128(x5, x4, x4);
  vpsrldq128(8, x3, x3);
  vpslldq128(8, x4, x4);
  vpor128(x3, x4, KB128);

  /*
   * Generate subkeys
   */
  vmovdqu128_memst(KB128, cmll_sub(32, ctx));
  vec_rol128(KR128, x4, 15, x15);
  vec_rol128(KA128, x5, 15, x15);
  vec_rol128(KR128, x6, 30, x15);
  vec_rol128(KB128, x7, 30, x15);
  vec_rol128(KL128, x8, 45, x15);
  vec_rol128(KA128, x9, 45, x15);
  vec_rol128(KL128, x10, 60, x15);
  vec_rol128(KR128, x11, 60, x15);
  vec_rol128(KB128, x12, 60, x15);

  /* absorb kw2 to other subkeys */
  vpslldq128(8, KL128, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, KB128, KB128);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x5, x5);

  /* subl(1) ^= subr(1) & ~subr(9); */
  vpandn128(x15, x6, x13);
  vpslldq128(12, x13, x13);
  vpsrldq128(8, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(1) & subl(9), subr(1) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x6, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpslldq128(8, x14, x14);
  vpsrldq128(12, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x7, x7);
  vpxor128(x15, x8, x8);
  vpxor128(x15, x9, x9);

  vpshufd128_0x1b(KL128, KL128);
  vpshufd128_0x1b(KB128, KB128);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x5, x5);
  vpshufd128_0x1b(x6, x6);
  vpshufd128_0x1b(x7, x7);
  vpshufd128_0x1b(x8, x8);
  vpshufd128_0x1b(x9, x9);

  vmovdqu128_memst(KL128, cmll_sub(0, ctx));
  vpshufd128_0x1b(KL128, KL128);
  vmovdqu128_memst(KB128, cmll_sub(2, ctx));
  vmovdqu128_memst(x4, cmll_sub(4, ctx));
  vmovdqu128_memst(x5, cmll_sub(6, ctx));
  vmovdqu128_memst(x6, cmll_sub(8, ctx));
  vmovdqu128_memst(x7, cmll_sub(10, ctx));
  vmovdqu128_memst(x8, cmll_sub(12, ctx));
  vmovdqu128_memst(x9, cmll_sub(14, ctx));

  vmovdqu128_memld(cmll_sub(32, ctx), KB128);

  /* subl(1) ^= subr(1) & ~subr(17); */
  vpandn128(x15, x10, x13);
  vpslldq128(12, x13, x13);
  vpsrldq128(8, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(1) & subl(17), subr(1) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x10, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpslldq128(8, x14, x14);
  vpsrldq128(12, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x11, x11);
  vpxor128(x15, x12, x12);

  vec_ror128(KL128, x4, 128-77, x14);
  vec_ror128(KA128, x5, 128-77, x14);
  vec_ror128(KR128, x6, 128-94, x14);
  vec_ror128(KA128, x7, 128-94, x14);
  vec_ror128(KL128, x8, 128-111, x14);
  vec_ror128(KB128, x9, 128-111, x14);

  vpxor128(x15, x4, x4);

  vpshufd128_0x1b(x10, x10);
  vpshufd128_0x1b(x11, x11);
  vpshufd128_0x1b(x12, x12);
  vpshufd128_0x1b(x4, x4);

  vmovdqu128_memst(x10, cmll_sub(16, ctx));
  vmovdqu128_memst(x11, cmll_sub(18, ctx));
  vmovdqu128_memst(x12, cmll_sub(20, ctx));
  vmovdqu128_memst(x4, cmll_sub(22, ctx));

  /* subl(1) ^= subr(1) & ~subr(25); */
  vpandn128(x15, x5, x13);
  vpslldq128(12, x13, x13);
  vpsrldq128(8, x13, x13);
  vpxor128(x13, x15, x15);
  /* dw = subl(1) & subl(25), subr(1) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x5, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpslldq128(8, x14, x14);
  vpsrldq128(12, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x6, x6);
  vpxor128(x15, x7, x7);
  vpxor128(x15, x8, x8);
  vpslldq128(8, x15, x15);
  vpxor128(x15, x9, x9);

  /* absorb kw4 to other subkeys */
  vpslldq128(8, x9, x15);
  vpxor128(x15, x8, x8);
  vpxor128(x15, x7, x7);
  vpxor128(x15, x6, x6);

  /* subl(33) ^= subr(33) & ~subr(24); */
  vpandn128(x15, x5, x14);
  vpslldq128(4, x14, x14);
  vpxor128(x14, x15, x15);
  /* dw = subl(33) & subl(24), subr(33) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x5, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpsrldq128(12, x14, x14);
  vpslldq128(8, x14, x14);
  vpxor128(x14, x15, x15);

  vpshufd128_0x1b(x5, x5);
  vpshufd128_0x1b(x6, x6);
  vpshufd128_0x1b(x7, x7);
  vpshufd128_0x1b(x8, x8);
  vpshufd128_0x1b(x9, x9);

  vmovdqu128_memst(x5, cmll_sub(24, ctx));
  vmovdqu128_memst(x6, cmll_sub(26, ctx));
  vmovdqu128_memst(x7, cmll_sub(28, ctx));
  vmovdqu128_memst(x8, cmll_sub(30, ctx));
  vmovdqu128_memst(x9, cmll_sub(32, ctx));

  vmovdqu128_memld(cmll_sub(22, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x0);
  vmovdqu128_memld(cmll_sub(20, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x1);
  vmovdqu128_memld(cmll_sub(18, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x2);
  vmovdqu128_memld(cmll_sub(16, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x3);
  vmovdqu128_memld(cmll_sub(14, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x4);
  vmovdqu128_memld(cmll_sub(12, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x5);
  vmovdqu128_memld(cmll_sub(10, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x6);
  vmovdqu128_memld(cmll_sub(8, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x7);

  vpxor128(x15, x0, x0);
  vpxor128(x15, x1, x1);
  vpxor128(x15, x2, x2);

  /* subl(33) ^= subr(33) & ~subr(24); */
  vpandn128(x15, x3, x14);
  vpslldq128(4, x14, x14);
  vpxor128(x14, x15, x15);
  /* dw = subl(33) & subl(24), subr(33) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x3, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpsrldq128(12, x14, x14);
  vpslldq128(8, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x4, x4);
  vpxor128(x15, x5, x5);
  vpxor128(x15, x6, x6);

  vpshufd128_0x1b(x0, x0);
  vpshufd128_0x1b(x1, x1);
  vpshufd128_0x1b(x2, x2);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x5, x5);
  vpshufd128_0x1b(x6, x6);

  vmovdqu128_memst(x0, cmll_sub(22, ctx));
  vmovdqu128_memst(x1, cmll_sub(20, ctx));
  vmovdqu128_memst(x2, cmll_sub(18, ctx));
  vmovdqu128_memst(x4, cmll_sub(14, ctx));
  vmovdqu128_memst(x5, cmll_sub(12, ctx));
  vmovdqu128_memst(x6, cmll_sub(10, ctx));

  vmovdqu128_memld(cmll_sub(6, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x6);
  vmovdqu128_memld(cmll_sub(4, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x4);
  vmovdqu128_memld(cmll_sub(2, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x2);
  vmovdqu128_memld(cmll_sub(0, ctx), tmp0);
  vpshufd128_0x1b(tmp0, x0);

  /* subl(33) ^= subr(33) & ~subr(24); */
  vpandn128(x15, x7, x14);
  vpslldq128(4, x14, x14);
  vpxor128(x14, x15, x15);
  /* dw = subl(33) & subl(24), subr(33) ^= CAMELLIA_RL1(dw); */
  vpand128(x15, x7, x14);
  vpslld128(1, x14, x13);
  vpsrld128(31, x14, x14);
  vpaddb128(x13, x14, x14);
  vpsrldq128(12, x14, x14);
  vpslldq128(8, x14, x14);
  vpxor128(x14, x15, x15);

  vpxor128(x15, x6, x6);
  vpxor128(x15, x4, x4);
  vpxor128(x15, x2, x2);
  vpxor128(x15, x0, x0);

  vpshufd128_0x1b(x6, x6);
  vpshufd128_0x1b(x4, x4);
  vpshufd128_0x1b(x2, x2);
  vpshufd128_0x1b(x0, x0);

  vpsrldq128(8, x2, x3);
  vpsrldq128(8, x4, x5);
  vpsrldq128(8, x6, x7);

  /*
    * key XOR is end of F-function.
    */
  vpxor128(x2, x0, x0);
  vpxor128(x4, x2, x2);

  vmovq128_memst(x0, cmll_sub(0, ctx));
  vmovq128_memst(x3, cmll_sub(2, ctx));
  vpxor128(x5, x3, x3);
  vpxor128(x6, x4, x4);
  vpxor128(x7, x5, x5);
  vmovq128_memst(x2, cmll_sub(3, ctx));
  vmovq128_memst(x3, cmll_sub(4, ctx));
  vmovq128_memst(x4, cmll_sub(5, ctx));
  vmovq128_memst(x5, cmll_sub(6, ctx));

  vmovq128_amemld(cmll_sub(7, ctx), x7);
  vmovq128_amemld(cmll_sub(8, ctx), x8);
  vmovq128_amemld(cmll_sub(9, ctx), x9);
  vmovq128_amemld(cmll_sub(10, ctx), x10);
  /* tl = subl(10) ^ (subr(10) & ~subr(8)); */
  vpandn128(x10, x8, x15);
  vpsrldq128(4, x15, x15);
  vpxor128(x15, x10, x0);
  /* dw = tl & subl(8), tr = subr(10) ^ CAMELLIA_RL1(dw); */
  vpand128(x8, x0, x15);
  vpslld128(1, x15, x14);
  vpsrld128(31, x15, x15);
  vpaddb128(x14, x15, x15);
  vpslldq128(12, x15, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, x0, x0);

  vpxor128(x0, x6, x6);
  vmovq128_memst(x6, cmll_sub(7, ctx));

  vmovq128_amemld(cmll_sub(11, ctx), x11);
  vmovq128_amemld(cmll_sub(12, ctx), x12);
  vmovq128_amemld(cmll_sub(13, ctx), x13);
  vmovq128_amemld(cmll_sub(14, ctx), x14);
  vmovq128_amemld(cmll_sub(15, ctx), x15);
  /* tl = subl(7) ^ (subr(7) & ~subr(9)); */
  vpandn128(x7, x9, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x7, x0);
  /* dw = tl & subl(9), tr = subr(7) ^ CAMELLIA_RL1(dw); */
  vpand128(x9, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vpxor128(x11, x0, x0);
  vpxor128(x12, x10, x10);
  vpxor128(x13, x11, x11);
  vpxor128(x14, x12, x12);
  vpxor128(x15, x13, x13);
  vmovq128_memst(x0, cmll_sub(10, ctx));
  vmovq128_memst(x10, cmll_sub(11, ctx));
  vmovq128_memst(x11, cmll_sub(12, ctx));
  vmovq128_memst(x12, cmll_sub(13, ctx));
  vmovq128_memst(x13, cmll_sub(14, ctx));

  vmovq128_amemld(cmll_sub(16, ctx), x6);
  vmovq128_amemld(cmll_sub(17, ctx), x7);
  vmovq128_amemld(cmll_sub(18, ctx), x8);
  vmovq128_amemld(cmll_sub(19, ctx), x9);
  vmovq128_amemld(cmll_sub(20, ctx), x10);
  /* tl = subl(18) ^ (subr(18) & ~subr(16)); */
  vpandn128(x8, x6, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x8, x0);
  /* dw = tl & subl(16), tr = subr(18) ^ CAMELLIA_RL1(dw); */
  vpand128(x6, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vpxor128(x14, x0, x0);
  vmovq128_memst(x0, cmll_sub(15, ctx));

  /* tl = subl(15) ^ (subr(15) & ~subr(17)); */
  vpandn128(x15, x7, x1);
  vpsrldq128(4, x1, x1);
  vpxor128(x1, x15, x0);
  /* dw = tl & subl(17), tr = subr(15) ^ CAMELLIA_RL1(dw); */
  vpand128(x7, x0, x1);
  vpslld128(1, x1, x2);
  vpsrld128(31, x1, x1);
  vpaddb128(x2, x1, x1);
  vpslldq128(12, x1, x1);
  vpsrldq128(8, x1, x1);
  vpxor128(x1, x0, x0);

  vmovq128_amemld(cmll_sub(21, ctx), x1);
  vmovq128_amemld(cmll_sub(22, ctx), x2);
  vmovq128_amemld(cmll_sub(23, ctx), x3);
  vmovq128_amemld(cmll_sub(24, ctx), x4);

  vpxor128(x9, x0, x0);
  vpxor128(x10, x8, x8);
  vpxor128(x1, x9, x9);
  vpxor128(x2, x10, x10);
  vpxor128(x3, x1, x1);

  vmovq128_memst(x0, cmll_sub(18, ctx));
  vmovq128_memst(x8, cmll_sub(19, ctx));
  vmovq128_memst(x9, cmll_sub(20, ctx));
  vmovq128_memst(x10, cmll_sub(21, ctx));
  vmovq128_memst(x1, cmll_sub(22, ctx));

  vmovq128_amemld(cmll_sub(25, ctx), x5);
  vmovq128_amemld(cmll_sub(26, ctx), x6);
  vmovq128_amemld(cmll_sub(27, ctx), x7);
  vmovq128_amemld(cmll_sub(28, ctx), x8);
  vmovq128_amemld(cmll_sub(29, ctx), x9);
  vmovq128_amemld(cmll_sub(30, ctx), x10);
  vmovq128_amemld(cmll_sub(31, ctx), x11);
  vmovq128_amemld(cmll_sub(32, ctx), x12);

  /* tl = subl(26) ^ (subr(26) & ~subr(24)); */
  vpandn128(x6, x4, x15);
  vpsrldq128(4, x15, x15);
  vpxor128(x15, x6, x0);
  /* dw = tl & subl(26), tr = subr(24) ^ CAMELLIA_RL1(dw); */
  vpand128(x4, x0, x15);
  vpslld128(1, x15, x14);
  vpsrld128(31, x15, x15);
  vpaddb128(x14, x15, x15);
  vpslldq128(12, x15, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, x0, x0);

  vpxor128(x0, x2, x2);
  vmovq128_memst(x2, cmll_sub(23, ctx));

  /* tl = subl(23) ^ (subr(23) &  ~subr(25)); */
  vpandn128(x3, x5, x15);
  vpsrldq128(4, x15, x15);
  vpxor128(x15, x3, x0);
  /* dw = tl & subl(26), tr = subr(24) ^ CAMELLIA_RL1(dw); */
  vpand128(x5, x0, x15);
  vpslld128(1, x15, x14);
  vpsrld128(31, x15, x15);
  vpaddb128(x14, x15, x15);
  vpslldq128(12, x15, x15);
  vpsrldq128(8, x15, x15);
  vpxor128(x15, x0, x0);

  vpxor128(x7, x0, x0);
  vpxor128(x8, x6, x6);
  vpxor128(x9, x7, x7);
  vpxor128(x10, x8, x8);
  vpxor128(x11, x9, x9);
  vpxor128(x12, x11, x11);

  vmovq128_memst(x0, cmll_sub(26, ctx));
  vmovq128_memst(x6, cmll_sub(27, ctx));
  vmovq128_memst(x7, cmll_sub(28, ctx));
  vmovq128_memst(x8, cmll_sub(29, ctx));
  vmovq128_memst(x9, cmll_sub(30, ctx));
  vmovq128_memst(x10, cmll_sub(31, ctx));
  vmovq128_memst(x11, cmll_sub(32, ctx));

#undef KL128
#undef KR128
#undef KA128
#undef KB128

  /* kw2 and kw4 are unused now. */
  load_zero(tmp0);
  vmovq128_memst(tmp0, cmll_sub(1, ctx));
  vmovq128_memst(tmp0, cmll_sub(33, ctx));

  clear_vec_regs();
}

void ASM_FUNC_ATTR_NOINLINE
FUNC_KEY_SETUP(void *key_table, const void *vkey, unsigned int keylen)
{
  const char *key = vkey;

  /* input:
   *   key_table: subkey storage at key_table(CTX)
   *   key_length_bits: output key length as number of bits
   *   key: input key buffer
   *   keylen: key length in bytes
   */

  __m128i x0, x1, x2;

  switch (keylen)
    {
      default:
	return; /* Unsupported key length! */

      case 16:
	vmovdqu128_memld(key, x0);
	camellia_setup128(key_table, x0);
	return;

      case 24:
	vmovdqu128_memld(key, x0);
	vmovq128_amemld((uint64_unaligned_t *)(key + 16), x1);

	x2[0] = -1;
	x2[1] = -1;
	vpxor128(x1, x2, x2);
	vpslldq128(8, x2, x2);
	vpor128(x2, x1, x1);
	break;

      case 32:
	vmovdqu128_memld(key, x0);
	vmovdqu128_memld(key + 16, x1);
	break;
    }

  camellia_setup256(key_table, x0, x1);
}
