From f28d715b2d44e0c1a80fd27546698241a48d4a13 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Mon, 20 Jul 2026 20:56:45 +1000 Subject: [PATCH] AES asm: Add GCM 8-bit tabl, fixes Added assembly to do 8-bit-table GCM_gmult_len. Wired it into aes.c and wired small to use C code. Fixed guards around assembly. --- wolfcrypt/src/aes.c | 54 +- wolfcrypt/src/port/arm/armv8-32-aes-asm.S | 907 +++++++++++++++++++ wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c | 929 ++++++++++++++++++++ wolfcrypt/src/port/arm/armv8-aes-asm.S | 382 ++++++++ wolfcrypt/src/port/arm/armv8-aes-asm.asm | 355 ++++++++ wolfcrypt/src/port/arm/armv8-aes-asm_c.c | 362 ++++++++ wolfcrypt/src/port/arm/armv8-chacha-asm.S | 2 +- wolfcrypt/src/port/arm/armv8-chacha-asm.asm | 2 +- wolfcrypt/src/port/arm/armv8-chacha-asm_c.c | 2 +- wolfcrypt/src/port/arm/armv8-sha512-asm.S | 2 + wolfcrypt/src/port/arm/armv8-sha512-asm.asm | 2 + wolfcrypt/src/port/arm/armv8-sha512-asm_c.c | 2 + wolfcrypt/src/port/arm/thumb2-aes-asm.S | 534 +++++++++++ wolfcrypt/src/port/arm/thumb2-aes-asm_c.c | 561 ++++++++++++ wolfssl/wolfcrypt/aes.h | 10 +- 15 files changed, 4089 insertions(+), 17 deletions(-) diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index 46a89a6ad80..b98b52295f0 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -8142,8 +8142,6 @@ static WC_INLINE void IncrementGcmCounter(byte* inOutCtr) #endif #endif /* !FREESCALE_LTC_AES_GCM */ -#if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) #if defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz) @@ -8297,7 +8295,6 @@ void GenerateM0(Gcm* gcm) } #endif /* GCM_TABLE */ -#endif #if defined(WOLFSSL_AESNI) && defined(USE_INTEL_SPEEDUP) #define HAVE_INTEL_AVX1 @@ -8328,6 +8325,28 @@ void GCM_generate_m0_avx2(const unsigned char *h, unsigned char *m) #endif #endif /* WOLFSSL_AESNI && GCM_TABLE_4BIT && WC_C_DYNAMIC_FALLBACK */ +#if defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \ + !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && defined(HAVE_AESGCM) +/* AES_GCM_set_key_AARCH32 produces H with the bits of each byte reflected - + * the form the PMULL bulk assembly wants. The portable GHASH used for AES-GCM + * streaming needs plain H, so H is kept un-reflected and reflected back only + * around the bulk assembly calls. Reflecting per byte is its own inverse. */ +static WC_INLINE void GcmReflectH(byte* h) +{ + int i; + int j; + for (i = 0; i < WC_AES_BLOCK_SIZE; i++) { + byte b = h[i]; + byte r = 0; + for (j = 0; j < 8; j++) { + r = (byte)((r << 1) | (b & 1)); + b >>= 1; + } + h[i] = r; + } +} +#endif + /* Software AES - GCM SetKey */ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) { @@ -8402,6 +8421,12 @@ int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) AES_GCM_set_key_AARCH32(iv, (byte*)aes->key, aes->gcm.H, aes->rounds); + /* Keep H un-reflected so the portable streaming GHASH can use it (and + * M0); the bulk assembly reflects a copy back per call. */ + GcmReflectH(aes->gcm.H); + #if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) + GenerateM0(&aes->gcm); + #endif #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { AES_GCM_set_key_AARCH64(iv, (byte*)aes->key, aes->gcm.H, @@ -8602,8 +8627,6 @@ void AES_GCM_decrypt_vaes(const unsigned char *in, unsigned char *out, #endif /* WOLFSSL_AESNI */ -#if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \ - defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) #if defined(WOLFSSL_RISCV_SCALAR_CRYPTO_ASM) && defined(HAVE_AESGCM) && \ !defined(WOLFSSL_RISCV_VECTOR_CRYPTO_ASM) /* GHASH using the RISC-V scalar carryless-multiply (Zbc) helper. Vector crypto @@ -8796,7 +8819,9 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, #if defined(WOLFSSL_ARMASM) && (!defined(__aarch64__) || \ defined(WOLFSSL_ARMASM_NO_NEON)) -static void GCM_gmult_len_armasm_C( +/* Unused when the batch GHASH is done in assembly (32-bit ARMv8 crypto), which + * only pulls in the streaming software GMULT. */ +static WC_MAYBE_UNUSED void GCM_gmult_len_armasm_C( byte* x, const byte* h, const unsigned char* a, unsigned long len) { byte Z[AES_BLOCK_SIZE]; @@ -9137,6 +9162,15 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, */ #define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING +#ifdef GCM_GMULT_LEN +/* GHASH one block of data. + * + * @param [in, out] aes AES GCM object. + * @param [in] block Block of AAD or cipher text. + */ +#define GHASH_ONE_BLOCK_SW(aes, block) \ + GCM_GMULT_LEN(&(aes)->gcm, AES_TAG(aes), block, WC_AES_BLOCK_SIZE) +#else /* GHASH one block of data.. * * XOR block into tag and GMULT with H using pre-computed table. @@ -9150,6 +9184,7 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, GMULT(AES_TAG(aes), aes->gcm.M0); \ } \ while (0) +#endif #endif /* WOLFSSL_AESGCM_STREAM */ /* end GCM_TABLE */ #elif defined(GCM_TABLE_4BIT) @@ -10205,7 +10240,6 @@ void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, #endif /* LITTLE_ENDIAN_ORDER */ #endif /* WOLFSSL_AESGCM_STREAM */ #endif /* end GCM_WORD32 */ -#endif #if !defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES) #ifdef WOLFSSL_AESGCM_STREAM @@ -11067,9 +11101,12 @@ int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz, #if defined(WOLFSSL_ARMASM) #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO #if !defined(__aarch64__) + /* Reflect H back to the form the PMULL assembly wants for this call. */ + GcmReflectH(aes->gcm.H); AES_GCM_encrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp, (byte*)aes->reg, aes->rounds); + GcmReflectH(aes->gcm.H); ret = 0; #else if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) { @@ -11873,9 +11910,12 @@ int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz, word32 reg[WC_AES_BLOCK_SIZE / sizeof(word32)]; XMEMCPY(reg, aes->reg, sizeof(reg)); #endif + /* Reflect H back to the form the PMULL assembly wants for this call. */ + GcmReflectH(aes->gcm.H); ret = AES_GCM_decrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp, (byte*)aes->reg, aes->rounds); + GcmReflectH(aes->gcm.H); #ifdef OPENSSL_EXTRA XMEMCPY(aes->reg, reg, sizeof(reg)); #endif diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm.S b/wolfcrypt/src/port/arm/armv8-32-aes-asm.S index 447b8923fc4..39df503977d 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm.S @@ -23540,6 +23540,7 @@ L_AES_CBC_decrypt_end: * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) #ifndef __APPLE__ .text .type L_GCM_gmult_len_r, %object @@ -24248,6 +24249,912 @@ L_GCM_gmult_len_start_block: bne L_GCM_gmult_len_start_block pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} .size GCM_gmult_len,.-GCM_gmult_len +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +#ifndef __APPLE__ + .text + .type L_GCM_gmult_len_r, %object + .size L_GCM_gmult_len_r, 512 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_GCM_gmult_len_r: + .byte 0x00,0x00,0x01,0xc2,0x03,0x84,0x02,0x46 + .byte 0x07,0x08,0x06,0xca,0x04,0x8c,0x05,0x4e + .byte 0x0e,0x10,0x0f,0xd2,0x0d,0x94,0x0c,0x56 + .byte 0x09,0x18,0x08,0xda,0x0a,0x9c,0x0b,0x5e + .byte 0x1c,0x20,0x1d,0xe2,0x1f,0xa4,0x1e,0x66 + .byte 0x1b,0x28,0x1a,0xea,0x18,0xac,0x19,0x6e + .byte 0x12,0x30,0x13,0xf2,0x11,0xb4,0x10,0x76 + .byte 0x15,0x38,0x14,0xfa,0x16,0xbc,0x17,0x7e + .byte 0x38,0x40,0x39,0x82,0x3b,0xc4,0x3a,0x06 + .byte 0x3f,0x48,0x3e,0x8a,0x3c,0xcc,0x3d,0x0e + .byte 0x36,0x50,0x37,0x92,0x35,0xd4,0x34,0x16 + .byte 0x31,0x58,0x30,0x9a,0x32,0xdc,0x33,0x1e + .byte 0x24,0x60,0x25,0xa2,0x27,0xe4,0x26,0x26 + .byte 0x23,0x68,0x22,0xaa,0x20,0xec,0x21,0x2e + .byte 0x2a,0x70,0x2b,0xb2,0x29,0xf4,0x28,0x36 + .byte 0x2d,0x78,0x2c,0xba,0x2e,0xfc,0x2f,0x3e + .byte 0x70,0x80,0x71,0x42,0x73,0x04,0x72,0xc6 + .byte 0x77,0x88,0x76,0x4a,0x74,0x0c,0x75,0xce + .byte 0x7e,0x90,0x7f,0x52,0x7d,0x14,0x7c,0xd6 + .byte 0x79,0x98,0x78,0x5a,0x7a,0x1c,0x7b,0xde + .byte 0x6c,0xa0,0x6d,0x62,0x6f,0x24,0x6e,0xe6 + .byte 0x6b,0xa8,0x6a,0x6a,0x68,0x2c,0x69,0xee + .byte 0x62,0xb0,0x63,0x72,0x61,0x34,0x60,0xf6 + .byte 0x65,0xb8,0x64,0x7a,0x66,0x3c,0x67,0xfe + .byte 0x48,0xc0,0x49,0x02,0x4b,0x44,0x4a,0x86 + .byte 0x4f,0xc8,0x4e,0x0a,0x4c,0x4c,0x4d,0x8e + .byte 0x46,0xd0,0x47,0x12,0x45,0x54,0x44,0x96 + .byte 0x41,0xd8,0x40,0x1a,0x42,0x5c,0x43,0x9e + .byte 0x54,0xe0,0x55,0x22,0x57,0x64,0x56,0xa6 + .byte 0x53,0xe8,0x52,0x2a,0x50,0x6c,0x51,0xae + .byte 0x5a,0xf0,0x5b,0x32,0x59,0x74,0x58,0xb6 + .byte 0x5d,0xf8,0x5c,0x3a,0x5e,0x7c,0x5f,0xbe + .byte 0xe1,0x00,0xe0,0xc2,0xe2,0x84,0xe3,0x46 + .byte 0xe6,0x08,0xe7,0xca,0xe5,0x8c,0xe4,0x4e + .byte 0xef,0x10,0xee,0xd2,0xec,0x94,0xed,0x56 + .byte 0xe8,0x18,0xe9,0xda,0xeb,0x9c,0xea,0x5e + .byte 0xfd,0x20,0xfc,0xe2,0xfe,0xa4,0xff,0x66 + .byte 0xfa,0x28,0xfb,0xea,0xf9,0xac,0xf8,0x6e + .byte 0xf3,0x30,0xf2,0xf2,0xf0,0xb4,0xf1,0x76 + .byte 0xf4,0x38,0xf5,0xfa,0xf7,0xbc,0xf6,0x7e + .byte 0xd9,0x40,0xd8,0x82,0xda,0xc4,0xdb,0x06 + .byte 0xde,0x48,0xdf,0x8a,0xdd,0xcc,0xdc,0x0e + .byte 0xd7,0x50,0xd6,0x92,0xd4,0xd4,0xd5,0x16 + .byte 0xd0,0x58,0xd1,0x9a,0xd3,0xdc,0xd2,0x1e + .byte 0xc5,0x60,0xc4,0xa2,0xc6,0xe4,0xc7,0x26 + .byte 0xc2,0x68,0xc3,0xaa,0xc1,0xec,0xc0,0x2e + .byte 0xcb,0x70,0xca,0xb2,0xc8,0xf4,0xc9,0x36 + .byte 0xcc,0x78,0xcd,0xba,0xcf,0xfc,0xce,0x3e + .byte 0x91,0x80,0x90,0x42,0x92,0x04,0x93,0xc6 + .byte 0x96,0x88,0x97,0x4a,0x95,0x0c,0x94,0xce + .byte 0x9f,0x90,0x9e,0x52,0x9c,0x14,0x9d,0xd6 + .byte 0x98,0x98,0x99,0x5a,0x9b,0x1c,0x9a,0xde + .byte 0x8d,0xa0,0x8c,0x62,0x8e,0x24,0x8f,0xe6 + .byte 0x8a,0xa8,0x8b,0x6a,0x89,0x2c,0x88,0xee + .byte 0x83,0xb0,0x82,0x72,0x80,0x34,0x81,0xf6 + .byte 0x84,0xb8,0x85,0x7a,0x87,0x3c,0x86,0xfe + .byte 0xa9,0xc0,0xa8,0x02,0xaa,0x44,0xab,0x86 + .byte 0xae,0xc8,0xaf,0x0a,0xad,0x4c,0xac,0x8e + .byte 0xa7,0xd0,0xa6,0x12,0xa4,0x54,0xa5,0x96 + .byte 0xa0,0xd8,0xa1,0x1a,0xa3,0x5c,0xa2,0x9e + .byte 0xb5,0xe0,0xb4,0x22,0xb6,0x64,0xb7,0xa6 + .byte 0xb2,0xe8,0xb3,0x2a,0xb1,0x6c,0xb0,0xae + .byte 0xbb,0xf0,0xba,0x32,0xb8,0x74,0xb9,0xb6 + .byte 0xbc,0xf8,0xbd,0x3a,0xbf,0x7c,0xbe,0xbe + .text + .align 4 + .globl GCM_gmult_len + .type GCM_gmult_len, %function +GCM_gmult_len: + push {r4, r5, r6, r7, r8, r9, r10, r11, lr} + adr lr, L_GCM_gmult_len_r +L_GCM_gmult_len_start_block: + ldm r0, {r4, r5, r6, r7} + ldm r2, {r8, r9, r10, r11} + eor r4, r4, r8 + eor r5, r5, r9 + eor r6, r6, r10 + eor r7, r7, r11 + stm r0, {r4, r5, r6, r7} + push {r3} + mov r8, #0 + mov r9, #0 + mov r10, #0 + mov r11, #0 + # Byte 15 + add r12, r0, #15 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 14 + add r12, r0, #14 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 13 + add r12, r0, #13 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 12 + add r12, r0, #12 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 11 + add r12, r0, #11 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 10 + add r12, r0, #10 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 9 + add r12, r0, #9 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 8 + add r12, r0, #8 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 7 + add r12, r0, #7 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 6 + add r12, r0, #6 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 5 + add r12, r0, #5 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 4 + add r12, r0, #4 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 3 + add r12, r0, #3 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 2 + add r12, r0, #2 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 1 + add r12, r0, #1 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 + and r4, r11, #0xff + lsr r11, r11, #8 + orr r11, r11, r10, lsl #24 + lsr r10, r10, #8 + orr r10, r10, r9, lsl #24 + lsr r9, r9, #8 + orr r9, r9, r8, lsl #24 + lsr r8, r8, #8 + add r4, lr, r4, lsl #1 + ldrb r5, [r4], #1 + ldrb r6, [r4] + orr r8, r8, r5, lsl #24 + eor r8, r8, r6, lsl #16 + # Byte 0 + add r12, r0, #0 + ldrb r12, [r12] + add r12, r1, r12, lsl #4 + ldm r12, {r4, r5, r6, r7} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r4, r4 + eor r3, r4, r4, ror #16 + bic r3, r3, #0xff0000 + ror r4, r4, #8 + eor r4, r4, r3, lsr #8 + # REV r5, r5 + eor r3, r5, r5, ror #16 + bic r3, r3, #0xff0000 + ror r5, r5, #8 + eor r5, r5, r3, lsr #8 + # REV r6, r6 + eor r3, r6, r6, ror #16 + bic r3, r3, #0xff0000 + ror r6, r6, #8 + eor r6, r6, r3, lsr #8 + # REV r7, r7 + eor r3, r7, r7, ror #16 + bic r3, r3, #0xff0000 + ror r7, r7, #8 + eor r7, r7, r3, lsr #8 +#else + rev r4, r4 + rev r5, r5 + rev r6, r6 + rev r7, r7 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + eor r8, r8, r4 + eor r9, r9, r5 + eor r10, r10, r6 + eor r11, r11, r7 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + # REV r8, r8 + eor r3, r8, r8, ror #16 + bic r3, r3, #0xff0000 + ror r8, r8, #8 + eor r8, r8, r3, lsr #8 + # REV r9, r9 + eor r3, r9, r9, ror #16 + bic r3, r3, #0xff0000 + ror r9, r9, #8 + eor r9, r9, r3, lsr #8 + # REV r10, r10 + eor r3, r10, r10, ror #16 + bic r3, r3, #0xff0000 + ror r10, r10, #8 + eor r10, r10, r3, lsr #8 + # REV r11, r11 + eor r3, r11, r11, ror #16 + bic r3, r3, #0xff0000 + ror r11, r11, #8 + eor r11, r11, r3, lsr #8 +#else + rev r8, r8 + rev r9, r9 + rev r10, r10 + rev r11, r11 +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + stm r0, {r8, r9, r10, r11} + pop {r3} + subs r3, r3, #16 + add r2, r2, #16 + bne L_GCM_gmult_len_start_block + pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} + .size GCM_gmult_len,.-GCM_gmult_len +#endif /* GCM_TABLE */ #ifndef __APPLE__ .text .type L_AES_ARM32_te_gcm, %object diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c index a046e48562d..3ec542a326d 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c @@ -24449,6 +24449,7 @@ WC_OMIT_FRAME_POINTER void AES_CBC_decrypt(const unsigned char* in, * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) XALIGNED(8) static const word32 L_GCM_gmult_len_r[] = { 0x00000000, 0x1c200000, 0x38400000, 0x24600000, 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, @@ -25179,6 +25180,934 @@ WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, ); } +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +XALIGNED(4) static const word8 L_GCM_gmult_len_r[] = { + 0x00, 0x00, 0x01, 0xc2, 0x03, 0x84, 0x02, 0x46, + 0x07, 0x08, 0x06, 0xca, 0x04, 0x8c, 0x05, 0x4e, + 0x0e, 0x10, 0x0f, 0xd2, 0x0d, 0x94, 0x0c, 0x56, + 0x09, 0x18, 0x08, 0xda, 0x0a, 0x9c, 0x0b, 0x5e, + 0x1c, 0x20, 0x1d, 0xe2, 0x1f, 0xa4, 0x1e, 0x66, + 0x1b, 0x28, 0x1a, 0xea, 0x18, 0xac, 0x19, 0x6e, + 0x12, 0x30, 0x13, 0xf2, 0x11, 0xb4, 0x10, 0x76, + 0x15, 0x38, 0x14, 0xfa, 0x16, 0xbc, 0x17, 0x7e, + 0x38, 0x40, 0x39, 0x82, 0x3b, 0xc4, 0x3a, 0x06, + 0x3f, 0x48, 0x3e, 0x8a, 0x3c, 0xcc, 0x3d, 0x0e, + 0x36, 0x50, 0x37, 0x92, 0x35, 0xd4, 0x34, 0x16, + 0x31, 0x58, 0x30, 0x9a, 0x32, 0xdc, 0x33, 0x1e, + 0x24, 0x60, 0x25, 0xa2, 0x27, 0xe4, 0x26, 0x26, + 0x23, 0x68, 0x22, 0xaa, 0x20, 0xec, 0x21, 0x2e, + 0x2a, 0x70, 0x2b, 0xb2, 0x29, 0xf4, 0x28, 0x36, + 0x2d, 0x78, 0x2c, 0xba, 0x2e, 0xfc, 0x2f, 0x3e, + 0x70, 0x80, 0x71, 0x42, 0x73, 0x04, 0x72, 0xc6, + 0x77, 0x88, 0x76, 0x4a, 0x74, 0x0c, 0x75, 0xce, + 0x7e, 0x90, 0x7f, 0x52, 0x7d, 0x14, 0x7c, 0xd6, + 0x79, 0x98, 0x78, 0x5a, 0x7a, 0x1c, 0x7b, 0xde, + 0x6c, 0xa0, 0x6d, 0x62, 0x6f, 0x24, 0x6e, 0xe6, + 0x6b, 0xa8, 0x6a, 0x6a, 0x68, 0x2c, 0x69, 0xee, + 0x62, 0xb0, 0x63, 0x72, 0x61, 0x34, 0x60, 0xf6, + 0x65, 0xb8, 0x64, 0x7a, 0x66, 0x3c, 0x67, 0xfe, + 0x48, 0xc0, 0x49, 0x02, 0x4b, 0x44, 0x4a, 0x86, + 0x4f, 0xc8, 0x4e, 0x0a, 0x4c, 0x4c, 0x4d, 0x8e, + 0x46, 0xd0, 0x47, 0x12, 0x45, 0x54, 0x44, 0x96, + 0x41, 0xd8, 0x40, 0x1a, 0x42, 0x5c, 0x43, 0x9e, + 0x54, 0xe0, 0x55, 0x22, 0x57, 0x64, 0x56, 0xa6, + 0x53, 0xe8, 0x52, 0x2a, 0x50, 0x6c, 0x51, 0xae, + 0x5a, 0xf0, 0x5b, 0x32, 0x59, 0x74, 0x58, 0xb6, + 0x5d, 0xf8, 0x5c, 0x3a, 0x5e, 0x7c, 0x5f, 0xbe, + 0xe1, 0x00, 0xe0, 0xc2, 0xe2, 0x84, 0xe3, 0x46, + 0xe6, 0x08, 0xe7, 0xca, 0xe5, 0x8c, 0xe4, 0x4e, + 0xef, 0x10, 0xee, 0xd2, 0xec, 0x94, 0xed, 0x56, + 0xe8, 0x18, 0xe9, 0xda, 0xeb, 0x9c, 0xea, 0x5e, + 0xfd, 0x20, 0xfc, 0xe2, 0xfe, 0xa4, 0xff, 0x66, + 0xfa, 0x28, 0xfb, 0xea, 0xf9, 0xac, 0xf8, 0x6e, + 0xf3, 0x30, 0xf2, 0xf2, 0xf0, 0xb4, 0xf1, 0x76, + 0xf4, 0x38, 0xf5, 0xfa, 0xf7, 0xbc, 0xf6, 0x7e, + 0xd9, 0x40, 0xd8, 0x82, 0xda, 0xc4, 0xdb, 0x06, + 0xde, 0x48, 0xdf, 0x8a, 0xdd, 0xcc, 0xdc, 0x0e, + 0xd7, 0x50, 0xd6, 0x92, 0xd4, 0xd4, 0xd5, 0x16, + 0xd0, 0x58, 0xd1, 0x9a, 0xd3, 0xdc, 0xd2, 0x1e, + 0xc5, 0x60, 0xc4, 0xa2, 0xc6, 0xe4, 0xc7, 0x26, + 0xc2, 0x68, 0xc3, 0xaa, 0xc1, 0xec, 0xc0, 0x2e, + 0xcb, 0x70, 0xca, 0xb2, 0xc8, 0xf4, 0xc9, 0x36, + 0xcc, 0x78, 0xcd, 0xba, 0xcf, 0xfc, 0xce, 0x3e, + 0x91, 0x80, 0x90, 0x42, 0x92, 0x04, 0x93, 0xc6, + 0x96, 0x88, 0x97, 0x4a, 0x95, 0x0c, 0x94, 0xce, + 0x9f, 0x90, 0x9e, 0x52, 0x9c, 0x14, 0x9d, 0xd6, + 0x98, 0x98, 0x99, 0x5a, 0x9b, 0x1c, 0x9a, 0xde, + 0x8d, 0xa0, 0x8c, 0x62, 0x8e, 0x24, 0x8f, 0xe6, + 0x8a, 0xa8, 0x8b, 0x6a, 0x89, 0x2c, 0x88, 0xee, + 0x83, 0xb0, 0x82, 0x72, 0x80, 0x34, 0x81, 0xf6, + 0x84, 0xb8, 0x85, 0x7a, 0x87, 0x3c, 0x86, 0xfe, + 0xa9, 0xc0, 0xa8, 0x02, 0xaa, 0x44, 0xab, 0x86, + 0xae, 0xc8, 0xaf, 0x0a, 0xad, 0x4c, 0xac, 0x8e, + 0xa7, 0xd0, 0xa6, 0x12, 0xa4, 0x54, 0xa5, 0x96, + 0xa0, 0xd8, 0xa1, 0x1a, 0xa3, 0x5c, 0xa2, 0x9e, + 0xb5, 0xe0, 0xb4, 0x22, 0xb6, 0x64, 0xb7, 0xa6, + 0xb2, 0xe8, 0xb3, 0x2a, 0xb1, 0x6c, 0xb0, 0xae, + 0xbb, 0xf0, 0xba, 0x32, 0xb8, 0x74, 0xb9, 0xb6, + 0xbc, 0xf8, 0xbd, 0x3a, 0xbf, 0x7c, 0xbe, 0xbe, +}; + +void GCM_gmult_len(unsigned char* x_p, const unsigned char** m_p, + const unsigned char* data_p, unsigned long len_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x_p, + const unsigned char** m_p, const unsigned char* data_p, unsigned long len_p) +#else +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, + const unsigned char** m, const unsigned char* data, unsigned long len) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register unsigned char* x __asm__ ("r0") = (unsigned char*)x_p; + register const unsigned char** m __asm__ ("r1") = + (const unsigned char**)m_p; + register const unsigned char* data __asm__ ("r2") = + (const unsigned char*)data_p; + register unsigned long len __asm__ ("r3") = (unsigned long)len_p; + register word8* L_GCM_gmult_len_r_c __asm__ ("r12") = + (word8*)&L_GCM_gmult_len_r; +#else + register word8* L_GCM_gmult_len_r_c = (word8*)&L_GCM_gmult_len_r; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_GCM_gmult_len_r]}\n\t" + "mov lr, %[L_GCM_gmult_len_r]\n\t" + "\n" + "L_GCM_gmult_len_start_block_%=:\n\t" + "ldm %[x], {r4, r5, r6, r7}\n\t" + "ldm %[data], {r8, r9, r10, r11}\n\t" + "eor r4, r4, r8\n\t" + "eor r5, r5, r9\n\t" + "eor r6, r6, r10\n\t" + "eor r7, r7, r11\n\t" + "stm %[x], {r4, r5, r6, r7}\n\t" + "push {r3}\n\t" + "mov r8, #0\n\t" + "mov r9, #0\n\t" + "mov r10, #0\n\t" + "mov r11, #0\n\t" + /* Byte 15 */ + "add r12, %[x], #15\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 14 */ + "add r12, %[x], #14\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 13 */ + "add r12, %[x], #13\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 12 */ + "add r12, %[x], #12\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 11 */ + "add r12, %[x], #11\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 10 */ + "add r12, %[x], #10\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 9 */ + "add r12, %[x], #9\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 8 */ + "add r12, %[x], #8\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 7 */ + "add r12, %[x], #7\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 6 */ + "add r12, %[x], #6\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 5 */ + "add r12, %[x], #5\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 4 */ + "add r12, %[x], #4\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 3 */ + "add r12, %[x], #3\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 2 */ + "add r12, %[x], #2\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 1 */ + "add r12, %[x], #1\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" + "and r4, r11, #0xff\n\t" + "lsr r11, r11, #8\n\t" + "orr r11, r11, r10, lsl #24\n\t" + "lsr r10, r10, #8\n\t" + "orr r10, r10, r9, lsl #24\n\t" + "lsr r9, r9, #8\n\t" + "orr r9, r9, r8, lsl #24\n\t" + "lsr r8, r8, #8\n\t" + "add r4, lr, r4, lsl #1\n\t" + "ldrb r5, [r4], #1\n\t" + "ldrb r6, [r4]\n\t" + "orr r8, r8, r5, lsl #24\n\t" + "eor r8, r8, r6, lsl #16\n\t" + /* Byte 0 */ + "add r12, %[x], #0\n\t" + "ldrb r12, [r12]\n\t" + "add r12, %[m], r12, lsl #4\n\t" + "ldm r12, {r4, r5, r6, r7}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r4, r4 */ + "eor %[len], r4, r4, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r4, r4, #8\n\t" + "eor r4, r4, %[len], lsr #8\n\t" + /* REV r5, r5 */ + "eor %[len], r5, r5, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r5, r5, #8\n\t" + "eor r5, r5, %[len], lsr #8\n\t" + /* REV r6, r6 */ + "eor %[len], r6, r6, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r6, r6, #8\n\t" + "eor r6, r6, %[len], lsr #8\n\t" + /* REV r7, r7 */ + "eor %[len], r7, r7, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r7, r7, #8\n\t" + "eor r7, r7, %[len], lsr #8\n\t" +#else + "rev r4, r4\n\t" + "rev r5, r5\n\t" + "rev r6, r6\n\t" + "rev r7, r7\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "eor r8, r8, r4\n\t" + "eor r9, r9, r5\n\t" + "eor r10, r10, r6\n\t" + "eor r11, r11, r7\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) + /* REV r8, r8 */ + "eor %[len], r8, r8, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r8, r8, #8\n\t" + "eor r8, r8, %[len], lsr #8\n\t" + /* REV r9, r9 */ + "eor %[len], r9, r9, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r9, r9, #8\n\t" + "eor r9, r9, %[len], lsr #8\n\t" + /* REV r10, r10 */ + "eor %[len], r10, r10, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r10, r10, #8\n\t" + "eor r10, r10, %[len], lsr #8\n\t" + /* REV r11, r11 */ + "eor %[len], r11, r11, ror #16\n\t" + "bic %[len], %[len], #0xff0000\n\t" + "ror r11, r11, #8\n\t" + "eor r11, r11, %[len], lsr #8\n\t" +#else + "rev r8, r8\n\t" + "rev r9, r9\n\t" + "rev r10, r10\n\t" + "rev r11, r11\n\t" +#endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ + "stm %[x], {r8, r9, r10, r11}\n\t" + "pop {r3}\n\t" + "subs %[len], %[len], #16\n\t" + "add %[data], %[data], #16\n\t" + "bne L_GCM_gmult_len_start_block_%=\n\t" + "pop {%[L_GCM_gmult_len_r]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [x] "+r" (x), [m] "+r" (m), [data] "+r" (data), [len] "+r" (len), + [L_GCM_gmult_len_r] "+r" (L_GCM_gmult_len_r_c) + : +#else + : + : [x] "r" (x), [m] "r" (m), [data] "r" (data), [len] "r" (len), + [L_GCM_gmult_len_r] "r" (L_GCM_gmult_len_r_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "r11" + ); +} + +#endif /* GCM_TABLE */ static const word32* L_AES_ARM32_te_gcm = L_AES_ARM32_te_data; void AES_GCM_encrypt(const unsigned char* in_p, unsigned char* out_p, unsigned long len_p, const unsigned char* ks_p, int nr_p, diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.S b/wolfcrypt/src/port/arm/armv8-aes-asm.S index 0e554fd777a..a734b8745a8 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.S @@ -56107,6 +56107,7 @@ L_AES_CBC_decrypt_end_dec: #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER || HAVE_AES_CBC || HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) #ifndef __APPLE__ .text .section .rodata @@ -56553,6 +56554,387 @@ L_GCM_gmult_len_start_block: #ifndef __APPLE__ .size GCM_gmult_len,.-GCM_gmult_len #endif /* __APPLE__ */ +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +#ifndef __APPLE__ + .text + .section .rodata + .type L_GCM_gmult_len_r, %object + .size L_GCM_gmult_len_r, 512 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_GCM_gmult_len_r: + .byte 0x00,0x00,0x01,0xc2,0x03,0x84,0x02,0x46 + .byte 0x07,0x08,0x06,0xca,0x04,0x8c,0x05,0x4e + .byte 0x0e,0x10,0x0f,0xd2,0x0d,0x94,0x0c,0x56 + .byte 0x09,0x18,0x08,0xda,0x0a,0x9c,0x0b,0x5e + .byte 0x1c,0x20,0x1d,0xe2,0x1f,0xa4,0x1e,0x66 + .byte 0x1b,0x28,0x1a,0xea,0x18,0xac,0x19,0x6e + .byte 0x12,0x30,0x13,0xf2,0x11,0xb4,0x10,0x76 + .byte 0x15,0x38,0x14,0xfa,0x16,0xbc,0x17,0x7e + .byte 0x38,0x40,0x39,0x82,0x3b,0xc4,0x3a,0x06 + .byte 0x3f,0x48,0x3e,0x8a,0x3c,0xcc,0x3d,0x0e + .byte 0x36,0x50,0x37,0x92,0x35,0xd4,0x34,0x16 + .byte 0x31,0x58,0x30,0x9a,0x32,0xdc,0x33,0x1e + .byte 0x24,0x60,0x25,0xa2,0x27,0xe4,0x26,0x26 + .byte 0x23,0x68,0x22,0xaa,0x20,0xec,0x21,0x2e + .byte 0x2a,0x70,0x2b,0xb2,0x29,0xf4,0x28,0x36 + .byte 0x2d,0x78,0x2c,0xba,0x2e,0xfc,0x2f,0x3e + .byte 0x70,0x80,0x71,0x42,0x73,0x04,0x72,0xc6 + .byte 0x77,0x88,0x76,0x4a,0x74,0x0c,0x75,0xce + .byte 0x7e,0x90,0x7f,0x52,0x7d,0x14,0x7c,0xd6 + .byte 0x79,0x98,0x78,0x5a,0x7a,0x1c,0x7b,0xde + .byte 0x6c,0xa0,0x6d,0x62,0x6f,0x24,0x6e,0xe6 + .byte 0x6b,0xa8,0x6a,0x6a,0x68,0x2c,0x69,0xee + .byte 0x62,0xb0,0x63,0x72,0x61,0x34,0x60,0xf6 + .byte 0x65,0xb8,0x64,0x7a,0x66,0x3c,0x67,0xfe + .byte 0x48,0xc0,0x49,0x02,0x4b,0x44,0x4a,0x86 + .byte 0x4f,0xc8,0x4e,0x0a,0x4c,0x4c,0x4d,0x8e + .byte 0x46,0xd0,0x47,0x12,0x45,0x54,0x44,0x96 + .byte 0x41,0xd8,0x40,0x1a,0x42,0x5c,0x43,0x9e + .byte 0x54,0xe0,0x55,0x22,0x57,0x64,0x56,0xa6 + .byte 0x53,0xe8,0x52,0x2a,0x50,0x6c,0x51,0xae + .byte 0x5a,0xf0,0x5b,0x32,0x59,0x74,0x58,0xb6 + .byte 0x5d,0xf8,0x5c,0x3a,0x5e,0x7c,0x5f,0xbe + .byte 0xe1,0x00,0xe0,0xc2,0xe2,0x84,0xe3,0x46 + .byte 0xe6,0x08,0xe7,0xca,0xe5,0x8c,0xe4,0x4e + .byte 0xef,0x10,0xee,0xd2,0xec,0x94,0xed,0x56 + .byte 0xe8,0x18,0xe9,0xda,0xeb,0x9c,0xea,0x5e + .byte 0xfd,0x20,0xfc,0xe2,0xfe,0xa4,0xff,0x66 + .byte 0xfa,0x28,0xfb,0xea,0xf9,0xac,0xf8,0x6e + .byte 0xf3,0x30,0xf2,0xf2,0xf0,0xb4,0xf1,0x76 + .byte 0xf4,0x38,0xf5,0xfa,0xf7,0xbc,0xf6,0x7e + .byte 0xd9,0x40,0xd8,0x82,0xda,0xc4,0xdb,0x06 + .byte 0xde,0x48,0xdf,0x8a,0xdd,0xcc,0xdc,0x0e + .byte 0xd7,0x50,0xd6,0x92,0xd4,0xd4,0xd5,0x16 + .byte 0xd0,0x58,0xd1,0x9a,0xd3,0xdc,0xd2,0x1e + .byte 0xc5,0x60,0xc4,0xa2,0xc6,0xe4,0xc7,0x26 + .byte 0xc2,0x68,0xc3,0xaa,0xc1,0xec,0xc0,0x2e + .byte 0xcb,0x70,0xca,0xb2,0xc8,0xf4,0xc9,0x36 + .byte 0xcc,0x78,0xcd,0xba,0xcf,0xfc,0xce,0x3e + .byte 0x91,0x80,0x90,0x42,0x92,0x04,0x93,0xc6 + .byte 0x96,0x88,0x97,0x4a,0x95,0x0c,0x94,0xce + .byte 0x9f,0x90,0x9e,0x52,0x9c,0x14,0x9d,0xd6 + .byte 0x98,0x98,0x99,0x5a,0x9b,0x1c,0x9a,0xde + .byte 0x8d,0xa0,0x8c,0x62,0x8e,0x24,0x8f,0xe6 + .byte 0x8a,0xa8,0x8b,0x6a,0x89,0x2c,0x88,0xee + .byte 0x83,0xb0,0x82,0x72,0x80,0x34,0x81,0xf6 + .byte 0x84,0xb8,0x85,0x7a,0x87,0x3c,0x86,0xfe + .byte 0xa9,0xc0,0xa8,0x02,0xaa,0x44,0xab,0x86 + .byte 0xae,0xc8,0xaf,0x0a,0xad,0x4c,0xac,0x8e + .byte 0xa7,0xd0,0xa6,0x12,0xa4,0x54,0xa5,0x96 + .byte 0xa0,0xd8,0xa1,0x1a,0xa3,0x5c,0xa2,0x9e + .byte 0xb5,0xe0,0xb4,0x22,0xb6,0x64,0xb7,0xa6 + .byte 0xb2,0xe8,0xb3,0x2a,0xb1,0x6c,0xb0,0xae + .byte 0xbb,0xf0,0xba,0x32,0xb8,0x74,0xb9,0xb6 + .byte 0xbc,0xf8,0xbd,0x3a,0xbf,0x7c,0xbe,0xbe +#ifndef __APPLE__ +.text +.globl GCM_gmult_len +.type GCM_gmult_len,@function +.align 2 +GCM_gmult_len: +#else +.section __TEXT,__text +.globl _GCM_gmult_len +.p2align 2 +_GCM_gmult_len: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x10, L_GCM_gmult_len_r + add x10, x10, :lo12:L_GCM_gmult_len_r +#else + adrp x10, L_GCM_gmult_len_r@PAGE + add x10, x10, L_GCM_gmult_len_r@PAGEOFF +#endif /* __APPLE__ */ +L_GCM_gmult_len_start_block: + ldp x6, x7, [x0] + ldp x8, x9, [x2] + eor x6, x6, x8 + eor x7, x7, x9 + mov x4, #0 + mov x5, #0 + # Byte 15 + ubfx x12, x7, #56, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 14 + ubfx x12, x7, #48, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 13 + ubfx x12, x7, #40, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 12 + ubfx x12, x7, #32, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 11 + ubfx x12, x7, #24, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 10 + ubfx x12, x7, #16, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 9 + ubfx x12, x7, #8, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 8 + ubfx x12, x7, #0, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 7 + ubfx x12, x6, #56, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 6 + ubfx x12, x6, #48, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 5 + ubfx x12, x6, #40, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 4 + ubfx x12, x6, #32, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 3 + ubfx x12, x6, #24, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 2 + ubfx x12, x6, #16, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 1 + ubfx x12, x6, #8, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + # Byte 0 + ubfx x12, x6, #0, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + rev x4, x4 + rev x5, x5 + stp x4, x5, [x0] + subs x3, x3, #16 + add x2, x2, #16 + bne L_GCM_gmult_len_start_block + ret +#ifndef __APPLE__ + .size GCM_gmult_len,.-GCM_gmult_len +#endif /* __APPLE__ */ +#endif /* GCM_TABLE */ #ifndef __APPLE__ .text .globl AES_GCM_encrypt diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.asm b/wolfcrypt/src/port/arm/armv8-aes-asm.asm index 1416b02bf50..9bbd6ef9b62 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.asm @@ -55275,6 +55275,7 @@ L_AES_CBC_decrypt_end_dec ENDIF ENDIF IF :DEF:HAVE_AESGCM + IF :LNOT::DEF:GCM_SMALL :LAND: :LNOT::DEF:GCM_TABLE AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_GCM_gmult_len_r @@ -55694,6 +55695,360 @@ L_GCM_gmult_len_start_block bne L_GCM_gmult_len_start_block ret ENDP + ENDIF + IF :DEF:GCM_TABLE + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_GCM_gmult_len_r + DCB 0x00, 0x00, 0x01, 0xc2, 0x03, 0x84, 0x02, 0x46 + DCB 0x07, 0x08, 0x06, 0xca, 0x04, 0x8c, 0x05, 0x4e + DCB 0x0e, 0x10, 0x0f, 0xd2, 0x0d, 0x94, 0x0c, 0x56 + DCB 0x09, 0x18, 0x08, 0xda, 0x0a, 0x9c, 0x0b, 0x5e + DCB 0x1c, 0x20, 0x1d, 0xe2, 0x1f, 0xa4, 0x1e, 0x66 + DCB 0x1b, 0x28, 0x1a, 0xea, 0x18, 0xac, 0x19, 0x6e + DCB 0x12, 0x30, 0x13, 0xf2, 0x11, 0xb4, 0x10, 0x76 + DCB 0x15, 0x38, 0x14, 0xfa, 0x16, 0xbc, 0x17, 0x7e + DCB 0x38, 0x40, 0x39, 0x82, 0x3b, 0xc4, 0x3a, 0x06 + DCB 0x3f, 0x48, 0x3e, 0x8a, 0x3c, 0xcc, 0x3d, 0x0e + DCB 0x36, 0x50, 0x37, 0x92, 0x35, 0xd4, 0x34, 0x16 + DCB 0x31, 0x58, 0x30, 0x9a, 0x32, 0xdc, 0x33, 0x1e + DCB 0x24, 0x60, 0x25, 0xa2, 0x27, 0xe4, 0x26, 0x26 + DCB 0x23, 0x68, 0x22, 0xaa, 0x20, 0xec, 0x21, 0x2e + DCB 0x2a, 0x70, 0x2b, 0xb2, 0x29, 0xf4, 0x28, 0x36 + DCB 0x2d, 0x78, 0x2c, 0xba, 0x2e, 0xfc, 0x2f, 0x3e + DCB 0x70, 0x80, 0x71, 0x42, 0x73, 0x04, 0x72, 0xc6 + DCB 0x77, 0x88, 0x76, 0x4a, 0x74, 0x0c, 0x75, 0xce + DCB 0x7e, 0x90, 0x7f, 0x52, 0x7d, 0x14, 0x7c, 0xd6 + DCB 0x79, 0x98, 0x78, 0x5a, 0x7a, 0x1c, 0x7b, 0xde + DCB 0x6c, 0xa0, 0x6d, 0x62, 0x6f, 0x24, 0x6e, 0xe6 + DCB 0x6b, 0xa8, 0x6a, 0x6a, 0x68, 0x2c, 0x69, 0xee + DCB 0x62, 0xb0, 0x63, 0x72, 0x61, 0x34, 0x60, 0xf6 + DCB 0x65, 0xb8, 0x64, 0x7a, 0x66, 0x3c, 0x67, 0xfe + DCB 0x48, 0xc0, 0x49, 0x02, 0x4b, 0x44, 0x4a, 0x86 + DCB 0x4f, 0xc8, 0x4e, 0x0a, 0x4c, 0x4c, 0x4d, 0x8e + DCB 0x46, 0xd0, 0x47, 0x12, 0x45, 0x54, 0x44, 0x96 + DCB 0x41, 0xd8, 0x40, 0x1a, 0x42, 0x5c, 0x43, 0x9e + DCB 0x54, 0xe0, 0x55, 0x22, 0x57, 0x64, 0x56, 0xa6 + DCB 0x53, 0xe8, 0x52, 0x2a, 0x50, 0x6c, 0x51, 0xae + DCB 0x5a, 0xf0, 0x5b, 0x32, 0x59, 0x74, 0x58, 0xb6 + DCB 0x5d, 0xf8, 0x5c, 0x3a, 0x5e, 0x7c, 0x5f, 0xbe + DCB 0xe1, 0x00, 0xe0, 0xc2, 0xe2, 0x84, 0xe3, 0x46 + DCB 0xe6, 0x08, 0xe7, 0xca, 0xe5, 0x8c, 0xe4, 0x4e + DCB 0xef, 0x10, 0xee, 0xd2, 0xec, 0x94, 0xed, 0x56 + DCB 0xe8, 0x18, 0xe9, 0xda, 0xeb, 0x9c, 0xea, 0x5e + DCB 0xfd, 0x20, 0xfc, 0xe2, 0xfe, 0xa4, 0xff, 0x66 + DCB 0xfa, 0x28, 0xfb, 0xea, 0xf9, 0xac, 0xf8, 0x6e + DCB 0xf3, 0x30, 0xf2, 0xf2, 0xf0, 0xb4, 0xf1, 0x76 + DCB 0xf4, 0x38, 0xf5, 0xfa, 0xf7, 0xbc, 0xf6, 0x7e + DCB 0xd9, 0x40, 0xd8, 0x82, 0xda, 0xc4, 0xdb, 0x06 + DCB 0xde, 0x48, 0xdf, 0x8a, 0xdd, 0xcc, 0xdc, 0x0e + DCB 0xd7, 0x50, 0xd6, 0x92, 0xd4, 0xd4, 0xd5, 0x16 + DCB 0xd0, 0x58, 0xd1, 0x9a, 0xd3, 0xdc, 0xd2, 0x1e + DCB 0xc5, 0x60, 0xc4, 0xa2, 0xc6, 0xe4, 0xc7, 0x26 + DCB 0xc2, 0x68, 0xc3, 0xaa, 0xc1, 0xec, 0xc0, 0x2e + DCB 0xcb, 0x70, 0xca, 0xb2, 0xc8, 0xf4, 0xc9, 0x36 + DCB 0xcc, 0x78, 0xcd, 0xba, 0xcf, 0xfc, 0xce, 0x3e + DCB 0x91, 0x80, 0x90, 0x42, 0x92, 0x04, 0x93, 0xc6 + DCB 0x96, 0x88, 0x97, 0x4a, 0x95, 0x0c, 0x94, 0xce + DCB 0x9f, 0x90, 0x9e, 0x52, 0x9c, 0x14, 0x9d, 0xd6 + DCB 0x98, 0x98, 0x99, 0x5a, 0x9b, 0x1c, 0x9a, 0xde + DCB 0x8d, 0xa0, 0x8c, 0x62, 0x8e, 0x24, 0x8f, 0xe6 + DCB 0x8a, 0xa8, 0x8b, 0x6a, 0x89, 0x2c, 0x88, 0xee + DCB 0x83, 0xb0, 0x82, 0x72, 0x80, 0x34, 0x81, 0xf6 + DCB 0x84, 0xb8, 0x85, 0x7a, 0x87, 0x3c, 0x86, 0xfe + DCB 0xa9, 0xc0, 0xa8, 0x02, 0xaa, 0x44, 0xab, 0x86 + DCB 0xae, 0xc8, 0xaf, 0x0a, 0xad, 0x4c, 0xac, 0x8e + DCB 0xa7, 0xd0, 0xa6, 0x12, 0xa4, 0x54, 0xa5, 0x96 + DCB 0xa0, 0xd8, 0xa1, 0x1a, 0xa3, 0x5c, 0xa2, 0x9e + DCB 0xb5, 0xe0, 0xb4, 0x22, 0xb6, 0x64, 0xb7, 0xa6 + DCB 0xb2, 0xe8, 0xb3, 0x2a, 0xb1, 0x6c, 0xb0, 0xae + DCB 0xbb, 0xf0, 0xba, 0x32, 0xb8, 0x74, 0xb9, 0xb6 + DCB 0xbc, 0xf8, 0xbd, 0x3a, 0xbf, 0x7c, 0xbe, 0xbe + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT GCM_gmult_len +GCM_gmult_len PROC + adrp x10, L_GCM_gmult_len_r + add x10, x10, L_GCM_gmult_len_r +L_GCM_gmult_len_start_block + ldp x6, x7, [x0] + ldp x8, x9, [x2] + eor x6, x6, x8 + eor x7, x7, x9 + mov x4, #0 + mov x5, #0 + ; Byte 15 + ubfx x12, x7, #56, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 14 + ubfx x12, x7, #48, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 13 + ubfx x12, x7, #40, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 12 + ubfx x12, x7, #32, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 11 + ubfx x12, x7, #24, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 10 + ubfx x12, x7, #16, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 9 + ubfx x12, x7, #8, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 8 + ubfx x12, x7, #0, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 7 + ubfx x12, x6, #56, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 6 + ubfx x12, x6, #48, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 5 + ubfx x12, x6, #40, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 4 + ubfx x12, x6, #32, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 3 + ubfx x12, x6, #24, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 2 + ubfx x12, x6, #16, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 1 + ubfx x12, x6, #8, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + and x13, x5, #0xff + lsr x5, x5, #8 + orr x5, x5, x4, lsl 56 + lsr x4, x4, #8 + add x11, x10, x13, lsl 1 + ldrb w8, [x11], #1 + ldrb w9, [x11] + orr x4, x4, x8, lsl 56 + eor x4, x4, x9, lsl 48 + ; Byte 0 + ubfx x12, x6, #0, #8 + add x11, x1, x12, lsl 4 + ldp x8, x9, [x11] + rev x8, x8 + rev x9, x9 + eor x4, x4, x8 + eor x5, x5, x9 + rev x4, x4 + rev x5, x5 + stp x4, x5, [x0] + subs x3, x3, #16 + add x2, x2, #16 + bne L_GCM_gmult_len_start_block + ret + ENDP + ENDIF AREA |.text|, CODE, READONLY ALIGN 4 EXPORT AES_GCM_encrypt diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c index ce3cf7f2bb6..a2736ae3218 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c @@ -56065,6 +56065,7 @@ void AES_CBC_decrypt(const unsigned char* in, unsigned char* out, * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) XALIGNED(8) static const word32 L_GCM_gmult_len_r[] = { 0x00000000, 0x1c200000, 0x38400000, 0x24600000, 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, @@ -56490,6 +56491,367 @@ void GCM_gmult_len(unsigned char* x, const unsigned char** m, ); } +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +XALIGNED(4) static const word8 L_GCM_gmult_len_r[] = { + 0x00, 0x00, 0x01, 0xc2, 0x03, 0x84, 0x02, 0x46, + 0x07, 0x08, 0x06, 0xca, 0x04, 0x8c, 0x05, 0x4e, + 0x0e, 0x10, 0x0f, 0xd2, 0x0d, 0x94, 0x0c, 0x56, + 0x09, 0x18, 0x08, 0xda, 0x0a, 0x9c, 0x0b, 0x5e, + 0x1c, 0x20, 0x1d, 0xe2, 0x1f, 0xa4, 0x1e, 0x66, + 0x1b, 0x28, 0x1a, 0xea, 0x18, 0xac, 0x19, 0x6e, + 0x12, 0x30, 0x13, 0xf2, 0x11, 0xb4, 0x10, 0x76, + 0x15, 0x38, 0x14, 0xfa, 0x16, 0xbc, 0x17, 0x7e, + 0x38, 0x40, 0x39, 0x82, 0x3b, 0xc4, 0x3a, 0x06, + 0x3f, 0x48, 0x3e, 0x8a, 0x3c, 0xcc, 0x3d, 0x0e, + 0x36, 0x50, 0x37, 0x92, 0x35, 0xd4, 0x34, 0x16, + 0x31, 0x58, 0x30, 0x9a, 0x32, 0xdc, 0x33, 0x1e, + 0x24, 0x60, 0x25, 0xa2, 0x27, 0xe4, 0x26, 0x26, + 0x23, 0x68, 0x22, 0xaa, 0x20, 0xec, 0x21, 0x2e, + 0x2a, 0x70, 0x2b, 0xb2, 0x29, 0xf4, 0x28, 0x36, + 0x2d, 0x78, 0x2c, 0xba, 0x2e, 0xfc, 0x2f, 0x3e, + 0x70, 0x80, 0x71, 0x42, 0x73, 0x04, 0x72, 0xc6, + 0x77, 0x88, 0x76, 0x4a, 0x74, 0x0c, 0x75, 0xce, + 0x7e, 0x90, 0x7f, 0x52, 0x7d, 0x14, 0x7c, 0xd6, + 0x79, 0x98, 0x78, 0x5a, 0x7a, 0x1c, 0x7b, 0xde, + 0x6c, 0xa0, 0x6d, 0x62, 0x6f, 0x24, 0x6e, 0xe6, + 0x6b, 0xa8, 0x6a, 0x6a, 0x68, 0x2c, 0x69, 0xee, + 0x62, 0xb0, 0x63, 0x72, 0x61, 0x34, 0x60, 0xf6, + 0x65, 0xb8, 0x64, 0x7a, 0x66, 0x3c, 0x67, 0xfe, + 0x48, 0xc0, 0x49, 0x02, 0x4b, 0x44, 0x4a, 0x86, + 0x4f, 0xc8, 0x4e, 0x0a, 0x4c, 0x4c, 0x4d, 0x8e, + 0x46, 0xd0, 0x47, 0x12, 0x45, 0x54, 0x44, 0x96, + 0x41, 0xd8, 0x40, 0x1a, 0x42, 0x5c, 0x43, 0x9e, + 0x54, 0xe0, 0x55, 0x22, 0x57, 0x64, 0x56, 0xa6, + 0x53, 0xe8, 0x52, 0x2a, 0x50, 0x6c, 0x51, 0xae, + 0x5a, 0xf0, 0x5b, 0x32, 0x59, 0x74, 0x58, 0xb6, + 0x5d, 0xf8, 0x5c, 0x3a, 0x5e, 0x7c, 0x5f, 0xbe, + 0xe1, 0x00, 0xe0, 0xc2, 0xe2, 0x84, 0xe3, 0x46, + 0xe6, 0x08, 0xe7, 0xca, 0xe5, 0x8c, 0xe4, 0x4e, + 0xef, 0x10, 0xee, 0xd2, 0xec, 0x94, 0xed, 0x56, + 0xe8, 0x18, 0xe9, 0xda, 0xeb, 0x9c, 0xea, 0x5e, + 0xfd, 0x20, 0xfc, 0xe2, 0xfe, 0xa4, 0xff, 0x66, + 0xfa, 0x28, 0xfb, 0xea, 0xf9, 0xac, 0xf8, 0x6e, + 0xf3, 0x30, 0xf2, 0xf2, 0xf0, 0xb4, 0xf1, 0x76, + 0xf4, 0x38, 0xf5, 0xfa, 0xf7, 0xbc, 0xf6, 0x7e, + 0xd9, 0x40, 0xd8, 0x82, 0xda, 0xc4, 0xdb, 0x06, + 0xde, 0x48, 0xdf, 0x8a, 0xdd, 0xcc, 0xdc, 0x0e, + 0xd7, 0x50, 0xd6, 0x92, 0xd4, 0xd4, 0xd5, 0x16, + 0xd0, 0x58, 0xd1, 0x9a, 0xd3, 0xdc, 0xd2, 0x1e, + 0xc5, 0x60, 0xc4, 0xa2, 0xc6, 0xe4, 0xc7, 0x26, + 0xc2, 0x68, 0xc3, 0xaa, 0xc1, 0xec, 0xc0, 0x2e, + 0xcb, 0x70, 0xca, 0xb2, 0xc8, 0xf4, 0xc9, 0x36, + 0xcc, 0x78, 0xcd, 0xba, 0xcf, 0xfc, 0xce, 0x3e, + 0x91, 0x80, 0x90, 0x42, 0x92, 0x04, 0x93, 0xc6, + 0x96, 0x88, 0x97, 0x4a, 0x95, 0x0c, 0x94, 0xce, + 0x9f, 0x90, 0x9e, 0x52, 0x9c, 0x14, 0x9d, 0xd6, + 0x98, 0x98, 0x99, 0x5a, 0x9b, 0x1c, 0x9a, 0xde, + 0x8d, 0xa0, 0x8c, 0x62, 0x8e, 0x24, 0x8f, 0xe6, + 0x8a, 0xa8, 0x8b, 0x6a, 0x89, 0x2c, 0x88, 0xee, + 0x83, 0xb0, 0x82, 0x72, 0x80, 0x34, 0x81, 0xf6, + 0x84, 0xb8, 0x85, 0x7a, 0x87, 0x3c, 0x86, 0xfe, + 0xa9, 0xc0, 0xa8, 0x02, 0xaa, 0x44, 0xab, 0x86, + 0xae, 0xc8, 0xaf, 0x0a, 0xad, 0x4c, 0xac, 0x8e, + 0xa7, 0xd0, 0xa6, 0x12, 0xa4, 0x54, 0xa5, 0x96, + 0xa0, 0xd8, 0xa1, 0x1a, 0xa3, 0x5c, 0xa2, 0x9e, + 0xb5, 0xe0, 0xb4, 0x22, 0xb6, 0x64, 0xb7, 0xa6, + 0xb2, 0xe8, 0xb3, 0x2a, 0xb1, 0x6c, 0xb0, 0xae, + 0xbb, 0xf0, 0xba, 0x32, 0xb8, 0x74, 0xb9, 0xb6, + 0xbc, 0xf8, 0xbd, 0x3a, 0xbf, 0x7c, 0xbe, 0xbe, +}; + +void GCM_gmult_len(unsigned char* x, const unsigned char** m, + const unsigned char* data, unsigned long len); +void GCM_gmult_len(unsigned char* x, const unsigned char** m, + const unsigned char* data, unsigned long len) +{ + const word8* r = L_GCM_gmult_len_r; + __asm__ __volatile__ ( + "\n" + "L_GCM_gmult_len_start_block_%=:\n\t" + "ldp x6, x7, [%x[x]]\n\t" + "ldp x8, x9, [%x[data]]\n\t" + "eor x6, x6, x8\n\t" + "eor x7, x7, x9\n\t" + "mov x4, #0\n\t" + "mov x5, #0\n\t" + /* Byte 15 */ + "ubfx x12, x7, #56, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 14 */ + "ubfx x12, x7, #48, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 13 */ + "ubfx x12, x7, #40, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 12 */ + "ubfx x12, x7, #32, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 11 */ + "ubfx x12, x7, #24, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 10 */ + "ubfx x12, x7, #16, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 9 */ + "ubfx x12, x7, #8, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 8 */ + "ubfx x12, x7, #0, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 7 */ + "ubfx x12, x6, #56, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 6 */ + "ubfx x12, x6, #48, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 5 */ + "ubfx x12, x6, #40, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 4 */ + "ubfx x12, x6, #32, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 3 */ + "ubfx x12, x6, #24, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 2 */ + "ubfx x12, x6, #16, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 1 */ + "ubfx x12, x6, #8, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "and x13, x5, #0xff\n\t" + "lsr x5, x5, #8\n\t" + "orr x5, x5, x4, lsl 56\n\t" + "lsr x4, x4, #8\n\t" + "add x11, %[r], x13, lsl 1\n\t" + "ldrb w8, [x11], #1\n\t" + "ldrb w9, [x11]\n\t" + "orr x4, x4, x8, lsl 56\n\t" + "eor x4, x4, x9, lsl 48\n\t" + /* Byte 0 */ + "ubfx x12, x6, #0, #8\n\t" + "add x11, %x[m], x12, lsl 4\n\t" + "ldp x8, x9, [x11]\n\t" + "rev x8, x8\n\t" + "rev x9, x9\n\t" + "eor x4, x4, x8\n\t" + "eor x5, x5, x9\n\t" + "rev x4, x4\n\t" + "rev x5, x5\n\t" + "stp x4, x5, [%x[x]]\n\t" + "subs %x[len], %x[len], #16\n\t" + "add %x[data], %x[data], #16\n\t" + "b.ne L_GCM_gmult_len_start_block_%=\n\t" + : [x] "+r" (x), [len] "+r" (len) + : [m] "r" (m), [data] "r" (data), [r] "r" (r) + : "memory", "cc", "x4", "x5", "x6", "x7", "x8", "x9", "x11", "x12", + "x13" + ); +} + +#endif /* GCM_TABLE */ void AES_GCM_encrypt(const unsigned char* in, unsigned char* out, unsigned long len, const unsigned char* ks, int nr, unsigned char* ctr); void AES_GCM_encrypt(const unsigned char* in, unsigned char* out, diff --git a/wolfcrypt/src/port/arm/armv8-chacha-asm.S b/wolfcrypt/src/port/arm/armv8-chacha-asm.S index 2b621209c98..a25f390da39 100644 --- a/wolfcrypt/src/port/arm/armv8-chacha-asm.S +++ b/wolfcrypt/src/port/arm/armv8-chacha-asm.S @@ -30,6 +30,7 @@ #ifdef __aarch64__ #ifndef WOLFSSL_ARMASM_INLINE #ifdef HAVE_CHACHA +#ifndef WOLFSSL_ARMASM_NO_NEON #ifndef __APPLE__ .text .section .rodata @@ -62,7 +63,6 @@ L_chacha20_arm64_ctr: #endif /* __APPLE__ */ L_chacha20_arm64_rol8: .long 0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f -#ifndef WOLFSSL_ARMASM_NO_NEON #ifndef __APPLE__ .text .globl wc_chacha_crypt_bytes diff --git a/wolfcrypt/src/port/arm/armv8-chacha-asm.asm b/wolfcrypt/src/port/arm/armv8-chacha-asm.asm index 842f25f3ff6..dbfdd064ae8 100644 --- a/wolfcrypt/src/port/arm/armv8-chacha-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-chacha-asm.asm @@ -25,6 +25,7 @@ ; ruby ./chacha/chacha.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-chacha-asm.asm IF :DEF:HAVE_CHACHA + IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_chacha20_arm64_ctr @@ -33,7 +34,6 @@ L_chacha20_arm64_ctr ALIGN 8 L_chacha20_arm64_rol8 DCD 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f - IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON AREA |.text|, CODE, READONLY ALIGN 4 EXPORT wc_chacha_crypt_bytes diff --git a/wolfcrypt/src/port/arm/armv8-chacha-asm_c.c b/wolfcrypt/src/port/arm/armv8-chacha-asm_c.c index d651fb0cdb5..58fb68081a2 100644 --- a/wolfcrypt/src/port/arm/armv8-chacha-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-chacha-asm_c.c @@ -33,6 +33,7 @@ #ifdef HAVE_CHACHA #include +#ifndef WOLFSSL_ARMASM_NO_NEON XALIGNED(8) static const word32 L_chacha20_arm64_ctr[] = { 0x00000000, 0x00000001, 0x00000002, 0x00000003, }; @@ -41,7 +42,6 @@ XALIGNED(8) static const word32 L_chacha20_arm64_rol8[] = { 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f, }; -#ifndef WOLFSSL_ARMASM_NO_NEON void wc_chacha_crypt_bytes(ChaCha* ctx, byte* c, const byte* m, word32 len) { const word32* rol8 = L_chacha20_arm64_rol8; diff --git a/wolfcrypt/src/port/arm/armv8-sha512-asm.S b/wolfcrypt/src/port/arm/armv8-sha512-asm.S index 1a48cf73177..8a1ae2b72f2 100644 --- a/wolfcrypt/src/port/arm/armv8-sha512-asm.S +++ b/wolfcrypt/src/port/arm/armv8-sha512-asm.S @@ -30,6 +30,7 @@ #ifdef __aarch64__ #ifndef WOLFSSL_ARMASM_INLINE #if defined(WOLFSSL_SHA512) || defined(WOLFSSL_SHA384) +#ifndef WOLFSSL_ARMASM_NO_NEON #ifndef __APPLE__ .text .section .rodata @@ -1645,6 +1646,7 @@ L_sha512_len_crypto_begin: .size Transform_Sha512_Len_crypto,.-Transform_Sha512_Len_crypto #endif /* __APPLE__ */ #endif /* WOLFSSL_ARMASM_CRYPTO_SHA512 */ +#endif /* !WOLFSSL_ARMASM_NO_NEON */ #endif /* WOLFSSL_SHA512 || WOLFSSL_SHA384 */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-sha512-asm.asm b/wolfcrypt/src/port/arm/armv8-sha512-asm.asm index c257f9183a4..eaf2f4f4452 100644 --- a/wolfcrypt/src/port/arm/armv8-sha512-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-sha512-asm.asm @@ -25,6 +25,7 @@ ; ruby ./sha2/sha512.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-sha512-asm.asm IF :DEF:WOLFSSL_SHA512 :LOR: :DEF:WOLFSSL_SHA384 + IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA512_transform_neon_len_k @@ -1568,4 +1569,5 @@ L_sha512_len_crypto_begin ENDP ENDIF ENDIF + ENDIF END diff --git a/wolfcrypt/src/port/arm/armv8-sha512-asm_c.c b/wolfcrypt/src/port/arm/armv8-sha512-asm_c.c index 532da05655f..bb7d37d7af4 100644 --- a/wolfcrypt/src/port/arm/armv8-sha512-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-sha512-asm_c.c @@ -33,6 +33,7 @@ #include #if defined(WOLFSSL_SHA512) || defined(WOLFSSL_SHA384) +#ifndef WOLFSSL_ARMASM_NO_NEON XALIGNED(16) static const word64 L_SHA512_transform_neon_len_k[] = { 0x428a2f98d728ae22UL, 0x7137449123ef65cdUL, 0xb5c0fbcfec4d3b2fUL, 0xe9b5dba58189dbbcUL, @@ -1564,6 +1565,7 @@ void Transform_Sha512_Len_crypto(wc_Sha512* sha512, const byte* data, } #endif /* WOLFSSL_ARMASM_CRYPTO_SHA512 */ +#endif /* !WOLFSSL_ARMASM_NO_NEON */ #endif /* WOLFSSL_SHA512 || WOLFSSL_SHA384 */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/thumb2-aes-asm.S b/wolfcrypt/src/port/arm/thumb2-aes-asm.S index e8c952537df..4ec33e3972b 100644 --- a/wolfcrypt/src/port/arm/thumb2-aes-asm.S +++ b/wolfcrypt/src/port/arm/thumb2-aes-asm.S @@ -5863,6 +5863,7 @@ L_AES_CBC_decrypt_end: * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) #ifndef __APPLE__ .text .type L_GCM_gmult_len_r, %object @@ -6417,6 +6418,539 @@ L_GCM_gmult_len_start_block: POP {r4, r5, r6, r7, r8, r9, r10, r11, pc} /* Cycle Count = 718 */ .size GCM_gmult_len,.-GCM_gmult_len +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +#ifndef __APPLE__ + .text + .type L_GCM_gmult_len_r, %object + .size L_GCM_gmult_len_r, 512 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + /* 4-byte aligned, 32-bit aligned */ +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_GCM_gmult_len_r: + .byte 0x00,0x00,0x01,0xc2,0x03,0x84,0x02,0x46 + .byte 0x07,0x08,0x06,0xca,0x04,0x8c,0x05,0x4e + .byte 0x0e,0x10,0x0f,0xd2,0x0d,0x94,0x0c,0x56 + .byte 0x09,0x18,0x08,0xda,0x0a,0x9c,0x0b,0x5e + .byte 0x1c,0x20,0x1d,0xe2,0x1f,0xa4,0x1e,0x66 + .byte 0x1b,0x28,0x1a,0xea,0x18,0xac,0x19,0x6e + .byte 0x12,0x30,0x13,0xf2,0x11,0xb4,0x10,0x76 + .byte 0x15,0x38,0x14,0xfa,0x16,0xbc,0x17,0x7e + .byte 0x38,0x40,0x39,0x82,0x3b,0xc4,0x3a,0x06 + .byte 0x3f,0x48,0x3e,0x8a,0x3c,0xcc,0x3d,0x0e + .byte 0x36,0x50,0x37,0x92,0x35,0xd4,0x34,0x16 + .byte 0x31,0x58,0x30,0x9a,0x32,0xdc,0x33,0x1e + .byte 0x24,0x60,0x25,0xa2,0x27,0xe4,0x26,0x26 + .byte 0x23,0x68,0x22,0xaa,0x20,0xec,0x21,0x2e + .byte 0x2a,0x70,0x2b,0xb2,0x29,0xf4,0x28,0x36 + .byte 0x2d,0x78,0x2c,0xba,0x2e,0xfc,0x2f,0x3e + .byte 0x70,0x80,0x71,0x42,0x73,0x04,0x72,0xc6 + .byte 0x77,0x88,0x76,0x4a,0x74,0x0c,0x75,0xce + .byte 0x7e,0x90,0x7f,0x52,0x7d,0x14,0x7c,0xd6 + .byte 0x79,0x98,0x78,0x5a,0x7a,0x1c,0x7b,0xde + .byte 0x6c,0xa0,0x6d,0x62,0x6f,0x24,0x6e,0xe6 + .byte 0x6b,0xa8,0x6a,0x6a,0x68,0x2c,0x69,0xee + .byte 0x62,0xb0,0x63,0x72,0x61,0x34,0x60,0xf6 + .byte 0x65,0xb8,0x64,0x7a,0x66,0x3c,0x67,0xfe + .byte 0x48,0xc0,0x49,0x02,0x4b,0x44,0x4a,0x86 + .byte 0x4f,0xc8,0x4e,0x0a,0x4c,0x4c,0x4d,0x8e + .byte 0x46,0xd0,0x47,0x12,0x45,0x54,0x44,0x96 + .byte 0x41,0xd8,0x40,0x1a,0x42,0x5c,0x43,0x9e + .byte 0x54,0xe0,0x55,0x22,0x57,0x64,0x56,0xa6 + .byte 0x53,0xe8,0x52,0x2a,0x50,0x6c,0x51,0xae + .byte 0x5a,0xf0,0x5b,0x32,0x59,0x74,0x58,0xb6 + .byte 0x5d,0xf8,0x5c,0x3a,0x5e,0x7c,0x5f,0xbe + .byte 0xe1,0x00,0xe0,0xc2,0xe2,0x84,0xe3,0x46 + .byte 0xe6,0x08,0xe7,0xca,0xe5,0x8c,0xe4,0x4e + .byte 0xef,0x10,0xee,0xd2,0xec,0x94,0xed,0x56 + .byte 0xe8,0x18,0xe9,0xda,0xeb,0x9c,0xea,0x5e + .byte 0xfd,0x20,0xfc,0xe2,0xfe,0xa4,0xff,0x66 + .byte 0xfa,0x28,0xfb,0xea,0xf9,0xac,0xf8,0x6e + .byte 0xf3,0x30,0xf2,0xf2,0xf0,0xb4,0xf1,0x76 + .byte 0xf4,0x38,0xf5,0xfa,0xf7,0xbc,0xf6,0x7e + .byte 0xd9,0x40,0xd8,0x82,0xda,0xc4,0xdb,0x06 + .byte 0xde,0x48,0xdf,0x8a,0xdd,0xcc,0xdc,0x0e + .byte 0xd7,0x50,0xd6,0x92,0xd4,0xd4,0xd5,0x16 + .byte 0xd0,0x58,0xd1,0x9a,0xd3,0xdc,0xd2,0x1e + .byte 0xc5,0x60,0xc4,0xa2,0xc6,0xe4,0xc7,0x26 + .byte 0xc2,0x68,0xc3,0xaa,0xc1,0xec,0xc0,0x2e + .byte 0xcb,0x70,0xca,0xb2,0xc8,0xf4,0xc9,0x36 + .byte 0xcc,0x78,0xcd,0xba,0xcf,0xfc,0xce,0x3e + .byte 0x91,0x80,0x90,0x42,0x92,0x04,0x93,0xc6 + .byte 0x96,0x88,0x97,0x4a,0x95,0x0c,0x94,0xce + .byte 0x9f,0x90,0x9e,0x52,0x9c,0x14,0x9d,0xd6 + .byte 0x98,0x98,0x99,0x5a,0x9b,0x1c,0x9a,0xde + .byte 0x8d,0xa0,0x8c,0x62,0x8e,0x24,0x8f,0xe6 + .byte 0x8a,0xa8,0x8b,0x6a,0x89,0x2c,0x88,0xee + .byte 0x83,0xb0,0x82,0x72,0x80,0x34,0x81,0xf6 + .byte 0x84,0xb8,0x85,0x7a,0x87,0x3c,0x86,0xfe + .byte 0xa9,0xc0,0xa8,0x02,0xaa,0x44,0xab,0x86 + .byte 0xae,0xc8,0xaf,0x0a,0xad,0x4c,0xac,0x8e + .byte 0xa7,0xd0,0xa6,0x12,0xa4,0x54,0xa5,0x96 + .byte 0xa0,0xd8,0xa1,0x1a,0xa3,0x5c,0xa2,0x9e + .byte 0xb5,0xe0,0xb4,0x22,0xb6,0x64,0xb7,0xa6 + .byte 0xb2,0xe8,0xb3,0x2a,0xb1,0x6c,0xb0,0xae + .byte 0xbb,0xf0,0xba,0x32,0xb8,0x74,0xb9,0xb6 + .byte 0xbc,0xf8,0xbd,0x3a,0xbf,0x7c,0xbe,0xbe + .text + .align 4 + .globl GCM_gmult_len + .type GCM_gmult_len, %function +GCM_gmult_len: + PUSH {r4, r5, r6, r7, r8, r9, r10, r11, lr} + ADR lr, L_GCM_gmult_len_r +L_GCM_gmult_len_start_block: + LDM r0, {r4, r5, r6, r7} + LDM r2, {r8, r9, r10, r11} + EOR r4, r4, r8 + EOR r5, r5, r9 + EOR r6, r6, r10 + EOR r7, r7, r11 + STM r0, {r4, r5, r6, r7} + MOV r8, #0 + MOV r9, #0 + MOV r10, #0 + MOV r11, #0 + /* Byte 15 */ + ADD r12, r0, #15 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 14 */ + ADD r12, r0, #14 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 13 */ + ADD r12, r0, #13 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 12 */ + ADD r12, r0, #12 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 11 */ + ADD r12, r0, #11 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 10 */ + ADD r12, r0, #10 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 9 */ + ADD r12, r0, #9 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 8 */ + ADD r12, r0, #8 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 7 */ + ADD r12, r0, #7 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 6 */ + ADD r12, r0, #6 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 5 */ + ADD r12, r0, #5 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 4 */ + ADD r12, r0, #4 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 3 */ + ADD r12, r0, #3 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 2 */ + ADD r12, r0, #2 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 1 */ + ADD r12, r0, #1 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + AND r4, r11, #0xff + LSR r11, r11, #8 + ORR r11, r11, r10, LSL #24 + LSR r10, r10, #8 + ORR r10, r10, r9, LSL #24 + LSR r9, r9, #8 + ORR r9, r9, r8, LSL #24 + LSR r8, r8, #8 + ADD r4, lr, r4, LSL #1 + LDRB r5, [r4] + ADD r4, r4, #1 + LDRB r6, [r4] + ORR r8, r8, r5, LSL #24 + EOR r8, r8, r6, LSL #16 + /* Byte 0 */ + ADD r12, r0, #0 + LDRB r12, [r12] + ADD r12, r1, r12, LSL #4 + LDM r12, {r4, r5, r6, r7} + REV r4, r4 + REV r5, r5 + REV r6, r6 + REV r7, r7 + EOR r8, r8, r4 + EOR r9, r9, r5 + EOR r10, r10, r6 + EOR r11, r11, r7 + REV r8, r8 + REV r9, r9 + REV r10, r10 + REV r11, r11 + STM r0, {r8, r9, r10, r11} + SUBS r3, r3, #16 + ADD r2, r2, #16 +#if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__) + BNE L_GCM_gmult_len_start_block +#else + BNE.W L_GCM_gmult_len_start_block +#endif + POP {r4, r5, r6, r7, r8, r9, r10, r11, pc} + /* Cycle Count = 571 */ + .size GCM_gmult_len,.-GCM_gmult_len +#endif /* GCM_TABLE */ #ifndef __APPLE__ .text .type L_AES_Thumb2_te_gcm, %object diff --git a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c index 07773174da3..a1f5ec1b518 100644 --- a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c @@ -6479,6 +6479,7 @@ WC_OMIT_FRAME_POINTER void AES_CBC_decrypt(const unsigned char* in, * HAVE_AES_ECB */ #endif /* HAVE_AES_DECRYPT */ #ifdef HAVE_AESGCM +#if !defined(GCM_SMALL) && !defined(GCM_TABLE) XALIGNED(8) static const word32 L_GCM_gmult_len_r[] = { 0x00000000, 0x1c200000, 0x38400000, 0x24600000, 0x70800000, 0x6ca00000, 0x48c00000, 0x54e00000, @@ -7060,6 +7061,566 @@ WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, ); } +#endif /* !defined(GCM_SMALL) && !defined(GCM_TABLE) */ +#ifdef GCM_TABLE +XALIGNED(4) static const word8 L_GCM_gmult_len_r[] = { + 0x00, 0x00, 0x01, 0xc2, 0x03, 0x84, 0x02, 0x46, + 0x07, 0x08, 0x06, 0xca, 0x04, 0x8c, 0x05, 0x4e, + 0x0e, 0x10, 0x0f, 0xd2, 0x0d, 0x94, 0x0c, 0x56, + 0x09, 0x18, 0x08, 0xda, 0x0a, 0x9c, 0x0b, 0x5e, + 0x1c, 0x20, 0x1d, 0xe2, 0x1f, 0xa4, 0x1e, 0x66, + 0x1b, 0x28, 0x1a, 0xea, 0x18, 0xac, 0x19, 0x6e, + 0x12, 0x30, 0x13, 0xf2, 0x11, 0xb4, 0x10, 0x76, + 0x15, 0x38, 0x14, 0xfa, 0x16, 0xbc, 0x17, 0x7e, + 0x38, 0x40, 0x39, 0x82, 0x3b, 0xc4, 0x3a, 0x06, + 0x3f, 0x48, 0x3e, 0x8a, 0x3c, 0xcc, 0x3d, 0x0e, + 0x36, 0x50, 0x37, 0x92, 0x35, 0xd4, 0x34, 0x16, + 0x31, 0x58, 0x30, 0x9a, 0x32, 0xdc, 0x33, 0x1e, + 0x24, 0x60, 0x25, 0xa2, 0x27, 0xe4, 0x26, 0x26, + 0x23, 0x68, 0x22, 0xaa, 0x20, 0xec, 0x21, 0x2e, + 0x2a, 0x70, 0x2b, 0xb2, 0x29, 0xf4, 0x28, 0x36, + 0x2d, 0x78, 0x2c, 0xba, 0x2e, 0xfc, 0x2f, 0x3e, + 0x70, 0x80, 0x71, 0x42, 0x73, 0x04, 0x72, 0xc6, + 0x77, 0x88, 0x76, 0x4a, 0x74, 0x0c, 0x75, 0xce, + 0x7e, 0x90, 0x7f, 0x52, 0x7d, 0x14, 0x7c, 0xd6, + 0x79, 0x98, 0x78, 0x5a, 0x7a, 0x1c, 0x7b, 0xde, + 0x6c, 0xa0, 0x6d, 0x62, 0x6f, 0x24, 0x6e, 0xe6, + 0x6b, 0xa8, 0x6a, 0x6a, 0x68, 0x2c, 0x69, 0xee, + 0x62, 0xb0, 0x63, 0x72, 0x61, 0x34, 0x60, 0xf6, + 0x65, 0xb8, 0x64, 0x7a, 0x66, 0x3c, 0x67, 0xfe, + 0x48, 0xc0, 0x49, 0x02, 0x4b, 0x44, 0x4a, 0x86, + 0x4f, 0xc8, 0x4e, 0x0a, 0x4c, 0x4c, 0x4d, 0x8e, + 0x46, 0xd0, 0x47, 0x12, 0x45, 0x54, 0x44, 0x96, + 0x41, 0xd8, 0x40, 0x1a, 0x42, 0x5c, 0x43, 0x9e, + 0x54, 0xe0, 0x55, 0x22, 0x57, 0x64, 0x56, 0xa6, + 0x53, 0xe8, 0x52, 0x2a, 0x50, 0x6c, 0x51, 0xae, + 0x5a, 0xf0, 0x5b, 0x32, 0x59, 0x74, 0x58, 0xb6, + 0x5d, 0xf8, 0x5c, 0x3a, 0x5e, 0x7c, 0x5f, 0xbe, + 0xe1, 0x00, 0xe0, 0xc2, 0xe2, 0x84, 0xe3, 0x46, + 0xe6, 0x08, 0xe7, 0xca, 0xe5, 0x8c, 0xe4, 0x4e, + 0xef, 0x10, 0xee, 0xd2, 0xec, 0x94, 0xed, 0x56, + 0xe8, 0x18, 0xe9, 0xda, 0xeb, 0x9c, 0xea, 0x5e, + 0xfd, 0x20, 0xfc, 0xe2, 0xfe, 0xa4, 0xff, 0x66, + 0xfa, 0x28, 0xfb, 0xea, 0xf9, 0xac, 0xf8, 0x6e, + 0xf3, 0x30, 0xf2, 0xf2, 0xf0, 0xb4, 0xf1, 0x76, + 0xf4, 0x38, 0xf5, 0xfa, 0xf7, 0xbc, 0xf6, 0x7e, + 0xd9, 0x40, 0xd8, 0x82, 0xda, 0xc4, 0xdb, 0x06, + 0xde, 0x48, 0xdf, 0x8a, 0xdd, 0xcc, 0xdc, 0x0e, + 0xd7, 0x50, 0xd6, 0x92, 0xd4, 0xd4, 0xd5, 0x16, + 0xd0, 0x58, 0xd1, 0x9a, 0xd3, 0xdc, 0xd2, 0x1e, + 0xc5, 0x60, 0xc4, 0xa2, 0xc6, 0xe4, 0xc7, 0x26, + 0xc2, 0x68, 0xc3, 0xaa, 0xc1, 0xec, 0xc0, 0x2e, + 0xcb, 0x70, 0xca, 0xb2, 0xc8, 0xf4, 0xc9, 0x36, + 0xcc, 0x78, 0xcd, 0xba, 0xcf, 0xfc, 0xce, 0x3e, + 0x91, 0x80, 0x90, 0x42, 0x92, 0x04, 0x93, 0xc6, + 0x96, 0x88, 0x97, 0x4a, 0x95, 0x0c, 0x94, 0xce, + 0x9f, 0x90, 0x9e, 0x52, 0x9c, 0x14, 0x9d, 0xd6, + 0x98, 0x98, 0x99, 0x5a, 0x9b, 0x1c, 0x9a, 0xde, + 0x8d, 0xa0, 0x8c, 0x62, 0x8e, 0x24, 0x8f, 0xe6, + 0x8a, 0xa8, 0x8b, 0x6a, 0x89, 0x2c, 0x88, 0xee, + 0x83, 0xb0, 0x82, 0x72, 0x80, 0x34, 0x81, 0xf6, + 0x84, 0xb8, 0x85, 0x7a, 0x87, 0x3c, 0x86, 0xfe, + 0xa9, 0xc0, 0xa8, 0x02, 0xaa, 0x44, 0xab, 0x86, + 0xae, 0xc8, 0xaf, 0x0a, 0xad, 0x4c, 0xac, 0x8e, + 0xa7, 0xd0, 0xa6, 0x12, 0xa4, 0x54, 0xa5, 0x96, + 0xa0, 0xd8, 0xa1, 0x1a, 0xa3, 0x5c, 0xa2, 0x9e, + 0xb5, 0xe0, 0xb4, 0x22, 0xb6, 0x64, 0xb7, 0xa6, + 0xb2, 0xe8, 0xb3, 0x2a, 0xb1, 0x6c, 0xb0, 0xae, + 0xbb, 0xf0, 0xba, 0x32, 0xb8, 0x74, 0xb9, 0xb6, + 0xbc, 0xf8, 0xbd, 0x3a, 0xbf, 0x7c, 0xbe, 0xbe, +}; + +void GCM_gmult_len(unsigned char* x_p, const unsigned char** m_p, + const unsigned char* data_p, unsigned long len_p); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x_p, + const unsigned char** m_p, const unsigned char* data_p, unsigned long len_p) +#else +WC_OMIT_FRAME_POINTER void GCM_gmult_len(unsigned char* x, + const unsigned char** m, const unsigned char* data, unsigned long len) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register unsigned char* x __asm__ ("r0") = (unsigned char*)x_p; + register const unsigned char** m __asm__ ("r1") = + (const unsigned char**)m_p; + register const unsigned char* data __asm__ ("r2") = + (const unsigned char*)data_p; + register unsigned long len __asm__ ("r3") = (unsigned long)len_p; + register word8* L_GCM_gmult_len_r_c __asm__ ("r4") = + (word8*)&L_GCM_gmult_len_r; +#else + register word8* L_GCM_gmult_len_r_c = (word8*)&L_GCM_gmult_len_r; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "PUSH {%[L_GCM_gmult_len_r]}\n\t" + "MOV lr, %[L_GCM_gmult_len_r]\n\t" + "\n" +#if defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000) + "L_GCM_gmult_len_start_block:\n\t" +#else + "L_GCM_gmult_len_start_block_%=:\n\t" +#endif + "LDM %[x], {r4, r5, r6, r7}\n\t" + "LDM %[data], {r8, r9, r10, r11}\n\t" + "EOR r4, r4, r8\n\t" + "EOR r5, r5, r9\n\t" + "EOR r6, r6, r10\n\t" + "EOR r7, r7, r11\n\t" + "STM %[x], {r4, r5, r6, r7}\n\t" + "MOV r8, #0\n\t" + "MOV r9, #0\n\t" + "MOV r10, #0\n\t" + "MOV r11, #0\n\t" + /* Byte 15 */ + "ADD r12, %[x], #15\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 14 */ + "ADD r12, %[x], #14\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 13 */ + "ADD r12, %[x], #13\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 12 */ + "ADD r12, %[x], #12\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 11 */ + "ADD r12, %[x], #11\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 10 */ + "ADD r12, %[x], #10\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 9 */ + "ADD r12, %[x], #9\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 8 */ + "ADD r12, %[x], #8\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 7 */ + "ADD r12, %[x], #7\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 6 */ + "ADD r12, %[x], #6\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 5 */ + "ADD r12, %[x], #5\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 4 */ + "ADD r12, %[x], #4\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 3 */ + "ADD r12, %[x], #3\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 2 */ + "ADD r12, %[x], #2\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 1 */ + "ADD r12, %[x], #1\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "AND r4, r11, #0xff\n\t" + "LSR r11, r11, #8\n\t" + "ORR r11, r11, r10, LSL #24\n\t" + "LSR r10, r10, #8\n\t" + "ORR r10, r10, r9, LSL #24\n\t" + "LSR r9, r9, #8\n\t" + "ORR r9, r9, r8, LSL #24\n\t" + "LSR r8, r8, #8\n\t" + "ADD r4, lr, r4, LSL #1\n\t" + "LDRB r5, [r4]\n\t" + "ADD r4, r4, #1\n\t" + "LDRB r6, [r4]\n\t" + "ORR r8, r8, r5, LSL #24\n\t" + "EOR r8, r8, r6, LSL #16\n\t" + /* Byte 0 */ + "ADD r12, %[x], #0\n\t" + "LDRB r12, [r12]\n\t" + "ADD r12, %[m], r12, LSL #4\n\t" + "LDM r12, {r4, r5, r6, r7}\n\t" + "REV r4, r4\n\t" + "REV r5, r5\n\t" + "REV r6, r6\n\t" + "REV r7, r7\n\t" + "EOR r8, r8, r4\n\t" + "EOR r9, r9, r5\n\t" + "EOR r10, r10, r6\n\t" + "EOR r11, r11, r7\n\t" + "REV r8, r8\n\t" + "REV r9, r9\n\t" + "REV r10, r10\n\t" + "REV r11, r11\n\t" + "STM %[x], {r8, r9, r10, r11}\n\t" + "SUBS %[len], %[len], #16\n\t" + "ADD %[data], %[data], #16\n\t" +#if defined(__GNUC__) + "BNE L_GCM_gmult_len_start_block_%=\n\t" +#elif defined(__IAR_SYSTEMS_ICC__) && (__VER__ < 9000000) + "BNE.W L_GCM_gmult_len_start_block\n\t" +#else + "BNE.W L_GCM_gmult_len_start_block_%=\n\t" +#endif + "POP {%[L_GCM_gmult_len_r]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [x] "+r" (x), [m] "+r" (m), [data] "+r" (data), [len] "+r" (len), + [L_GCM_gmult_len_r] "+r" (L_GCM_gmult_len_r_c) + : +#else + : + : [x] "r" (x), [m] "r" (m), [data] "r" (data), [len] "r" (len), + [L_GCM_gmult_len_r] "r" (L_GCM_gmult_len_r_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "lr", "r5", "r6", "r7", "r8", "r9", "r10", + "r11" + ); +} + +#endif /* GCM_TABLE */ static const word32* L_AES_Thumb2_te_gcm = L_AES_Thumb2_te_data; void AES_GCM_encrypt(const unsigned char* in_p, unsigned char* out_p, unsigned long len_p, const unsigned char* ks_p, int nr_p, diff --git a/wolfssl/wolfcrypt/aes.h b/wolfssl/wolfcrypt/aes.h index 183aad072f8..7f7e5b55a70 100644 --- a/wolfssl/wolfcrypt/aes.h +++ b/wolfssl/wolfcrypt/aes.h @@ -67,10 +67,6 @@ typedef struct Gcm { #endif WOLFSSL_LOCAL void GenerateM0(Gcm* gcm); -#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \ - !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) -WOLFSSL_LOCAL void GMULT(byte* X, byte* Y); -#endif WOLFSSL_LOCAL void WC_ARG_NOT_NULL(1) GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, word32 cSz, byte* s, word32 sSz); @@ -985,12 +981,12 @@ WOLFSSL_LOCAL void AES_CBC_decrypt_NEON(const unsigned char* in, WOLFSSL_LOCAL void AES_CTR_encrypt_NEON(const unsigned char* in, unsigned char* out, unsigned long len, const unsigned char* ks, int nr, unsigned char* ctr); -#if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) -/* in pre-C2x C, constness conflicts for dimensioned arrays can't be resolved. +/* Defined whenever HAVE_AESGCM - the NEON GHASH is used for every GCM table + * layout, not just GCM_TABLE / GCM_TABLE_4BIT. + * in pre-C2x C, constness conflicts for dimensioned arrays can't be resolved. */ WOLFSSL_LOCAL void GCM_gmult_len_NEON(byte* x, const byte* h, const unsigned char* data, unsigned long len); -#endif WOLFSSL_LOCAL void AES_GCM_encrypt_NEON(const unsigned char* in, unsigned char* out, unsigned long len, const unsigned char* ks, int nr, unsigned char* ctr);