From 37369a8a574c43c6eaabec6acb6b4bfe72dbd126 Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Thu, 16 Jan 2020 14:31:10 +0800 Subject: [PATCH] crypto: SHA and AES accelerator bring up for S2 Brings up, fixes and enables AES and SHA hardware acceleration. Closes IDF-714 Closes IDF-716 --- components/mbedtls/CMakeLists.txt | 11 +- components/mbedtls/Kconfig | 25 +- components/mbedtls/port/esp32/aes.c | 245 +-- .../mbedtls/port/{ => esp32}/esp_sha1.c | 0 .../mbedtls/port/{ => esp32}/esp_sha256.c | 0 .../mbedtls/port/{ => esp32}/esp_sha512.c | 0 components/mbedtls/port/esp32s2/aes.c | 1587 +++++++---------- components/mbedtls/port/esp32s2/esp_bignum.c | 26 +- components/mbedtls/port/esp32s2/esp_sha1.c | 255 +++ components/mbedtls/port/esp32s2/esp_sha256.c | 267 +++ components/mbedtls/port/esp32s2/esp_sha512.c | 317 ++++ components/mbedtls/port/esp32s2/sha.c | 371 ++-- components/mbedtls/port/esp_aes_xts.c | 288 +++ components/mbedtls/port/esp_sha.c | 2 - components/mbedtls/port/include/esp32s2/aes.h | 15 +- .../mbedtls/port/include/esp32s2/crypto_dma.h | 4 +- components/mbedtls/port/include/esp32s2/gcm.h | 94 +- components/mbedtls/port/include/esp32s2/sha.h | 183 +- components/mbedtls/port/include/gcm_alt.h | 55 + .../mbedtls/port/include/mbedtls/esp_config.h | 6 + components/mbedtls/port/include/sha1_alt.h | 29 +- components/mbedtls/port/include/sha256_alt.h | 32 +- components/mbedtls/port/include/sha512_alt.h | 44 +- components/mbedtls/test/test_aes.c | 1025 +++++++++++ components/mbedtls/test/test_aes_perf.c | 63 +- components/mbedtls/test/test_mbedtls_sha.c | 177 +- components/mbedtls/test/test_sha_perf.c | 8 +- .../soc/include/soc/soc_memory_layout.h | 9 + .../soc/esp32s2/include/soc/hwcrypto_reg.h | 18 - .../soc/soc/esp32s2/include/soc/periph_defs.h | 6 + components/soc/soc/esp32s2/include/soc/soc.h | 6 +- .../soc/src/esp32s2/include/hal/clk_gate_ll.h | 69 + 32 files changed, 3687 insertions(+), 1550 deletions(-) rename components/mbedtls/port/{ => esp32}/esp_sha1.c (100%) rename components/mbedtls/port/{ => esp32}/esp_sha256.c (100%) rename components/mbedtls/port/{ => esp32}/esp_sha512.c (100%) create mode 100644 components/mbedtls/port/esp32s2/esp_sha1.c create mode 100644 components/mbedtls/port/esp32s2/esp_sha256.c create mode 100644 components/mbedtls/port/esp32s2/esp_sha512.c create mode 100644 components/mbedtls/port/esp_aes_xts.c create mode 100644 components/mbedtls/port/include/gcm_alt.h create mode 100644 components/mbedtls/test/test_aes.c diff --git a/components/mbedtls/CMakeLists.txt b/components/mbedtls/CMakeLists.txt index eca043e01..348636a13 100644 --- a/components/mbedtls/CMakeLists.txt +++ b/components/mbedtls/CMakeLists.txt @@ -74,14 +74,15 @@ target_sources(mbedtls PRIVATE "${COMPONENT_DIR}/port/mbedtls_debug.c" target_sources(mbedcrypto PRIVATE "${COMPONENT_DIR}/port/esp_hardware.c" "${COMPONENT_DIR}/port/esp_mem.c" - "${COMPONENT_DIR}/port/esp_sha.c" - "${COMPONENT_DIR}/port/esp_sha1.c" - "${COMPONENT_DIR}/port/esp_sha256.c" - "${COMPONENT_DIR}/port/esp_sha512.c" "${COMPONENT_DIR}/port/esp_timing.c" + "${COMPONENT_DIR}/port/esp_sha.c" + "${COMPONENT_DIR}/port/esp_aes_xts.c" "${COMPONENT_DIR}/port/${idf_target}/esp_bignum.c" "${COMPONENT_DIR}/port/${idf_target}/aes.c" - "${COMPONENT_DIR}/port/${idf_target}/sha.c") + "${COMPONENT_DIR}/port/${idf_target}/sha.c" + "${COMPONENT_DIR}/port/${idf_target}/esp_sha1.c" + "${COMPONENT_DIR}/port/${idf_target}/esp_sha256.c" + "${COMPONENT_DIR}/port/${idf_target}/esp_sha512.c") foreach(target ${mbedtls_targets}) target_compile_definitions(${target} PUBLIC -DMBEDTLS_CONFIG_FILE="mbedtls/esp_config.h") diff --git a/components/mbedtls/Kconfig b/components/mbedtls/Kconfig index 5bdacfcd0..49ec176ce 100644 --- a/components/mbedtls/Kconfig +++ b/components/mbedtls/Kconfig @@ -189,13 +189,33 @@ menu "mbedTLS" config MBEDTLS_HARDWARE_AES bool "Enable hardware AES acceleration" default y - depends on !IDF_TARGET_ESP32S2 help Enable hardware accelerated AES encryption & decryption. Note that if the ESP32 CPU is running at 240MHz, hardware AES does not offer any speed boost over software AES. + config MBEDTLS_AES_USE_INTERRUPT + bool "Use interrupt for long AES operations" + depends on MBEDTLS_HARDWARE_AES + default y + help + Use an interrupt to coordinate long AES operations. + + This allows other code to run on the CPU while an AES operation is pending. + Otherwise the CPU busy-waits. + + config MBEDTLS_HARDWARE_GCM + bool "Enable partially hardware accelerated GCM" + #depends on IDF_TARGET_ESP32S2 && MBEDTLS_HARDWARE_AES + depends on 0 + default n + help + Enable partially hardware accelerated GCM. + + Due to hardware limitations, hardware acceleration currently does not + offer any speed boost over software GCM with hardware accelerated AES operations. + config MBEDTLS_HARDWARE_MPI bool "Enable hardware MPI (bignum) acceleration" default y @@ -211,11 +231,10 @@ menu "mbedTLS" config MBEDTLS_HARDWARE_SHA bool "Enable hardware SHA acceleration" default y - depends on !IDF_TARGET_ESP32S2 help Enable hardware accelerated SHA1, SHA256, SHA384 & SHA512 in mbedTLS. - Due to a hardware limitation, hardware acceleration is only + Due to a hardware limitation, on the ESP32 hardware acceleration is only guaranteed if SHA digests are calculated one at a time. If more than one SHA digest is calculated at the same time, one will be calculated fully in hardware and the rest will be calculated diff --git a/components/mbedtls/port/esp32/aes.c b/components/mbedtls/port/esp32/aes.c index edf2e2f08..3557237f4 100644 --- a/components/mbedtls/port/esp32/aes.c +++ b/components/mbedtls/port/esp32/aes.c @@ -508,247 +508,4 @@ int esp_aes_crypt_ofb( esp_aes_context *ctx, esp_aes_release_hardware(); return( ret ); -} - -/* Below XTS implementation is copied aes.c of mbedtls library. - * When MBEDTLS_AES_ALT is defined mbedtls expects alternate - * definition of XTS functions to be available. Even if this - * could have been avoided, it is done for consistency reason. - */ - -void esp_aes_xts_init( esp_aes_xts_context *ctx ) -{ - esp_aes_init( &ctx->crypt ); - esp_aes_init( &ctx->tweak ); -} - -void esp_aes_xts_free( esp_aes_xts_context *ctx ) -{ - esp_aes_free( &ctx->crypt ); - esp_aes_free( &ctx->tweak ); -} - -static int esp_aes_xts_decode_keys( const unsigned char *key, - unsigned int keybits, - const unsigned char **key1, - unsigned int *key1bits, - const unsigned char **key2, - unsigned int *key2bits ) -{ - const unsigned int half_keybits = keybits / 2; - const unsigned int half_keybytes = half_keybits / 8; - - switch( keybits ) - { - case 256: break; - case 512: break; - default : return( MBEDTLS_ERR_AES_INVALID_KEY_LENGTH ); - } - - *key1bits = half_keybits; - *key2bits = half_keybits; - *key1 = &key[0]; - *key2 = &key[half_keybytes]; - - return 0; -} - -int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits) -{ - int ret; - const unsigned char *key1, *key2; - unsigned int key1bits, key2bits; - - ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, - &key2, &key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set the tweak key. Always set tweak key for the encryption mode. */ - ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set crypt key for encryption. */ - return esp_aes_setkey( &ctx->crypt, key1, key1bits ); -} - -int esp_aes_xts_setkey_dec( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits) -{ - int ret; - const unsigned char *key1, *key2; - unsigned int key1bits, key2bits; - - ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, - &key2, &key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set the tweak key. Always set tweak key for encryption. */ - ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set crypt key for decryption. */ - return esp_aes_setkey( &ctx->crypt, key1, key1bits ); -} - -/* Endianess with 64 bits values */ -#ifndef GET_UINT64_LE -#define GET_UINT64_LE(n,b,i) \ -{ \ - (n) = ( (uint64_t) (b)[(i) + 7] << 56 ) \ - | ( (uint64_t) (b)[(i) + 6] << 48 ) \ - | ( (uint64_t) (b)[(i) + 5] << 40 ) \ - | ( (uint64_t) (b)[(i) + 4] << 32 ) \ - | ( (uint64_t) (b)[(i) + 3] << 24 ) \ - | ( (uint64_t) (b)[(i) + 2] << 16 ) \ - | ( (uint64_t) (b)[(i) + 1] << 8 ) \ - | ( (uint64_t) (b)[(i) ] ); \ -} -#endif - -#ifndef PUT_UINT64_LE -#define PUT_UINT64_LE(n,b,i) \ -{ \ - (b)[(i) + 7] = (unsigned char) ( (n) >> 56 ); \ - (b)[(i) + 6] = (unsigned char) ( (n) >> 48 ); \ - (b)[(i) + 5] = (unsigned char) ( (n) >> 40 ); \ - (b)[(i) + 4] = (unsigned char) ( (n) >> 32 ); \ - (b)[(i) + 3] = (unsigned char) ( (n) >> 24 ); \ - (b)[(i) + 2] = (unsigned char) ( (n) >> 16 ); \ - (b)[(i) + 1] = (unsigned char) ( (n) >> 8 ); \ - (b)[(i) ] = (unsigned char) ( (n) ); \ -} -#endif - -typedef unsigned char esp_be128[16]; - -/* - * GF(2^128) multiplication function - * - * This function multiplies a field element by x in the polynomial field - * representation. It uses 64-bit word operations to gain speed but compensates - * for machine endianess and hence works correctly on both big and little - * endian machines. - */ -static void esp_gf128mul_x_ble( unsigned char r[16], - const unsigned char x[16] ) -{ - uint64_t a, b, ra, rb; - - GET_UINT64_LE( a, x, 0 ); - GET_UINT64_LE( b, x, 8 ); - - ra = ( a << 1 ) ^ 0x0087 >> ( 8 - ( ( b >> 63 ) << 3 ) ); - rb = ( a >> 63 ) | ( b << 1 ); - - PUT_UINT64_LE( ra, r, 0 ); - PUT_UINT64_LE( rb, r, 8 ); -} - -/* - * AES-XTS buffer encryption/decryption - */ -int esp_aes_crypt_xts( esp_aes_xts_context *ctx, - int mode, - size_t length, - const unsigned char data_unit[16], - const unsigned char *input, - unsigned char *output ) -{ - int ret; - size_t blocks = length / 16; - size_t leftover = length % 16; - unsigned char tweak[16]; - unsigned char prev_tweak[16]; - unsigned char tmp[16]; - - /* Sectors must be at least 16 bytes. */ - if( length < 16 ) - return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; - - /* NIST SP 80-38E disallows data units larger than 2**20 blocks. */ - if( length > ( 1 << 20 ) * 16 ) - return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; - - /* Compute the tweak. */ - ret = esp_aes_crypt_ecb( &ctx->tweak, MBEDTLS_AES_ENCRYPT, - data_unit, tweak ); - if( ret != 0 ) - return( ret ); - - while( blocks-- ) - { - size_t i; - - if( leftover && ( mode == MBEDTLS_AES_DECRYPT ) && blocks == 0 ) - { - /* We are on the last block in a decrypt operation that has - * leftover bytes, so we need to use the next tweak for this block, - * and this tweak for the lefover bytes. Save the current tweak for - * the leftovers and then update the current tweak for use on this, - * the last full block. */ - memcpy( prev_tweak, tweak, sizeof( tweak ) ); - esp_gf128mul_x_ble( tweak, tweak ); - } - - for( i = 0; i < 16; i++ ) - tmp[i] = input[i] ^ tweak[i]; - - ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); - if( ret != 0 ) - return( ret ); - - for( i = 0; i < 16; i++ ) - output[i] = tmp[i] ^ tweak[i]; - - /* Update the tweak for the next block. */ - esp_gf128mul_x_ble( tweak, tweak ); - - output += 16; - input += 16; - } - - if( leftover ) - { - /* If we are on the leftover bytes in a decrypt operation, we need to - * use the previous tweak for these bytes (as saved in prev_tweak). */ - unsigned char *t = mode == MBEDTLS_AES_DECRYPT ? prev_tweak : tweak; - - /* We are now on the final part of the data unit, which doesn't divide - * evenly by 16. It's time for ciphertext stealing. */ - size_t i; - unsigned char *prev_output = output - 16; - - /* Copy ciphertext bytes from the previous block to our output for each - * byte of cyphertext we won't steal. At the same time, copy the - * remainder of the input for this final round (since the loop bounds - * are the same). */ - for( i = 0; i < leftover; i++ ) - { - output[i] = prev_output[i]; - tmp[i] = input[i] ^ t[i]; - } - - /* Copy ciphertext bytes from the previous block for input in this - * round. */ - for( ; i < 16; i++ ) - tmp[i] = prev_output[i] ^ t[i]; - - ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); - if( ret != 0 ) - return ret; - - /* Write the result back to the previous block, overriding the previous - * output we copied. */ - for( i = 0; i < 16; i++ ) - prev_output[i] = tmp[i] ^ t[i]; - } - - return( 0 ); -} +} \ No newline at end of file diff --git a/components/mbedtls/port/esp_sha1.c b/components/mbedtls/port/esp32/esp_sha1.c similarity index 100% rename from components/mbedtls/port/esp_sha1.c rename to components/mbedtls/port/esp32/esp_sha1.c diff --git a/components/mbedtls/port/esp_sha256.c b/components/mbedtls/port/esp32/esp_sha256.c similarity index 100% rename from components/mbedtls/port/esp_sha256.c rename to components/mbedtls/port/esp32/esp_sha256.c diff --git a/components/mbedtls/port/esp_sha512.c b/components/mbedtls/port/esp32/esp_sha512.c similarity index 100% rename from components/mbedtls/port/esp_sha512.c rename to components/mbedtls/port/esp32/esp_sha512.c diff --git a/components/mbedtls/port/esp32s2/aes.c b/components/mbedtls/port/esp32s2/aes.c index 47e33d7ec..ef3efa90b 100644 --- a/components/mbedtls/port/esp32s2/aes.c +++ b/components/mbedtls/port/esp32s2/aes.c @@ -1,9 +1,9 @@ /** - * \brief AES block cipher, ESP32 hardware accelerated version + * \brief AES block cipher, ESP32-S2 hardware accelerated version * Based on mbedTLS FIPS-197 compliant version. * * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved - * Additions Copyright (C) 2016-2017, Espressif Systems (Shanghai) PTE Ltd + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE Ltd * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -25,125 +25,104 @@ * http://csrc.nist.gov/encryption/aes/rijndael/Rijndael.pdf * http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf */ + #include #include #include #include "mbedtls/aes.h" #include "esp32s2/aes.h" #include "esp32s2/gcm.h" -#include "soc/soc.h" #include "soc/cpu.h" #include "soc/dport_reg.h" #include "soc/hwcrypto_reg.h" #include "soc/crypto_dma_reg.h" +#include "soc/periph_defs.h" #include "esp32s2/crypto_dma.h" #include "esp32s2/rom/lldesc.h" #include "esp32s2/rom/cache.h" -#include "soc/periph_defs.h" #include "esp_intr_alloc.h" +#include "driver/periph_ctrl.h" +#include "esp_log.h" +#include "soc/lldesc.h" +#include "esp_heap_caps.h" +#include "sys/param.h" #include "freertos/FreeRTOS.h" -#include "freertos/task.h" #include "freertos/semphr.h" #define AES_BLOCK_BYTES 16 +#define IV_WORDS 4 -#define LE_TO_BE(x) (((x) >> 24) & 0x000000ff) | \ - (((x) << 24) & 0xff000000) | \ - (((x) >> 8) & 0x0000ff00) | \ - (((x) << 8) & 0x00ff0000) +/* Max size of each chunk to process when output buffer is in unaligned external ram + must be a multiple of block size +*/ +#define AES_MAX_CHUNK_WRITE_SIZE 1600 -static inline uint32_t WPA_GET_BE32(const uint8_t *a) -{ - return ((uint32_t) a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]; -} +/* Input over this length will yield and wait for interrupt instead of + busy-waiting, 30000 bytes is approx 0.5 ms */ +#define AES_DMA_INTR_TRIG_LEN 2000 -static inline void WPA_PUT_BE32(uint8_t *a, uint32_t val) -{ - a[0] = (val >> 24) & 0xff; - a[1] = (val >> 16) & 0xff; - a[2] = (val >> 8) & 0xff; - a[3] = val & 0xff; -} +#define ESP_GET_BE32(a) __builtin_bswap32( *(uint32_t*)(a) ) + +#define ESP_PUT_BE32(a, val) \ + do { \ + *(uint32_t*)(a) = __builtin_bswap32( (uint32_t)(val) ); \ + } while (0) + +#define ESP_PUT_BE64(a, val) \ + do { \ + *(uint64_t*)(a) = __builtin_bswap64( (uint64_t)(val) ); \ + } while (0) -/* Enable this if want to use AES interrupt */ -//#define CONFIG_MBEDTLS_AES_USE_INTERRUPT +/* DMA AES working modes*/ +typedef enum { + ESP_AES_BLOCK_MODE_ECB = 0, + ESP_AES_BLOCK_MODE_CBC, + ESP_AES_BLOCK_MODE_OFB, + ESP_AES_BLOCK_MODE_CTR, + ESP_AES_BLOCK_MODE_CFB8, + ESP_AES_BLOCK_MODE_CFB128, + ESP_AES_BLOCK_MODE_GCM, +} esp_aes_mode_t; -portMUX_TYPE crypto_dma_spinlock = portMUX_INITIALIZER_UNLOCKED; #if defined(CONFIG_MBEDTLS_AES_USE_INTERRUPT) static SemaphoreHandle_t op_complete_sem; #endif -/* AES uses a spinlock mux not a lock as the underlying block operation - only takes a small number of cycles, much less than using - a mutex for this. +_lock_t crypto_dma_lock; +static _lock_t aes_lock; - For CBC, CFB, etc. this may mean that interrupts are disabled for a longer - period of time for bigger data lengths. -*/ -portMUX_TYPE aes_spinlock = portMUX_INITIALIZER_UNLOCKED; - -/* The function will pad 0 if the length is not multiple of - * 16 bytes for the incoming data buffer - */ -static uint8_t *textpad_zero(const unsigned char *buf, unsigned char *len) -{ - uint8_t offset; - uint8_t *data = NULL; - - offset = *len % AES_BLOCK_BYTES; - - if (offset) { - data = (uint8_t *)calloc(1, (*len + (AES_BLOCK_BYTES - offset))); - if (data) { - memcpy(data, buf, *len); - *len += (AES_BLOCK_BYTES - offset); - } - } - - return data; -} +static const char *TAG = "esp-aes"; static inline bool valid_key_length(const esp_aes_context *ctx) { - return ctx->key_bytes == 128/8 || ctx->key_bytes == 192/8 || ctx->key_bytes == 256/8; + return ctx->key_bytes == 128 / 8 || ctx->key_bytes == 192 / 8 || ctx->key_bytes == 256 / 8; } void esp_aes_acquire_hardware( void ) { - /* newlib locks lazy initialize on ESP-IDF */ - portENTER_CRITICAL(&aes_spinlock); - /* Need to lock DMA since it is shared with SHA block */ - portENTER_CRITICAL(&crypto_dma_spinlock); + _lock_acquire(&aes_lock); + _lock_acquire(&crypto_dma_lock); /* Enable AES hardware */ - REG_SET_BIT(DPORT_PERIP_CLK_EN1_REG, DPORT_CRYPTO_AES_CLK_EN | DPORT_CRYPTO_DMA_CLK_EN); - /* Clear reset on digital signature unit, - otherwise AES unit is held in reset also. */ - REG_CLR_BIT(DPORT_PERIP_RST_EN1_REG, - DPORT_CRYPTO_AES_RST | DPORT_CRYPTO_DMA_RST | DPORT_CRYPTO_DS_RST); + periph_module_enable(PERIPH_AES_DMA_MODULE); } /* Function to disable AES and Crypto DMA clocks and release locks */ void esp_aes_release_hardware( void ) { - /* Disable DMA mode */ - REG_WRITE(AES_DMA_ENABLE_REG, 0); /* Disable AES hardware */ - REG_SET_BIT(DPORT_PERIP_RST_EN1_REG, DPORT_CRYPTO_AES_RST | DPORT_CRYPTO_DMA_RST); - /* Don't return other units to reset, as this pulls - reset on RSA & SHA units, respectively. */ - REG_CLR_BIT(DPORT_PERIP_CLK_EN1_REG, DPORT_CRYPTO_AES_CLK_EN | DPORT_CRYPTO_DMA_CLK_EN); + periph_module_disable(PERIPH_AES_DMA_MODULE); - portEXIT_CRITICAL(&crypto_dma_spinlock); - - portEXIT_CRITICAL(&aes_spinlock); + _lock_release(&crypto_dma_lock); + _lock_release(&aes_lock); } + /* Function to init AES context to zero */ void esp_aes_init( esp_aes_context *ctx ) { @@ -182,105 +161,70 @@ int esp_aes_setkey( esp_aes_context *ctx, const unsigned char *key, /* * Helper function to copy key from esp_aes_context buffer - * to hardware key registers. Also, set the AES block mode - * (ECb, CBC, CFB, OFB, GCM, CTR) and crypt mode (ENCRYPT/DECRYPT) - * Enable the DMA mode of operation + * to hardware key registers. * * Call only while holding esp_aes_acquire_hardware(). */ -static inline void esp_aes_setkey_hardware( esp_aes_context *ctx, int crypt_mode, uint8_t block_mode) +static void esp_aes_setkey_hardware( esp_aes_context *ctx, int crypt_mode) { const uint32_t MODE_DECRYPT_BIT = 4; unsigned mode_reg_base = (crypt_mode == ESP_AES_ENCRYPT) ? 0 : MODE_DECRYPT_BIT; ctx->key_in_hardware = 0; - for (int i = 0; i < ctx->key_bytes/4; ++i) { + for (int i = 0; i < ctx->key_bytes / 4; ++i) { REG_WRITE(AES_KEY_BASE + i * 4, *(((uint32_t *)ctx->key) + i)); ctx->key_in_hardware += 4; } REG_WRITE(AES_MODE_REG, mode_reg_base + ((ctx->key_bytes / 8) - 2)); - /* Set the algorithm mode CBC, CFB ... */ - REG_WRITE(AES_BLOCK_MODE_REG, block_mode); - - /* Set the ENDIAN reg */ - REG_WRITE(AES_ENDIAN_REG, 0x3F); - - /* Enable DMA mode */ - REG_WRITE(AES_DMA_ENABLE_REG, 1); - - /* Presently hard-coding the INC function to 32 bit */ - if (block_mode == AES_BLOCK_MODE_CTR) { - REG_WRITE(AES_INC_SEL_REG, 0); - } - /* Fault injection check: all words of key data should have been written to hardware */ if (ctx->key_in_hardware < 16 - || ctx->key_in_hardware != ctx->key_bytes) { + || ctx->key_in_hardware != ctx->key_bytes) { abort(); } } /* - * Function to write IV to hardware iv registers + * Sets the AES DMA block mode (ECB, CBC, CFB, OFB, GCM, CTR) + * and intializes the required registers for that working mode */ -static inline void esp_aes_set_iv(uint8_t *iv, uint8_t len) +static inline void esp_aes_mode_init(esp_aes_mode_t mode) { - memcpy((uint8_t *)AES_IV_BASE, iv, len); + /* Set the algorithm mode CBC, CFB ... */ + REG_WRITE(AES_BLOCK_MODE_REG, mode); + + /* Presently hard-coding the INC function to 32 bit */ + if (mode == ESP_AES_BLOCK_MODE_CTR) { + REG_WRITE(AES_INC_SEL_REG, 0); + } } /* - * Function to read IV from hardware iv registers + * Write IV to hardware iv registers */ -static inline void esp_aes_get_iv(uint8_t *iv, uint8_t len) +static inline void esp_aes_set_iv(uint8_t *iv) { - memcpy(iv, (uint8_t *)AES_IV_BASE, len); -} + uint32_t *iv_words = (uint32_t*)iv; + uint32_t *reg_addr_buf = (uint32_t *)(AES_IV_BASE); -/* Function to set block number & trigger bit for AES GCM operation -*/ -static inline void esp_aes_gcm_set_block_num_and_trigger(uint16_t len) -{ - /* Write the number of blocks */ - REG_WRITE(AES_BLOCK_NUM_REG, (len / AES_BLOCK_BYTES)); - - REG_WRITE(AES_TRIGGER_REG, 1); - while (REG_READ(AES_STATE_REG) != AES_STATE_IDLE) { + for (int i = 0; iowner == 0) ) { + break; + } + } } -/* Perform AES-DMA operation and wait until the DMA operation is over - * - * input = Input data buffer, length len - * output = Output data buffer, length len - * len = Length of data in bytes, may not be multiple of AES block size - * stream_out = Pointer to 16 byte buffer to hold final stream block, if - * len is not a multiple of AES block size (16) - * - * The DMA processing works in following way: - * - * - If len >= AES_BLOCK_BYTES, there are DMA input and output - * descriptors which point to input & output (in_block_desc, out_block_desc) directly, - * to process block_bytes bytes. - * - * - If len % AES_BLOCK_BYTES != 0 then unaligned bytes are copied to stream_in block which is - * padded with zeroes. DMA descriptors (stream_in_desc, stream_out_desc) process this block and output to - * stream_out buffer argument. Otherwise, stream_out argument is ignored (may be NULL). - * - * - If both above conditions are true, DMA has two linked list input buffers and two linked list output buffers, - * and processes first the blocks and then the partial stream block. - * - * After the DMA operation, if we only processed full bytes then the IV memory block will contain the next IV - * for the configured AES mode. - * - * If a partial "stream block" was processed then IV memory block will contain a garbage IV value, because of - * the partial stream block. The IV can be recovered from the stream_out block (depending on algorithm some - * post-processing of the 'output' bytes also in stream_out may be needed, to translate them back to correct IV bytes). - * - */ -static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, uint16_t len, uint8_t *stream_out) +/* Init DMA related registers for AES operation */ +static void esp_aes_dma_init(lldesc_t *input, lldesc_t *output) { - volatile lldesc_t block_in_desc, block_out_desc, stream_in_desc, stream_out_desc; - volatile lldesc_t *in_desc_head, *out_desc_head; - volatile uint32_t dma_done = 0; - uint8_t stream_in[16]; - unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block - unsigned block_bytes = len - stream_bytes; // bytes which are in a full block - unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0); - - assert(len > 0); // caller shouldn't ever have len set to zero - assert(stream_bytes == 0 || stream_out != NULL); // stream_out can be NULL if we're processing full block(s) - - /* If no key is written to hardware yet, either the user hasn't called - mbedtls_aes_setkey_enc/mbedtls_aes_setkey_dec - meaning we also don't - know which mode to use - or a fault skipped the - key write to hardware. Treat this as a fatal error and zero the output block. - */ - if (ctx->key_in_hardware != ctx->key_bytes) { - bzero(output, 16); - return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; - } - - if (block_bytes > 0) { - /* If the block length is less than 16 we use internal RAM so no - * need to flush Cache - */ -#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) - if ((unsigned int)input >= SOC_EXTRAM_DATA_LOW && (unsigned int)input <= SOC_EXTRAM_DATA_HIGH) { - assert((((unsigned int)(input) & 0xF) == 0)); - Cache_WriteBack_All(); - } - if ((unsigned int)output >= SOC_EXTRAM_DATA_LOW && (unsigned int)output <= SOC_EXTRAM_DATA_HIGH) { - assert((((unsigned int)(output) & 0xF) == 0)); - } -#endif - block_in_desc = (lldesc_t) { - .length = block_bytes, - .size = block_bytes, - .buf = (void *)input, - .owner = 1, - .empty = (stream_bytes > 0) ? (uint32_t)&stream_in_desc : 0, - .eof = (stream_bytes == 0), - }; - - block_out_desc = (lldesc_t) { - .length = block_bytes, - .size = block_bytes, - .buf = output, - .owner = 1, - .empty = (stream_bytes > 0) ? (uint32_t)&stream_out_desc : 0, - .eof = (stream_bytes == 0), - }; - - in_desc_head = &block_in_desc; - out_desc_head = &block_out_desc; - } - - if (stream_bytes > 0) { - // can't read past end of 'input', so make a zero padded input buffer in RAM - memcpy(stream_in, input + block_bytes, stream_bytes); - bzero(stream_in + stream_bytes, AES_BLOCK_BYTES - stream_bytes); - - stream_in_desc = (lldesc_t) { - .length = AES_BLOCK_BYTES, - .size = AES_BLOCK_BYTES, - .buf = stream_in, - .owner = 1, - .eof = 1, - }; - - stream_out_desc = (lldesc_t) { - .length = AES_BLOCK_BYTES, - .size = AES_BLOCK_BYTES, - .buf = stream_out, - .owner = 1, - .eof = 1, - }; - } - - // block buffers are sent to DMA first, unless there aren't any - in_desc_head = (block_bytes > 0) ? &block_in_desc : &stream_in_desc; - out_desc_head = (block_bytes > 0) ? &block_out_desc : &stream_out_desc; - - /* Enable the DMA clock - currently only for FPGA test */ -#if CONFIG_IDF_ENV_FPGA - SET_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_GEN_CLK_EN); -#endif + /* Enable DMA mode */ + REG_WRITE(AES_DMA_ENABLE_REG, 1); /* Reset DMA */ SET_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_AHBM_RST | CONF0_REG_IN_RST | CONF0_REG_OUT_RST | CONF0_REG_AHBM_FIFO_RST); @@ -436,55 +292,225 @@ static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, /* Set descriptors */ CLEAR_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_ADDR); - SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, ((uint32_t)(in_desc_head))&OUT_LINK_REG_OUTLINK_ADDR); + SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, ((uint32_t)(input))&OUT_LINK_REG_OUTLINK_ADDR); CLEAR_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, IN_LINK_REG_INLINK_ADDR); - SET_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, ((uint32_t)(out_desc_head))&IN_LINK_REG_INLINK_ADDR); + SET_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, ((uint32_t)(output))&IN_LINK_REG_INLINK_ADDR); /* Start transfer */ SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_START); SET_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, IN_LINK_REG_INLINK_START); +} + +static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out); + + +/* Output buffers in external ram needs to be 16-byte aligned and DMA cant access input in the iCache mem range, + reallocate them into internal memory and encrypt in chunks to avoid + having to malloc too big of a buffer +*/ + +static int esp_aes_process_dma_ext_ram(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out, bool realloc_input, bool realloc_output) +{ + size_t chunk_len; + int offset = 0; + unsigned char *input_buf = NULL; + unsigned char *output_buf = NULL; + const unsigned char *dma_input; + + chunk_len = MIN(AES_MAX_CHUNK_WRITE_SIZE, len); + + if (realloc_input) { + input_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA); + + if (input_buf == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory"); + return -1; + } + } + + if (realloc_output) { + output_buf = heap_caps_malloc(chunk_len, MALLOC_CAP_DMA); + + if (output_buf == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory"); + return -1; + } + } else { + output_buf = output; + } + + while (len) { + chunk_len = MIN(AES_MAX_CHUNK_WRITE_SIZE, len); + + /* If input needs realloc then copy it, else use the input with offset*/ + if (realloc_input) { + memcpy(input_buf, input + offset, chunk_len); + dma_input = input_buf; + } else { + dma_input = input + offset; + } + + if (esp_aes_process_dma(ctx, dma_input, output_buf, chunk_len, stream_out) != 0) { + return -1; + } + + if (realloc_output) { + memcpy(output + offset, output_buf, chunk_len); + } else { + output_buf = output + offset + chunk_len; + } + + len -= chunk_len; + offset += chunk_len; + } + + if (realloc_input) { + free(input_buf); + } + if (realloc_output) { + free(output_buf); + } + + return 0; +} + +/* Encrypt/decrypt the input using DMA */ +static int esp_aes_process_dma(esp_aes_context *ctx, const unsigned char *input, unsigned char *output, size_t len, uint8_t *stream_out) +{ + lldesc_t stream_in_desc, stream_out_desc; + lldesc_t *in_desc_head, *out_desc_head; + lldesc_t *block_desc = NULL, *block_in_desc, *block_out_desc; + size_t lldesc_num; + uint8_t stream_in[16] = {}; + unsigned stream_bytes = len % AES_BLOCK_BYTES; // bytes which aren't in a full block + unsigned block_bytes = len - stream_bytes; // bytes which are in a full block + unsigned char *non_icache_input = NULL; + unsigned blocks = (block_bytes / AES_BLOCK_BYTES) + ((stream_bytes > 0) ? 1 : 0); + bool use_intr = false; + bool input_needs_realloc = false; + bool output_needs_realloc = false; + int ret = 0; + + assert(len > 0); // caller shouldn't ever have len set to zero + assert(stream_bytes == 0 || stream_out != NULL); // stream_out can be NULL if we're processing full block(s) + + /* If no key is written to hardware yet, either the user hasn't called + mbedtls_aes_setkey_enc/mbedtls_aes_setkey_dec - meaning we also don't + know which mode to use - or a fault skipped the + key write to hardware. Treat this as a fatal error and zero the output block. + */ + if (ctx->key_in_hardware != ctx->key_bytes) { + bzero(output, len); + return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; + } + + if (block_bytes > 0) { + /* Flush cache if input in external ram */ +#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) + if (esp_ptr_external_ram(input)) { + Cache_WriteBack_All(); + } + if (esp_ptr_external_ram(output)) { + if (((intptr_t)(output) & 0xF) != 0) { + // Non aligned ext-mem buffer + output_needs_realloc = true; + } + } +#endif + /* DMA cannot access memory in the iCache range, copy input to internal ram */ + if (!esp_ptr_dma_ext_capable(input) && !esp_ptr_dma_capable(input)) { + input_needs_realloc = true; + } + + /* If either input or output is unaccessible to the DMA then they need to be reallocated */ + if (input_needs_realloc || output_needs_realloc) { + return esp_aes_process_dma_ext_ram(ctx, input, output, len, stream_out, input_needs_realloc, output_needs_realloc); + } + + + /* Set up dma descriptors for input and output */ + lldesc_num = lldesc_get_required_num(block_bytes); + + /* Allocate both in and out descriptors to save a malloc/free per function call */ + block_desc = heap_caps_malloc(sizeof(lldesc_t) * lldesc_num * 2, MALLOC_CAP_DMA); + if (block_desc == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory"); + ret = -1; + goto cleanup; + } + + block_in_desc = block_desc; + block_out_desc = block_desc + lldesc_num; + + lldesc_setup_link(block_desc, input, block_bytes, 0); + lldesc_setup_link(block_desc + lldesc_num, output, block_bytes, 0); + } + + /* Any leftover bytes which are appended as an additional DMA list */ + if (stream_bytes > 0) { + memcpy(stream_in, input + block_bytes, stream_bytes); + + lldesc_setup_link(&stream_in_desc, stream_in, AES_BLOCK_BYTES, 0); + lldesc_setup_link(&stream_out_desc, stream_out, AES_BLOCK_BYTES, 0); + + if (block_bytes > 0) { + /* Link with block descriptors*/ + block_in_desc[lldesc_num - 1].empty = (uint32_t)&stream_in_desc; + block_out_desc[lldesc_num - 1].empty = (uint32_t)&stream_out_desc; + } + } + + // block buffers are sent to DMA first, unless there aren't any + in_desc_head = (block_bytes > 0) ? block_in_desc : &stream_in_desc; + out_desc_head = (block_bytes > 0) ? block_out_desc : &stream_out_desc; + + esp_aes_dma_init(in_desc_head, out_desc_head); /* Write the number of blocks */ REG_WRITE(AES_BLOCK_NUM_REG, blocks); + #if defined (CONFIG_MBEDTLS_AES_USE_INTERRUPT) - REG_WRITE(AES_INT_CLR_REG, 1); - if (op_complete_sem == NULL) { - op_complete_sem = xSemaphoreCreateBinary(); - esp_intr_alloc(ETS_AES_INTR_SOURCE, 0, esp_aes_dma_isr, 0, 0); - } - REG_WRITE(AES_INT_ENA_REG, 1); + /* Only use interrupt for long AES operations */ + if (len > AES_DMA_INTR_TRIG_LEN) { + use_intr = true; + esp_aes_isr_initialise(); + } else #endif + { + REG_WRITE(AES_INT_ENA_REG, 0); + } /* Start AES operation */ REG_WRITE(AES_TRIGGER_REG, 1); - esp_aes_dma_complete(); + esp_aes_dma_wait_complete(use_intr, out_desc_head); - /* Wait for DMA operation to complete */ - while (1) { - dma_done = REG_READ(CRYPTO_DMA_INT_RAW_REG); - if ( (dma_done & INT_RAW_IN_SUC_EOF) == INT_RAW_IN_SUC_EOF) { - break; - } - } - - REG_WRITE(AES_DMA_EXIT_REG, 1); #if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) if (block_bytes > 0) { - if ((unsigned int)input >= SOC_EXTRAM_DATA_LOW && (unsigned int)input <= SOC_EXTRAM_DATA_HIGH) { + if (esp_ptr_external_ram(output)) { Cache_Invalidate_DCache_All(); } } #endif + REG_WRITE(AES_DMA_EXIT_REG, 0); + /* Disable DMA mode */ + REG_WRITE(AES_DMA_ENABLE_REG, 0); + if (stream_bytes > 0) { memcpy(output + block_bytes, stream_out, stream_bytes); } - return 0; + +cleanup: + free(non_icache_input); + free(block_desc); + return ret; } + + /* * AES-ECB single block encryption */ @@ -499,7 +525,9 @@ int esp_internal_aes_encrypt( esp_aes_context *ctx, } esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, ESP_AES_ENCRYPT, AES_BLOCK_MODE_ECB); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, ESP_AES_ENCRYPT); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_ECB); r = esp_aes_process_dma(ctx, input, output, AES_BLOCK_BYTES, NULL); esp_aes_release_hardware(); @@ -527,7 +555,9 @@ int esp_internal_aes_decrypt( esp_aes_context *ctx, } esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT, AES_BLOCK_MODE_ECB); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_ECB); r = esp_aes_process_dma(ctx, input, output, AES_BLOCK_BYTES, NULL); esp_aes_release_hardware(); @@ -558,7 +588,8 @@ int esp_aes_crypt_ecb( esp_aes_context *ctx, esp_aes_acquire_hardware(); ctx->key_in_hardware = 0; - esp_aes_setkey_hardware(ctx, mode, AES_BLOCK_MODE_ECB); + esp_aes_setkey_hardware(ctx, mode); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_ECB); r = esp_aes_process_dma(ctx, input, output, AES_BLOCK_BYTES, NULL); esp_aes_release_hardware(); @@ -575,28 +606,35 @@ int esp_aes_crypt_cbc( esp_aes_context *ctx, const unsigned char *input, unsigned char *output ) { + int r = 0; + /* For CBC input length should be multiple of * AES BLOCK BYTES * */ if ( length % AES_BLOCK_BYTES ) { return ERR_ESP_AES_INVALID_INPUT_LENGTH; } - if ( length == 0 ) { - return 0; - } + if (!valid_key_length(ctx)) { return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; } esp_aes_acquire_hardware(); ctx->key_in_hardware = 0; - esp_aes_setkey_hardware(ctx, mode, AES_BLOCK_MODE_CBC); - esp_aes_set_iv(iv, AES_BLOCK_BYTES); - esp_aes_process_dma(ctx, input, output, length, NULL); - esp_aes_get_iv(iv, AES_BLOCK_BYTES); + esp_aes_setkey_hardware(ctx, mode); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_CBC); + esp_aes_set_iv(iv); + + r = esp_aes_process_dma(ctx, input, output, length, NULL); + if (r != 0) { + esp_aes_release_hardware(); + return r; + } + + esp_aes_get_iv(iv); esp_aes_release_hardware(); - return 0; + return r; } /* @@ -611,18 +649,31 @@ int esp_aes_crypt_cfb8( esp_aes_context *ctx, { unsigned char c; unsigned char ov[17]; + int r = 0; size_t block_bytes = length - (length % AES_BLOCK_BYTES); + if (!valid_key_length(ctx)) { + return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; + } + /* The DMA engine will only output correct IV if it runs full blocks of input in CFB8 mode */ + esp_aes_acquire_hardware(); + if (block_bytes > 0) { - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, mode, AES_BLOCK_MODE_CFB8); - esp_aes_set_iv(iv, AES_BLOCK_BYTES); - esp_aes_process_dma(ctx, input, output, block_bytes, NULL); - esp_aes_get_iv(iv, AES_BLOCK_BYTES); - esp_aes_release_hardware(); + + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, mode); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_CFB8); + esp_aes_set_iv(iv); + r = esp_aes_process_dma(ctx, input, output, block_bytes, NULL); + esp_aes_get_iv(iv); + + if (r != 0) { + esp_aes_release_hardware(); + return r; + } length -= block_bytes; input += block_bytes; @@ -631,12 +682,18 @@ int esp_aes_crypt_cfb8( esp_aes_context *ctx, // Process remaining bytes block-at-a-time in ECB mode if (length > 0) { - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, MBEDTLS_AES_ENCRYPT, AES_BLOCK_MODE_ECB); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, MBEDTLS_AES_ENCRYPT); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_ECB); while ( length-- ) { memcpy( ov, iv, 16 ); - esp_aes_process_dma(ctx, iv, iv, AES_BLOCK_BYTES, NULL); + + r = esp_aes_process_dma(ctx, iv, iv, AES_BLOCK_BYTES, NULL); + if (r != 0) { + esp_aes_release_hardware(); + return r; + } if ( mode == MBEDTLS_AES_DECRYPT ) { ov[16] = *input; @@ -649,10 +706,11 @@ int esp_aes_crypt_cfb8( esp_aes_context *ctx, } memcpy( iv, ov + 1, 16 ); } - esp_aes_release_hardware(); - } - return 0; + } + esp_aes_release_hardware(); + + return r; } /* @@ -668,6 +726,7 @@ int esp_aes_crypt_cfb128( esp_aes_context *ctx, { uint8_t c; + int r = 0; size_t stream_bytes = 0; size_t n = *iv_off; @@ -675,18 +734,15 @@ int esp_aes_crypt_cfb128( esp_aes_context *ctx, return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; } - esp_aes_acquire_hardware(); - ctx->key_in_hardware = 0; - - /* Lets process the *iv_off bytes first + /* First process the *iv_off bytes * which are pending from the previous call to this API */ while (n > 0 && length > 0) { if (mode == MBEDTLS_AES_ENCRYPT) { - iv[n] = *output++ = (unsigned char)(*input++ ^ iv[n]); + iv[n] = *output++ = *input++ ^ iv[n]; } else { c = *input++; - *output++ = (c ^ iv[n]); + *output++ = c ^ iv[n]; iv[n] = c; } n = (n + 1) % AES_BLOCK_BYTES; @@ -696,16 +752,21 @@ int esp_aes_crypt_cfb128( esp_aes_context *ctx, if (length > 0) { stream_bytes = length % AES_BLOCK_BYTES; - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, mode, AES_BLOCK_MODE_CFB128); - esp_aes_set_iv(iv, AES_BLOCK_BYTES); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, mode); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_CFB128); + esp_aes_set_iv(iv); - esp_aes_process_dma(ctx, input, output, length, iv); + r = esp_aes_process_dma(ctx, input, output, length, iv); + if (r != 0) { + esp_aes_release_hardware(); + return r; + } if (stream_bytes == 0) { // if we didn't need the partial 'stream block' then the new IV is in the IV register - esp_aes_get_iv(iv, AES_BLOCK_BYTES); + esp_aes_get_iv(iv); } else { // if we did process a final partial block the new IV is already processed via DMA (and has some bytes of output in it), // In decrypt mode any partial bytes are output plaintext (iv ^ c) and need to be swapped back to ciphertext (as the next @@ -720,8 +781,52 @@ int esp_aes_crypt_cfb128( esp_aes_context *ctx, } *iv_off = n + stream_bytes; + return r; +} - return 0; +/* + * AES-OFB (Output Feedback Mode) buffer encryption/decryption + */ + +int esp_aes_crypt_ofb( esp_aes_context *ctx, + size_t length, + size_t *iv_off, + unsigned char iv[16], + const unsigned char *input, + unsigned char *output ) +{ + int r = 0; + size_t n = *iv_off; + size_t stream_bytes = 0; + /* If there is an offset then use the output of the previous AES block + (the updated IV) to calculate the new output */ + while (n > 0 && length > 0) { + *output++ = (*input++ ^ iv[n]); + n = (n + 1) & 0xF; + length--; + } + if (length > 0) { + stream_bytes = (length % AES_BLOCK_BYTES); + + esp_aes_acquire_hardware(); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_OFB); + esp_aes_set_iv(iv); + + r = esp_aes_process_dma(ctx, input, output, length, iv); + if (r != 0) { + esp_aes_release_hardware(); + return r; + } + + esp_aes_get_iv(iv); + esp_aes_release_hardware(); + } + + *iv_off = n + stream_bytes; + + return r; } /* @@ -735,8 +840,15 @@ int esp_aes_crypt_ctr( esp_aes_context *ctx, const unsigned char *input, unsigned char *output ) { + int r = 0; size_t n = *nc_off; + if (!valid_key_length(ctx)) { + return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH; + } + + /* Process any unprocessed bytes left in stream block from + last operation */ while (n > 0 && length > 0) { *output++ = (unsigned char)(*input++ ^ stream_block[n]); n = (n + 1) & 0xF; @@ -744,93 +856,160 @@ int esp_aes_crypt_ctr( esp_aes_context *ctx, } if (length > 0) { - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT, AES_BLOCK_MODE_CTR); - esp_aes_set_iv(nonce_counter, AES_BLOCK_BYTES); - esp_aes_process_dma(ctx, input, output, length, stream_block); - esp_aes_get_iv(nonce_counter, AES_BLOCK_BYTES); - esp_aes_release_hardware(); - } + esp_aes_acquire_hardware(); + ctx->key_in_hardware = 0; + esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT); + + esp_aes_mode_init(ESP_AES_BLOCK_MODE_CTR); + esp_aes_set_iv(nonce_counter); + + r = esp_aes_process_dma(ctx, input, output, length, stream_block); + + if (r != 0) { + esp_aes_release_hardware(); + return r; + } + + esp_aes_get_iv(nonce_counter); + + esp_aes_release_hardware(); + + } *nc_off = n + (length % AES_BLOCK_BYTES); - return 0; + return r; } -/* XOR two 32 bit words */ -static void xor_block(uint8_t *dst, const uint8_t *src) -{ - uint32_t *d = (uint32_t *) dst; - uint32_t *s = (uint32_t *) src; +static void esp_gcm_ghash(uint8_t *h0, const unsigned char *x, size_t x_len, uint8_t *j0); - *d++ ^= *s++; - *d++ ^= *s++; - *d++ ^= *s++; - *d++ ^= *s++; +/* + * Calculates the Initial Counter Block, J0 + * and copies to to the esp_gcm_context + */ +static void esp_gcm_derive_J0(esp_gcm_context *ctx) +{ + uint8_t len_buf[16]; + + memset(ctx->J0, 0, AES_BLOCK_BYTES); + memset(len_buf, 0, AES_BLOCK_BYTES); + + /* If IV is 96 bits J0 = ( IV || 0^31 || 1 ) */ + if (ctx->iv_len == 12) { + memcpy(ctx->J0, ctx->iv, ctx->iv_len); + ctx->J0[AES_BLOCK_BYTES - 1] |= 1; + } else { + /* For IV != 96 bit, J0 = GHASH(IV || 0[s+64] || [len(IV)]64) */ + /* First calculate GHASH on IV */ + esp_gcm_ghash(ctx->H, ctx->iv, ctx->iv_len, ctx->J0); + /* Next create 128 bit block which is equal to + 64 bit 0 + iv length truncated to 64 bits */ + ESP_PUT_BE64(len_buf + 8, ctx->iv_len * 8); + /* Calculate GHASH on last block */ + esp_gcm_ghash(ctx->H, len_buf, 16, ctx->J0); + } } -static void right_shift(uint8_t *v) + +/* + * Increment J0 as per GCM spec, by applying the Standard Incrementing + Function INC_32 to it. + * j is the counter which needs to be incremented which is + * copied to ctx->J0 after incrementing + */ +static void increment32_j0(esp_gcm_context *ctx, uint8_t *j) { - uint32_t val; - - val = WPA_GET_BE32(v + 12); - val >>= 1; - - if (v[11] & 0x01) { - val |= 0x80000000; - } - WPA_PUT_BE32(v + 12, val); - val = WPA_GET_BE32(v + 8); - val >>= 1; - - if (v[7] & 0x01) { - val |= 0x80000000; - } - WPA_PUT_BE32(v + 8, val); - val = WPA_GET_BE32(v + 4); - val >>= 1; - - if (v[3] & 0x01) { - val |= 0x80000000; - } - WPA_PUT_BE32(v + 4, val); - val = WPA_GET_BE32(v); - val >>= 1; - WPA_PUT_BE32(v, val); -} - -/* AES-GCM multiplication z = y * h */ -static int gcm_mult(uint8_t *y, uint8_t *h, uint8_t *z) -{ - uint8_t v0[AES_BLOCK_BYTES]; - int i, j; - - if (!y || !h || !z) { - return -1; - } - - memset(z, 0, AES_BLOCK_BYTES); - memcpy(v0, y, AES_BLOCK_BYTES); - - for (i = 0; i < AES_BLOCK_BYTES; i++) { - for (j = 0; j < 8; j++) { - if (y[i] & BIT(7 - j)) { - xor_block(z, v0); + uint8_t j_len = AES_BLOCK_BYTES; + memcpy(j, ctx->J0, AES_BLOCK_BYTES); + if (j) { + for (uint32_t i = j_len; i > (j_len - 4); i--) { + if (++j[i - 1] != 0) { + break; } - if (v0[15] & 0x01) { - right_shift(v0); - v0[0] ^= 0xE1; + } + memcpy(ctx->J0, j, AES_BLOCK_BYTES); + } +} + +/* Function to xor two data blocks */ +static void xor_data(uint8_t *d, const uint8_t *s) +{ + uint32_t *dst = (uint32_t *) d; + uint32_t *src = (uint32_t *) s; + *dst++ ^= *src++; + *dst++ ^= *src++; + *dst++ ^= *src++; + *dst++ ^= *src++; +} + +/* Right shift 128 bits by 1 in Big Endian format */ +static void right_shift_be(uint8_t *v) +{ + uint8_t prev_lsb = 0, cur_lsb; + uint32_t data; + + for (int i = 0; i < 16; i += 4) { + data = ESP_GET_BE32(v + i); + + cur_lsb = v[i + 3] & 0x1; + data = (data >> 1) | (prev_lsb << 31); + + ESP_PUT_BE32((v + i), data); + prev_lsb = cur_lsb; + } +} + +/* Multiplication in GF(2^128) + * z = x * y + * + * Steps: + * 1. Let x0.x1...x127 denote the sequence of bits in X. + * 2. Let Z0 =[0]128 and V0 = Y. + * 3. For i = 0 to 127, calculate blocks Zi+1 and Vi+1 as follows: + * + * Zi+1 = Zi if [x]i = 0, else Zi+1 = Zi ^ Vi + * Vi+1 = Vi >> 1 if LSB(Vi) = 0, else Vi+1 = (Vi >> 1) ^ R + * + * Note: as per AES-GCM spec 800-38D for Vi+1 calculation LSB(Vi) + * should be check for 1 but this is actually big endian format so + * we need to check MSB(V[15]) + */ +static void gcm_mult(const uint8_t *x, const uint8_t *y, uint8_t *z) +{ + uint8_t v[16]; + int i, j; + uint32_t R = 0x000000E1; /* Field polynomial in Big endian format */ + + memset(z, 0, 16); /* Z_0 = 0^128 */ + memcpy(v, y, 16); /* V_0 = Y */ + + for (i = 0; i < 16; i++) { + /* Test each bit in a byte of x[i] + * Again as per spec we need to test each bit + * in x from index 0 to 127, however its big + * endian format for each sub byte + */ + for (j = 0; j < 8; j++) { + if (x[i] & (1 << (7 - j))) { + xor_data(z, v); + } + /* https://pdfs.semanticscholar.org/1246/a9ad98dc0421ccfc945e6529c886f23e848d.pdf + * page 9 + */ + if (v[15] & 0x1) { + right_shift_be(v); + v[0] ^= R; } else { - right_shift(v0); + right_shift_be(v); } } } - - return 0; } + + /* Update the key value in gcm context */ -int esp_aes_gcm_setkey( esp_aes_gcm_context *ctx, +int esp_aes_gcm_setkey( esp_gcm_context *ctx, mbedtls_cipher_id_t cipher, const unsigned char *key, unsigned int keybits ) @@ -840,602 +1019,199 @@ int esp_aes_gcm_setkey( esp_aes_gcm_context *ctx, } ctx->aes_ctx.key_bytes = keybits / 8; + memcpy(ctx->aes_ctx.key, key, ctx->aes_ctx.key_bytes); return ( 0 ); } - -/* AES-GCM GHASH calculation for IV != 12 bytes */ -static void esp_aes_gcm_ghash(uint8_t *h0, uint8_t *iv, uint8_t iv_len, uint8_t *j0, uint16_t s) +/* AES-GCM GHASH calculation j0 = GHASH(x) using h0 hash key +*/ +static void esp_gcm_ghash(uint8_t *h0, const unsigned char *x, size_t x_len, uint8_t *j0) { - uint8_t *hash_buf; - uint8_t y0[AES_BLOCK_BYTES], old_y0[AES_BLOCK_BYTES]; - uint16_t len, rem = 0; + uint8_t y0[AES_BLOCK_BYTES], tmp[AES_BLOCK_BYTES]; - memset(old_y0, 0, AES_BLOCK_BYTES); + memset(tmp, 0, AES_BLOCK_BYTES); - /* We need to concatenate IV, s + 8 byte zeros & 8 byte IV length - * J0 = GHASH( IV || 0 ^(s+64) || len(IV)^64 ) + /* GHASH(X) is calculated on input string which is multiple of 128 bits + * If input string bit length is not multiple of 128 bits it needs to + * be padded by 0 + * + * Steps: + * 1. Let X1, X2, ... , Xm-1, Xm denote the unique sequence of blocks such + * that X = X1 || X2 || ... || Xm-1 || Xm. + * 2. Let Y0 be the “zero block,” 0128. + * 3. Fori=1,...,m,letYi =(Yi-1 ^ Xi)•H. + * 4. Return Ym */ - len = ( iv_len + (s + 8) + 8 ); - hash_buf = calloc( 1, len ); - if (hash_buf) { - memcpy(hash_buf, iv, iv_len); - // ToDo: iv_len is 1 byte in size, how to copy 8 bytes to other memory? -#if 0 - memcpy((hash_buf + iv_len + s + 8), &iv_len, 8); -#endif + /* If input bit string is >= 128 bits, process full 128 bit blocks */ + while (x_len >= AES_BLOCK_BYTES) { + xor_data(j0, x); + gcm_mult(j0, h0, y0); - /* GHASH(x) calculation - * Let X1, X2, ... , Xm-1, Xm denote the unique sequence of blocks - * such that X = X1 || X2 || ... || Xm-1 || Xm. - * Let Y0 be the “zero block,” 0 ^ 128. - * Fori=1,...,m, let Yi =(Yi-1 xor Xi) • H. - * Return Ym - */ - while (len) { - rem = ( len > AES_BLOCK_BYTES ) ? AES_BLOCK_BYTES : len; - /* Yi-1 xor hash_buf */ - for (uint8_t j = 0; j < rem; j++) { - y0[j] = old_y0[j] ^ hash_buf[j]; - } - - /* gcm multiplication : y0 x h0 */ - gcm_mult(y0, h0, j0); - memcpy(old_y0, y0, rem); - hash_buf += rem; - len -= rem; - } - - memcpy(j0, y0, rem); - free(hash_buf); - } -} - -/* AES-GCM J0 calculation - */ -static void inline esp_aes_gcm_process_J0(uint8_t *data, uint8_t iv_len) -{ - uint8_t j_buf[AES_BLOCK_BYTES]; - uint8_t iv[32]; - uint16_t s; - - esp_aes_get_iv(iv, iv_len); - - /* If IV is 96 bits J0 = ( IV || 0^31 || 1 ) */ - if (iv_len == 12) { - memset(j_buf, 0, AES_BLOCK_BYTES); - memcpy(j_buf, iv, iv_len); - j_buf[AES_BLOCK_BYTES - 1] |= 1; - } else { - /* If IV is != 96 bits then - * J0 = GHASH( IV || 0 ^(s+64) || len(IV)^64 ) where - * s = ( 128 * floor of ( len(IV) / 128 ) - len(IV) ) - * floor of (x) denotes to be the least integer no less than x - * for example: floor of (1.5) = 2 since 2 is the least integer - * which is no less than 1.5 - */ - s = ((iv_len / AES_BLOCK_BYTES) + - ((iv_len % AES_BLOCK_BYTES) == 0) ? 0 : 1); - s = (s * AES_BLOCK_BYTES) - iv_len; - esp_aes_gcm_ghash(data, iv, iv_len, j_buf, s); + x += AES_BLOCK_BYTES; + x_len -= AES_BLOCK_BYTES; + memcpy(j0, y0, AES_BLOCK_BYTES); } - /* Write J0 to hardware registers */ - memcpy((uint8_t *)AES_J_BASE, j_buf, AES_BLOCK_BYTES); -} - -/* Configure & start crypto DMA for AES GCM operation - */ -static void esp_aes_gcm_dma(unsigned char *aad, esp_aes_gcm_context *ctx, - unsigned char *input, size_t ilen, - unsigned char *len_buf, unsigned char *output) -{ - volatile lldesc_t dma_descr[4]; - int i = 0; - - bzero( (void *)dma_descr, sizeof( dma_descr ) ); - -#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) - if ((unsigned int)input >= SOC_EXTRAM_DATA_LOW && (unsigned int)input <= SOC_EXTRAM_DATA_HIGH) { - assert((((unsigned int)(input) & 0xF) == 0)); - Cache_WriteBack_All(); - } - if ((unsigned int)output >= SOC_EXTRAM_DATA_LOW && (unsigned int)output <= SOC_EXTRAM_DATA_HIGH) { - assert((((unsigned int)(output) & 0xF) == 0)); - } -#endif - - dma_descr[0].length = ctx->aad_len; - dma_descr[0].size = ctx->aad_len; - dma_descr[0].buf = aad; - dma_descr[0].owner = 1; - dma_descr[0].eof = 0; - dma_descr[0].empty = (uint32_t)&dma_descr[1]; - - dma_descr[1].length = ilen; - dma_descr[1].size = ilen; - dma_descr[1].buf = input; - dma_descr[1].owner = 1; - dma_descr[1].eof = 0; - dma_descr[1].empty = (uint32_t)&dma_descr[2]; - - dma_descr[2].length = AES_BLOCK_BYTES; - dma_descr[2].size = AES_BLOCK_BYTES; - dma_descr[2].buf = len_buf; - dma_descr[2].owner = 1; - dma_descr[2].eof = 1; - dma_descr[2].empty = 0; - - dma_descr[3].length = ctx->aad_len + ilen + AES_BLOCK_BYTES; - dma_descr[3].size = ctx->aad_len + ilen + AES_BLOCK_BYTES; - dma_descr[3].buf = output; - dma_descr[3].owner = 1; - dma_descr[3].eof = 1; - dma_descr[3].empty = 0; - - /* If no additional authentication data */ - if (ctx->aad_len == 0) { - i = 1; - } - /* If no input data */ - if (ilen == 0) { - i = 2; - } - - /* Enable the DMA clock - currently only for FPGA test */ -#if CONFIG_IDF_ENV_FPGA - SET_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_GEN_CLK_EN); -#endif - - /* Reset DMA */ - SET_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_AHBM_RST | CONF0_REG_IN_RST | CONF0_REG_OUT_RST | CONF0_REG_AHBM_FIFO_RST); - CLEAR_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_AHBM_RST | CONF0_REG_IN_RST | CONF0_REG_OUT_RST | CONF0_REG_AHBM_FIFO_RST); - - /* Set DMA for AES Use */ - REG_WRITE(CRYPTO_DMA_AES_SHA_SELECT_REG, 0); - - /* Set descriptors */ - CLEAR_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_ADDR); - SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, ((uint32_t)(&dma_descr[i]))&OUT_LINK_REG_OUTLINK_ADDR); - CLEAR_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, IN_LINK_REG_INLINK_ADDR); - SET_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, ((uint32_t)(&dma_descr[3]))&IN_LINK_REG_INLINK_ADDR); - - /* Start transfer */ - SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_START); - SET_PERI_REG_MASK(CRYPTO_DMA_IN_LINK_REG, IN_LINK_REG_INLINK_START); - - /* Trigger AES: Let hardware perform GCTR operation */ - esp_aes_gcm_set_block_num_and_trigger(ilen); - - /* While hardware is busy meanwhile software will calculate GHASH - * Read H from hardware register + /* If input bit string is not multiple of 128 create last 128 bit + * block by padding necessary 0s */ - memcpy(ctx->H, (uint8_t *)AES_H_BASE, AES_BLOCK_BYTES); - - esp_aes_gcm_process_J0(ctx->H, ctx->iv_len); - - /* After following call the ciphertext is available in output buffer */ - esp_aes_gcm_continue(ilen); - -#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) - if ((unsigned int)input >= SOC_EXTRAM_DATA_LOW && (unsigned int)input <= SOC_EXTRAM_DATA_HIGH) { - Cache_Invalidate_DCache_All(); + if (x_len) { + memcpy(tmp, x, x_len); + xor_data(j0, tmp); + gcm_mult(j0, h0, y0); + memcpy(j0, y0, AES_BLOCK_BYTES); } -#endif } + + /* Function to init AES GCM context to zero */ -void esp_aes_gcm_init( esp_aes_gcm_context *ctx) +void esp_aes_gcm_init( esp_gcm_context *ctx) { if (ctx == NULL) { return; } - bzero(ctx, sizeof(esp_aes_gcm_context)); + bzero(ctx, sizeof(esp_gcm_context)); } /* Function to clear AES-GCM context */ -void esp_aes_gcm_free( esp_aes_gcm_context *ctx) +void esp_aes_gcm_free( esp_gcm_context *ctx) { if (ctx == NULL) { return; } - bzero(ctx, sizeof(esp_aes_gcm_context)); + bzero(ctx, sizeof(esp_gcm_context)); } /* Setup AES-GCM */ -int esp_aes_gcm_starts( esp_aes_gcm_context *ctx, +int esp_aes_gcm_starts( esp_gcm_context *ctx, int mode, const unsigned char *iv, size_t iv_len, const unsigned char *aad, size_t aad_len ) { - uint8_t temp[AES_BLOCK_BYTES] = {0}; + /* IV and AD are limited to 2^64 bits, so 2^61 bytes */ + /* IV is not allowed to be zero length */ + if ( iv_len == 0 || + ( (uint64_t) iv_len ) >> 61 != 0 || + ( (uint64_t) aad_len ) >> 61 != 0 ) { + return ( MBEDTLS_ERR_GCM_BAD_INPUT ); + } - memcpy(temp, iv, iv_len); - - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware( &ctx->aes_ctx, mode, AES_BLOCK_MODE_GCM); - /* AES-GCM HW does not use IV but we program anyways so that - * we can retrieve later for J0 calculation */ - esp_aes_set_iv(temp, iv_len); + /* Initialize AES-GCM context */ + ctx->iv = iv; ctx->iv_len = iv_len; ctx->aad = aad; ctx->aad_len = aad_len; + ctx->gcm_state = ESP_AES_GCM_STATE_INIT; + ctx->mode = mode; + + /* Lock the AES engine to calculate ghash key H in hardware */ + esp_aes_acquire_hardware(); + esp_aes_setkey_hardware( &ctx->aes_ctx, mode); + esp_aes_mode_init(ESP_AES_BLOCK_MODE_GCM); + /* Enable DMA mode */ + REG_WRITE(AES_DMA_ENABLE_REG, 1); + REG_WRITE(AES_TRIGGER_REG, 1); + while (REG_READ(AES_STATE_REG) != AES_STATE_IDLE); + + memcpy(ctx->H, (uint8_t *)AES_H_BASE, AES_BLOCK_BYTES); + + esp_aes_release_hardware(); + + /* Once H is obtained we need to derive J0 (Initial Counter Block) */ + esp_gcm_derive_J0(ctx); + + /* The initial counter block keeps updating during the esp_gcm_update call + * however to calculate final authentication tag T we need original J0 + * so we make a copy here + */ + memcpy(ctx->ori_j0, ctx->J0, 16); return ( 0 ); } /* Perform AES-GCM operation */ -int esp_aes_gcm_update( esp_aes_gcm_context *ctx, +int esp_aes_gcm_update( esp_gcm_context *ctx, size_t length, const unsigned char *input, unsigned char *output ) { - const uint8_t *dbuf = input; - uint8_t *abuf = (uint8_t *)ctx->aad; - uint64_t ori_aad_len = ctx->aad_len, ori_p_len = length; - uint32_t temp[4] = {0}; - bool abuf_alloc = false, dbuf_alloc = false; + size_t nc_off = 0; + uint8_t stream[AES_BLOCK_BYTES] = {0}; + uint8_t nonce_counter[AES_BLOCK_BYTES] = {0}; + uint8_t gcm_s[AES_BLOCK_BYTES] = {0}; if ( output > input && (size_t) ( output - input ) < length ) { return ( MBEDTLS_ERR_GCM_BAD_INPUT ); } - - /* Check length of AAD & pad if required */ - if (ctx->aad_len % AES_BLOCK_BYTES) { - ori_aad_len = ctx->aad_len; - abuf = textpad_zero(ctx->aad, (uint8_t *)&ctx->aad_len); - if (!abuf) { - return -1; - } else { - abuf_alloc = true; - } + /* If this is the first time esp_gcm_update is getting called + * calculate GHASH on aad and preincrement the ICB + */ + if (ctx->gcm_state == ESP_AES_GCM_STATE_INIT) { + /* The GHASH calculation is done at multiple stages + * Here we calculate GHASH of AAD and save it + */ + esp_gcm_ghash(ctx->H, ctx->aad, ctx->aad_len, gcm_s); + /* Jo needs to be incremented first time, later the GCTR + * operation will auto update it + */ + increment32_j0(ctx, nonce_counter); + ctx->gcm_state = ESP_AES_GCM_STATE_UPDATE; + } else if (ctx->gcm_state == ESP_AES_GCM_STATE_UPDATE) { + memcpy(gcm_s, ctx->S, AES_BLOCK_BYTES); + memcpy(nonce_counter, ctx->J0, AES_BLOCK_BYTES); } - /* Check length of input & pad if required */ - if ( length % AES_BLOCK_BYTES ) { - ori_p_len = length; - REG_WRITE(AES_BIT_VALID_NUM_REG, (length % AES_BLOCK_BYTES) * 8); - dbuf = textpad_zero(input, (uint8_t *)&length); - if (!dbuf) { - if (abuf_alloc) { - free(abuf); - return -1; - } - } else { - dbuf_alloc = true; - } + /* Output = GCTR(J0, Input): Encrypt/Decrypt the input */ + esp_aes_crypt_ctr(&ctx->aes_ctx, length, &nc_off, nonce_counter, stream, input, output); + /* ICB gets auto incremented after GCTR operation here so update the context */ + memcpy(ctx->J0, nonce_counter, AES_BLOCK_BYTES); + + /* Keep updating the length counter for final tag calculation */ + ctx->data_len += length; + + /* Perform intermediate GHASH on "encrypted" data irrespective of mode */ + if (ctx->mode == ESP_AES_DECRYPT) { + esp_gcm_ghash(ctx->H, input, length, gcm_s); + } else { + esp_gcm_ghash(ctx->H, output, length, gcm_s); } - /* Update number of AAD blocks in hardware register */ - REG_WRITE(AES_AAD_BLOCK_NUM_REG, (ctx->aad_len / AES_BLOCK_BYTES)); - - /* Input buffer is: length[textpad(input)] + length[textpad(aad)] - * + [length]64 + [aad_len]64 - */ - ori_aad_len *= 8; - ori_p_len *= 8; - - temp[0] = (uint32_t)LE_TO_BE(ori_aad_len >> 32); - temp[1] = (uint32_t)LE_TO_BE(ori_aad_len); - temp[2] = (uint32_t)LE_TO_BE(ori_p_len >> 32); - temp[3] = (uint32_t)LE_TO_BE(ori_p_len); - - esp_aes_gcm_dma(abuf, ctx, (uint8_t *)dbuf, length, (uint8_t *)temp, output); - - if (abuf_alloc) { - free((void *)abuf); - } - if (dbuf_alloc) { - free((void *)dbuf); - } + memcpy(ctx->S, gcm_s, AES_BLOCK_BYTES); return 0; } /* Function to read the tag value */ -int esp_aes_gcm_finish( esp_aes_gcm_context *ctx, +int esp_aes_gcm_finish( esp_gcm_context *ctx, unsigned char *tag, size_t tag_len ) { - memcpy(tag, (uint8_t *)AES_T_BASE, tag_len); + size_t nc_off = 0; + uint8_t len_block[AES_BLOCK_BYTES] = {0}; + + if ( tag_len > 16 || tag_len < 4 ) { + return ( MBEDTLS_ERR_GCM_BAD_INPUT ); + } + + /* Calculate final GHASH on aad_len, data length */ + ESP_PUT_BE64(len_block, ctx->aad_len * 8); + ESP_PUT_BE64(len_block + 8, ctx->data_len * 8); + esp_gcm_ghash(ctx->H, len_block, AES_BLOCK_BYTES, ctx->S); + + /* Tag T = GCTR(J0, S) where T is truncated to tag_len */ + esp_aes_crypt_ctr(&ctx->aes_ctx, tag_len, &nc_off, ctx->ori_j0, 0, ctx->S, tag); return 0; } -/* - * AES-OFB (Output Feedback Mode) buffer encryption/decryption - */ -int esp_aes_crypt_ofb( esp_aes_context *ctx, - size_t length, - size_t *iv_off, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output ) -{ - size_t n = *iv_off; - size_t stream_bytes = 0; - - while (n > 0 && length > 0) { - *output++ = (unsigned char)(*input++ ^ iv[n]); - n = (n + 1) & 0xF; - length--; - } - - if (length > 0) { - stream_bytes = (length % AES_BLOCK_BYTES); - - esp_aes_acquire_hardware(); - esp_aes_setkey_hardware(ctx, ESP_AES_DECRYPT, AES_BLOCK_MODE_OFB); - esp_aes_set_iv(iv, AES_BLOCK_BYTES); - esp_aes_process_dma(ctx, input, output, length, iv); - if (stream_bytes == 0) { - // new IV is in the IV block - esp_aes_get_iv(iv, AES_BLOCK_BYTES); - } else { - // IV is in the iv buffer (stream_out param), however 'stream_bytes' - // of it are already XORed with input bytes so need to un-XOR them - for (int i = 0; i < stream_bytes; i++) { - iv[i] ^= input[length - stream_bytes + i]; - } - } - esp_aes_release_hardware(); - } - - *iv_off = n + stream_bytes; - - return 0; -} - - -/* Below XTS implementation is copied aes.c of mbedtls library. - * When MBEDTLS_AES_ALT is defined mbedtls expects alternate - * definition of XTS functions to be available. Even if this - * could have been avoided, it is done for consistency reason. - */ - -void esp_aes_xts_init( esp_aes_xts_context *ctx ) -{ - esp_aes_init( &ctx->crypt ); - esp_aes_init( &ctx->tweak ); -} - -void esp_aes_xts_free( esp_aes_xts_context *ctx ) -{ - esp_aes_free( &ctx->crypt ); - esp_aes_free( &ctx->tweak ); -} - -static int esp_aes_xts_decode_keys( const unsigned char *key, - unsigned int keybits, - const unsigned char **key1, - unsigned int *key1bits, - const unsigned char **key2, - unsigned int *key2bits ) -{ - const unsigned int half_keybits = keybits / 2; - const unsigned int half_keybytes = half_keybits / 8; - - switch( keybits ) - { - case 256: break; - case 512: break; - default : return( MBEDTLS_ERR_AES_INVALID_KEY_LENGTH ); - } - - *key1bits = half_keybits; - *key2bits = half_keybits; - *key1 = &key[0]; - *key2 = &key[half_keybytes]; - - return 0; -} - -int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits) -{ - int ret; - const unsigned char *key1, *key2; - unsigned int key1bits, key2bits; - - ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, - &key2, &key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set the tweak key. Always set tweak key for the encryption mode. */ - ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set crypt key for encryption. */ - return esp_aes_setkey( &ctx->crypt, key1, key1bits ); -} - -int esp_aes_xts_setkey_dec( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits) -{ - int ret; - const unsigned char *key1, *key2; - unsigned int key1bits, key2bits; - - ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, - &key2, &key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set the tweak key. Always set tweak key for encryption. */ - ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); - if( ret != 0 ) - return( ret ); - - /* Set crypt key for decryption. */ - return esp_aes_setkey( &ctx->crypt, key1, key1bits ); -} - -/* Endianess with 64 bits values */ -#ifndef GET_UINT64_LE -#define GET_UINT64_LE(n,b,i) \ -{ \ - (n) = ( (uint64_t) (b)[(i) + 7] << 56 ) \ - | ( (uint64_t) (b)[(i) + 6] << 48 ) \ - | ( (uint64_t) (b)[(i) + 5] << 40 ) \ - | ( (uint64_t) (b)[(i) + 4] << 32 ) \ - | ( (uint64_t) (b)[(i) + 3] << 24 ) \ - | ( (uint64_t) (b)[(i) + 2] << 16 ) \ - | ( (uint64_t) (b)[(i) + 1] << 8 ) \ - | ( (uint64_t) (b)[(i) ] ); \ -} -#endif - -#ifndef PUT_UINT64_LE -#define PUT_UINT64_LE(n,b,i) \ -{ \ - (b)[(i) + 7] = (unsigned char) ( (n) >> 56 ); \ - (b)[(i) + 6] = (unsigned char) ( (n) >> 48 ); \ - (b)[(i) + 5] = (unsigned char) ( (n) >> 40 ); \ - (b)[(i) + 4] = (unsigned char) ( (n) >> 32 ); \ - (b)[(i) + 3] = (unsigned char) ( (n) >> 24 ); \ - (b)[(i) + 2] = (unsigned char) ( (n) >> 16 ); \ - (b)[(i) + 1] = (unsigned char) ( (n) >> 8 ); \ - (b)[(i) ] = (unsigned char) ( (n) ); \ -} -#endif - -typedef unsigned char esp_be128[16]; - -/* - * GF(2^128) multiplication function - * - * This function multiplies a field element by x in the polynomial field - * representation. It uses 64-bit word operations to gain speed but compensates - * for machine endianess and hence works correctly on both big and little - * endian machines. - */ -static void esp_gf128mul_x_ble( unsigned char r[16], - const unsigned char x[16] ) -{ - uint64_t a, b, ra, rb; - - GET_UINT64_LE( a, x, 0 ); - GET_UINT64_LE( b, x, 8 ); - - ra = ( a << 1 ) ^ 0x0087 >> ( 8 - ( ( b >> 63 ) << 3 ) ); - rb = ( a >> 63 ) | ( b << 1 ); - - PUT_UINT64_LE( ra, r, 0 ); - PUT_UINT64_LE( rb, r, 8 ); -} - -/* - * AES-XTS buffer encryption/decryption - */ -int esp_aes_crypt_xts( esp_aes_xts_context *ctx, - int mode, - size_t length, - const unsigned char data_unit[16], - const unsigned char *input, - unsigned char *output ) -{ - int ret; - size_t blocks = length / 16; - size_t leftover = length % 16; - unsigned char tweak[16]; - unsigned char prev_tweak[16]; - unsigned char tmp[16]; - - /* Sectors must be at least 16 bytes. */ - if( length < 16 ) - return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; - - /* NIST SP 80-38E disallows data units larger than 2**20 blocks. */ - if( length > ( 1 << 20 ) * 16 ) - return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; - - /* Compute the tweak. */ - ret = esp_aes_crypt_ecb( &ctx->tweak, MBEDTLS_AES_ENCRYPT, - data_unit, tweak ); - if( ret != 0 ) - return( ret ); - - while( blocks-- ) - { - size_t i; - - if( leftover && ( mode == MBEDTLS_AES_DECRYPT ) && blocks == 0 ) - { - /* We are on the last block in a decrypt operation that has - * leftover bytes, so we need to use the next tweak for this block, - * and this tweak for the lefover bytes. Save the current tweak for - * the leftovers and then update the current tweak for use on this, - * the last full block. */ - memcpy( prev_tweak, tweak, sizeof( tweak ) ); - esp_gf128mul_x_ble( tweak, tweak ); - } - - for( i = 0; i < 16; i++ ) - tmp[i] = input[i] ^ tweak[i]; - - ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); - if( ret != 0 ) - return( ret ); - - for( i = 0; i < 16; i++ ) - output[i] = tmp[i] ^ tweak[i]; - - /* Update the tweak for the next block. */ - esp_gf128mul_x_ble( tweak, tweak ); - - output += 16; - input += 16; - } - - if( leftover ) - { - /* If we are on the leftover bytes in a decrypt operation, we need to - * use the previous tweak for these bytes (as saved in prev_tweak). */ - unsigned char *t = mode == MBEDTLS_AES_DECRYPT ? prev_tweak : tweak; - - /* We are now on the final part of the data unit, which doesn't divide - * evenly by 16. It's time for ciphertext stealing. */ - size_t i; - unsigned char *prev_output = output - 16; - - /* Copy ciphertext bytes from the previous block to our output for each - * byte of cyphertext we won't steal. At the same time, copy the - * remainder of the input for this final round (since the loop bounds - * are the same). */ - for( i = 0; i < leftover; i++ ) - { - output[i] = prev_output[i]; - tmp[i] = input[i] ^ t[i]; - } - - /* Copy ciphertext bytes from the previous block for input in this - * round. */ - for( ; i < 16; i++ ) - tmp[i] = prev_output[i] ^ t[i]; - - ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); - if( ret != 0 ) - return ret; - - /* Write the result back to the previous block, overriding the previous - * output we copied. */ - for( i = 0; i < 16; i++ ) - prev_output[i] = tmp[i] ^ t[i]; - } - - return( 0 ); -} -int esp_aes_gcm_crypt_and_tag( esp_aes_gcm_context *ctx, +int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx, int mode, size_t length, const unsigned char *iv, @@ -1449,6 +1225,7 @@ int esp_aes_gcm_crypt_and_tag( esp_aes_gcm_context *ctx, { int ret; + if ( ( ret = esp_aes_gcm_starts( ctx, mode, iv, iv_len, add, add_len ) ) != 0 ) { return ( ret ); } @@ -1464,7 +1241,7 @@ int esp_aes_gcm_crypt_and_tag( esp_aes_gcm_context *ctx, return ( 0 ); } -int esp_aes_gcm_auth_decrypt( esp_aes_gcm_context *ctx, +int esp_aes_gcm_auth_decrypt( esp_gcm_context *ctx, size_t length, const unsigned char *iv, size_t iv_len, @@ -1498,5 +1275,3 @@ int esp_aes_gcm_auth_decrypt( esp_aes_gcm_context *ctx, return ( 0 ); } - - diff --git a/components/mbedtls/port/esp32s2/esp_bignum.c b/components/mbedtls/port/esp32s2/esp_bignum.c index 6a8a88a56..d169423b7 100644 --- a/components/mbedtls/port/esp32s2/esp_bignum.c +++ b/components/mbedtls/port/esp32s2/esp_bignum.c @@ -223,8 +223,8 @@ static inline void start_op(uint32_t op_reg) */ static inline void wait_op_complete(uint32_t op_reg) { - while(DPORT_REG_READ(RSA_QUERY_INTERRUPT_REG) != 1) - { } + while (DPORT_REG_READ(RSA_QUERY_INTERRUPT_REG) != 1) + { } /* clear the interrupt */ DPORT_REG_WRITE(RSA_CLEAR_INTERRUPT_REG, 1); @@ -258,7 +258,7 @@ int esp_mpi_mul_mpi_mod(mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi esp_mpi_acquire_hardware(); - DPORT_REG_WRITE(RSA_LENGTH_REG, (num_words-1)); + DPORT_REG_WRITE(RSA_LENGTH_REG, (num_words - 1)); DPORT_REG_WRITE(RSA_M_DASH_REG, (uint32_t)Mprime); /* Load M, X, Rinv, Mprime (Mprime is mod 2^32) */ @@ -404,17 +404,17 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi size_t num_words = MAX(x_words, y_words); size_t z_words = x_words + y_words; - /* Short-circuit eval if either argument is 0 or 1. + /* Short-circuit eval if either argument is 0 or 1. - This is needed as the mpi modular division - argument will sometimes call in here when one - argument is too large for the hardware unit, but the other - argument is zero or one. + This is needed as the mpi modular division + argument will sometimes call in here when one + argument is too large for the hardware unit, but the other + argument is zero or one. - This leaks some timing information, although overall there is a - lot less timing variation than a software MPI approach. + This leaks some timing information, although overall there is a + lot less timing variation than a software MPI approach. */ - if (x_bits == 0 || y_bits== 0) { + if (x_bits == 0 || y_bits == 0) { mbedtls_mpi_lset(Z, 0); return 0; } @@ -449,7 +449,7 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi } else { /* Still too long for the hardware unit... */ mbedtls_mpi_grow(Z, z_words); - if(y_words > x_words) { + if (y_words > x_words) { return mpi_mult_mpi_overlong(Z, X, Y, y_words, z_words); } else { return mpi_mult_mpi_overlong(Z, Y, X, x_words, z_words); @@ -468,7 +468,7 @@ int mbedtls_mpi_mul_mpi( mbedtls_mpi *Z, const mbedtls_mpi *X, const mbedtls_mpi */ DPORT_REG_WRITE(RSA_M_DASH_REG, 0); - DPORT_REG_WRITE(RSA_LENGTH_REG, (num_words*2 - 1)); + DPORT_REG_WRITE(RSA_LENGTH_REG, (num_words * 2 - 1)); start_op(RSA_MULT_START_REG); wait_op_complete(RSA_MULT_START_REG); diff --git a/components/mbedtls/port/esp32s2/esp_sha1.c b/components/mbedtls/port/esp32s2/esp_sha1.c new file mode 100644 index 000000000..f84b62774 --- /dev/null +++ b/components/mbedtls/port/esp32s2/esp_sha1.c @@ -0,0 +1,255 @@ +/* + * SHA-1 implementation with hardware ESP32 support added. + * Uses mbedTLS software implementation for failover when concurrent + * SHA operations are in use. + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE LTD + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/* + * The SHA-1 standard was published by NIST in 1993. + * + * http://www.itl.nist.gov/fipspubs/fip180-1.htm + */ + +#if !defined(MBEDTLS_CONFIG_FILE) +#include "mbedtls/config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + +#if defined(MBEDTLS_SHA1_C) && defined(MBEDTLS_SHA1_ALT) + +#include "mbedtls/sha1.h" + +#include + +#if defined(MBEDTLS_SELF_TEST) +#if defined(MBEDTLS_PLATFORM_C) +#include "mbedtls/platform.h" +#else +#include +#define mbedtls_printf printf +#endif /* MBEDTLS_PLATFORM_C */ +#endif /* MBEDTLS_SELF_TEST */ + +#include "esp32s2/sha.h" + +/* Implementation that should never be optimized out by the compiler */ +static void mbedtls_zeroize( void *v, size_t n ) +{ + volatile unsigned char *p = (unsigned char *)v; while ( n-- ) *p++ = 0; +} + +/* + * 32-bit integer manipulation macros (big endian) + */ + +#ifndef PUT_UINT32_BE +#define PUT_UINT32_BE(n,b,i) \ +{ \ + (b)[(i) ] = (unsigned char) ( (n) >> 24 ); \ + (b)[(i) + 1] = (unsigned char) ( (n) >> 16 ); \ + (b)[(i) + 2] = (unsigned char) ( (n) >> 8 ); \ + (b)[(i) + 3] = (unsigned char) ( (n) ); \ +} +#endif + +void mbedtls_sha1_init( mbedtls_sha1_context *ctx ) +{ + memset( ctx, 0, sizeof( mbedtls_sha1_context ) ); +} + +void mbedtls_sha1_free( mbedtls_sha1_context *ctx ) +{ + if ( ctx == NULL ) { + return; + } + + mbedtls_zeroize( ctx, sizeof( mbedtls_sha1_context ) ); +} + +void mbedtls_sha1_clone( mbedtls_sha1_context *dst, + const mbedtls_sha1_context *src ) +{ + memcpy(dst, src, sizeof(mbedtls_sha1_context)); +} + +/* + * SHA-1 context setup + */ +int mbedtls_sha1_starts_ret( mbedtls_sha1_context *ctx ) +{ + ctx->total[0] = 0; + ctx->total[1] = 0; + + memset( ctx, 0, sizeof( mbedtls_sha1_context ) ); + ctx->mode = SHA1; + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha1_starts( mbedtls_sha1_context *ctx ) +{ + mbedtls_sha1_starts_ret( ctx ); +} +#endif + +static int esp_internal_sha1_dma_process(mbedtls_sha1_context *ctx, + const uint8_t *data, size_t len, + uint8_t *buf, size_t buf_len) +{ + return esp_sha_dma(SHA1, data, len, buf, buf_len, ctx->first_block); +} + +int mbedtls_internal_sha1_process( mbedtls_sha1_context *ctx, const unsigned char data[64] ) +{ + int ret; + esp_sha_acquire_hardware(); + ret = esp_sha_dma(ctx->mode, data, 64, 0, 0, ctx->first_block); + esp_sha_release_hardware(); + return ret; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha1_process( mbedtls_sha1_context *ctx, + const unsigned char data[64] ) +{ + mbedtls_internal_sha1_process( ctx, data ); +} +#endif + +int mbedtls_sha1_update_ret( mbedtls_sha1_context *ctx, const unsigned char *input, size_t ilen ) +{ + int ret; + size_t fill; + uint32_t left, len, local_len = 0; + + if ( !ilen || (input == NULL)) { + return 0; + } + + left = ctx->total[0] & 0x3F; + fill = 64 - left; + + ctx->total[0] += (uint32_t) ilen; + ctx->total[0] &= 0xFFFFFFFF; + + if ( ctx->total[0] < (uint32_t) ilen ) { + ctx->total[1]++; + } + + if ( left && ilen >= fill ) { + memcpy( (void *) (ctx->buffer + left), input, fill ); + + input += fill; + ilen -= fill; + left = 0; + local_len = 64; + } + + len = (ilen / 64) * 64; + if ( len || local_len) { + + esp_sha_acquire_hardware(); + + if (ctx->sha_state == ESP_SHA1_STATE_INIT) { + ctx->first_block = true; + ctx->sha_state = ESP_SHA1_STATE_IN_PROCESS; + } else if (ctx->sha_state == ESP_SHA1_STATE_IN_PROCESS) { + ctx->first_block = false; + esp_sha_write_digest_state(SHA1, ctx->state); + } + + ret = esp_internal_sha1_dma_process(ctx, input, len, ctx->buffer, local_len); + + esp_sha_read_digest_state(SHA1, ctx->state); + + esp_sha_release_hardware(); + + if (ret != 0) { + return ret; + } + + } + + if ( ilen > 0 ) { + memcpy( (void *) (ctx->buffer + left), input + len, ilen - len ); + } + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha1_update( mbedtls_sha1_context *ctx, + const unsigned char *input, + size_t ilen ) +{ + mbedtls_sha1_update_ret( ctx, input, ilen ); +} +#endif + +static const unsigned char sha1_padding[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* +* SHA-1 final digest + */ +int mbedtls_sha1_finish_ret( mbedtls_sha1_context *ctx, unsigned char output[20] ) +{ + int ret; + uint32_t last, padn; + uint32_t high, low; + unsigned char msglen[8]; + + high = ( ctx->total[0] >> 29 ) + | ( ctx->total[1] << 3 ); + low = ( ctx->total[0] << 3 ); + + PUT_UINT32_BE( high, msglen, 0 ); + PUT_UINT32_BE( low, msglen, 4 ); + + last = ctx->total[0] & 0x3F; + padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); + + + if ( ( ret = mbedtls_sha1_update_ret( ctx, sha1_padding, padn ) ) != 0 ) { + return ret; + } + if ( ( ret = mbedtls_sha1_update_ret( ctx, msglen, 8 ) ) != 0 ) { + return ret; + } + + memcpy(output, ctx->state, 20); + + return ret; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha1_finish( mbedtls_sha1_context *ctx, + unsigned char output[20] ) +{ + mbedtls_sha1_finish_ret( ctx, output ); +} +#endif + +#endif /* MBEDTLS_SHA1_C && MBEDTLS_SHA1_ALT */ diff --git a/components/mbedtls/port/esp32s2/esp_sha256.c b/components/mbedtls/port/esp32s2/esp_sha256.c new file mode 100644 index 000000000..f7ecf3e10 --- /dev/null +++ b/components/mbedtls/port/esp32s2/esp_sha256.c @@ -0,0 +1,267 @@ +/* + * SHA-256 implementation with hardware ESP32 support added. + * Uses mbedTLS software implementation for failover when concurrent + * SHA operations are in use. + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE LTD + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * The SHA-256 Secure Hash Standard was published by NIST in 2002. + * + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf + */ + +#if !defined(MBEDTLS_CONFIG_FILE) +#include "mbedtls/config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + +#if defined(MBEDTLS_SHA256_C) && defined(MBEDTLS_SHA256_ALT) + +#include "mbedtls/sha256.h" + +#include + +#if defined(MBEDTLS_SELF_TEST) +#if defined(MBEDTLS_PLATFORM_C) +#include "mbedtls/platform.h" +#else +#include +#define mbedtls_printf printf +#endif /* MBEDTLS_PLATFORM_C */ +#endif /* MBEDTLS_SELF_TEST */ + +#include "esp32s2/sha.h" + +/* Implementation that should never be optimized out by the compiler */ +static void mbedtls_zeroize( void *v, size_t n ) +{ + volatile unsigned char *p = v; while ( n-- ) *p++ = 0; +} + +/* + * 32-bit integer manipulation macros (big endian) + */ +#ifndef GET_UINT32_BE +#define GET_UINT32_BE(n,b,i) \ +do { \ + (n) = ( (uint32_t) (b)[(i) ] << 24 ) \ + | ( (uint32_t) (b)[(i) + 1] << 16 ) \ + | ( (uint32_t) (b)[(i) + 2] << 8 ) \ + | ( (uint32_t) (b)[(i) + 3] ); \ +} while( 0 ) +#endif + +#ifndef PUT_UINT32_BE +#define PUT_UINT32_BE(n,b,i) \ +do { \ + (b)[(i) ] = (unsigned char) ( (n) >> 24 ); \ + (b)[(i) + 1] = (unsigned char) ( (n) >> 16 ); \ + (b)[(i) + 2] = (unsigned char) ( (n) >> 8 ); \ + (b)[(i) + 3] = (unsigned char) ( (n) ); \ +} while( 0 ) +#endif + +void mbedtls_sha256_init( mbedtls_sha256_context *ctx ) +{ + memset( ctx, 0, sizeof( mbedtls_sha256_context ) ); +} + +void mbedtls_sha256_free( mbedtls_sha256_context *ctx ) +{ + if ( ctx == NULL ) { + return; + } + + mbedtls_zeroize( ctx, sizeof( mbedtls_sha256_context ) ); +} + +void mbedtls_sha256_clone( mbedtls_sha256_context *dst, + const mbedtls_sha256_context *src ) +{ + *dst = *src; +} + +/* + * SHA-256 context setup + */ +int mbedtls_sha256_starts_ret( mbedtls_sha256_context *ctx, int is224 ) +{ + memset( ctx, 0, sizeof( mbedtls_sha256_context ) ); + + if ( is224 ) { + ctx->mode = SHA2_224; + } else { + ctx->mode = SHA2_256; + } + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha256_starts( mbedtls_sha256_context *ctx, + int is224 ) +{ + mbedtls_sha256_starts_ret( ctx, is224 ); +} +#endif + + +int mbedtls_internal_sha256_process( mbedtls_sha256_context *ctx, const unsigned char data[64] ) +{ + int ret; + esp_sha_acquire_hardware(); + ret = esp_sha_dma(ctx->mode, data, 64, 0, 0, ctx->first_block); + esp_sha_release_hardware(); + + return ret; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha256_process( mbedtls_sha256_context *ctx, + const unsigned char data[64] ) +{ + mbedtls_internal_sha256_process( ctx, data ); +} +#endif + +/* + * SHA-256 process buffer + */ +int mbedtls_sha256_update_ret( mbedtls_sha256_context *ctx, const unsigned char *input, + size_t ilen ) +{ + int ret = 0; + size_t fill; + uint32_t left, len, local_len = 0; + + if ( ilen == 0 ) { + return 0; + } + + left = ctx->total[0] & 0x3F; + fill = 64 - left; + + ctx->total[0] += (uint32_t) ilen; + ctx->total[0] &= 0xFFFFFFFF; + + if ( ctx->total[0] < (uint32_t) ilen ) { + ctx->total[1]++; + } + + /* Check if any data pending from previous call to this API */ + if ( left && ilen >= fill ) { + memcpy( (void *) (ctx->buffer + left), input, fill ); + + input += fill; + ilen -= fill; + left = 0; + local_len = 64; + } + + len = (ilen / 64) * 64; + + if ( len || local_len) { + esp_sha_acquire_hardware(); + + if (ctx->sha_state == ESP_SHA256_STATE_INIT) { + ctx->first_block = true; + ctx->sha_state = ESP_SHA256_STATE_IN_PROCESS; + } else if (ctx->sha_state == ESP_SHA256_STATE_IN_PROCESS) { + ctx->first_block = false; + esp_sha_write_digest_state(ctx->mode, ctx->state); + } + + ret = esp_sha_dma(ctx->mode, input, len, ctx->buffer, local_len, ctx->first_block); + + esp_sha_read_digest_state(ctx->mode, ctx->state); + + esp_sha_release_hardware(); + + if (ret != 0) { + return ret; + } + } + + if ( ilen > 0 ) { + memcpy( (void *) (ctx->buffer + left), input + len, ilen - len ); + } + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha256_update( mbedtls_sha256_context *ctx, + const unsigned char *input, + size_t ilen ) +{ + mbedtls_sha256_update_ret( ctx, input, ilen ); +} +#endif + +static const unsigned char sha256_padding[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* + * SHA-256 final digest + */ +int mbedtls_sha256_finish_ret( mbedtls_sha256_context *ctx, unsigned char output[32] ) +{ + int ret; + uint32_t last, padn; + uint32_t high, low; + unsigned char msglen[8]; + + high = ( ctx->total[0] >> 29 ) + | ( ctx->total[1] << 3 ); + low = ( ctx->total[0] << 3 ); + + PUT_UINT32_BE( high, msglen, 0 ); + PUT_UINT32_BE( low, msglen, 4 ); + + last = ctx->total[0] & 0x3F; + padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); + + if ( ( ret = mbedtls_sha256_update_ret( ctx, sha256_padding, padn ) ) != 0 ) { + return ret; + } + + if ( ( ret = mbedtls_sha256_update_ret( ctx, msglen, 8 ) ) != 0 ) { + return ret; + } + + memcpy(output, ctx->state, 32); + + return ret; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha256_finish( mbedtls_sha256_context *ctx, + unsigned char output[32] ) +{ + mbedtls_sha256_finish_ret( ctx, output ); +} +#endif + +#endif /* MBEDTLS_SHA256_C && MBEDTLS_SHA256_ALT */ diff --git a/components/mbedtls/port/esp32s2/esp_sha512.c b/components/mbedtls/port/esp32s2/esp_sha512.c new file mode 100644 index 000000000..155ee25b1 --- /dev/null +++ b/components/mbedtls/port/esp32s2/esp_sha512.c @@ -0,0 +1,317 @@ +/* + * SHA-512 implementation with hardware ESP32 support added. + * Uses mbedTLS software implementation for failover when concurrent + * SHA operations are in use. + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE LTD + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * The SHA-512 Secure Hash Standard was published by NIST in 2002. + * + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf + */ + +#if !defined(MBEDTLS_CONFIG_FILE) +#include "mbedtls/config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + +#if defined(MBEDTLS_SHA512_C) && defined(MBEDTLS_SHA512_ALT) + +#include "mbedtls/sha512.h" + +#if defined(_MSC_VER) || defined(__WATCOMC__) +#define UL64(x) x##ui64 +#else +#define UL64(x) x##ULL +#endif + +#include + +#if defined(MBEDTLS_SELF_TEST) +#if defined(MBEDTLS_PLATFORM_C) +#include "mbedtls/platform.h" +#else +#include +#define mbedtls_printf printf +#endif /* MBEDTLS_PLATFORM_C */ +#endif /* MBEDTLS_SELF_TEST */ + +#include "esp32s2/sha.h" + +/* Implementation that should never be optimized out by the compiler */ +static void mbedtls_zeroize( void *v, size_t n ) +{ + volatile unsigned char *p = v; while ( n-- ) *p++ = 0; +} + +/* + * 64-bit integer manipulation macros (big endian) + */ +#ifndef PUT_UINT64_BE +#define PUT_UINT64_BE(n,b,i) \ +{ \ + (b)[(i) ] = (unsigned char) ( (n) >> 56 ); \ + (b)[(i) + 1] = (unsigned char) ( (n) >> 48 ); \ + (b)[(i) + 2] = (unsigned char) ( (n) >> 40 ); \ + (b)[(i) + 3] = (unsigned char) ( (n) >> 32 ); \ + (b)[(i) + 4] = (unsigned char) ( (n) >> 24 ); \ + (b)[(i) + 5] = (unsigned char) ( (n) >> 16 ); \ + (b)[(i) + 6] = (unsigned char) ( (n) >> 8 ); \ + (b)[(i) + 7] = (unsigned char) ( (n) ); \ +} +#endif /* PUT_UINT64_BE */ + +void esp_sha512_set_mode(mbedtls_sha512_context *ctx, esp_sha_type type) +{ + switch (type) { + case SHA2_384: + case SHA2_512224: + case SHA2_512256: + case SHA2_512T: + ctx->mode = type; + break; + default: + ctx->mode = SHA2_512; + break; + } +} + + +/* For SHA512/t mode the intial hash value will depend on t */ +void esp_sha512_set_t( mbedtls_sha512_context *ctx, uint16_t t_val) +{ + ctx->t_val = t_val; +} + +void mbedtls_sha512_init( mbedtls_sha512_context *ctx ) +{ + memset( ctx, 0, sizeof( mbedtls_sha512_context ) ); +} + +void mbedtls_sha512_free( mbedtls_sha512_context *ctx ) +{ + if ( ctx == NULL ) { + return; + } + + mbedtls_zeroize( ctx, sizeof( mbedtls_sha512_context ) ); +} + +void mbedtls_sha512_clone( mbedtls_sha512_context *dst, + const mbedtls_sha512_context *src ) +{ + memcpy(dst, src, sizeof(mbedtls_sha512_context)); +} + +/* + * SHA-512 context setup + */ +int mbedtls_sha512_starts_ret( mbedtls_sha512_context *ctx, int is384 ) +{ + mbedtls_zeroize( ctx, sizeof( mbedtls_sha512_context ) ); + + if ( is384 ) { + ctx->mode = SHA2_384; + } else { + ctx->mode = SHA2_512; + } + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha512_starts( mbedtls_sha512_context *ctx, + int is384 ) +{ + mbedtls_sha512_starts_ret( ctx, is384 ); +} +#endif + +static int esp_internal_sha512_dma_process(mbedtls_sha512_context *ctx, + const uint8_t *data, size_t len, + uint8_t *buf, size_t buf_len) +{ + + + return esp_sha_dma(ctx->mode, data, len, buf, buf_len, ctx->first_block); + + +} + +int mbedtls_internal_sha512_process( mbedtls_sha512_context *ctx, const unsigned char data[128] ) +{ + int ret; + esp_sha_acquire_hardware(); + ret = esp_internal_sha512_dma_process(ctx, data, 128, 0, 0); + esp_sha_release_hardware(); + + return ret; + +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha512_process( mbedtls_sha512_context *ctx, + const unsigned char data[128] ) +{ + mbedtls_internal_sha512_process( ctx, data ); +} +#endif + +/* + * SHA-512 process buffer + */ +int mbedtls_sha512_update_ret( mbedtls_sha512_context *ctx, const unsigned char *input, + size_t ilen ) +{ + int ret; + size_t fill; + unsigned int left, len, local_len = 0; + + if ( ilen == 0 ) { + return 0; + } + + left = (unsigned int) (ctx->total[0] & 0x7F); + fill = 128 - left; + + ctx->total[0] += (uint64_t) ilen; + + if ( ctx->total[0] < (uint64_t) ilen ) { + ctx->total[1]++; + } + + if ( left && ilen >= fill ) { + memcpy( (void *) (ctx->buffer + left), input, fill ); + + input += fill; + ilen -= fill; + left = 0; + local_len = 128; + } + + len = (ilen / 128) * 128; + + if ( len || local_len) { + + esp_sha_acquire_hardware(); + + if (ctx->sha_state == ESP_SHA512_STATE_INIT) { + + if (ctx->mode == SHA2_512T) { + esp_sha_512_t_init_hash(ctx->t_val); + ctx->first_block = false; + } else { + ctx->first_block = true; + } + ctx->sha_state = ESP_SHA512_STATE_IN_PROCESS; + + } else if (ctx->sha_state == ESP_SHA512_STATE_IN_PROCESS) { + ctx->first_block = false; + esp_sha_write_digest_state(ctx->mode, ctx->state); + } + + ret = esp_internal_sha512_dma_process(ctx, input, len, ctx->buffer, local_len); + + esp_sha_read_digest_state(ctx->mode, ctx->state); + + esp_sha_release_hardware(); + + if (ret != 0) { + return ret; + } + + } + + + if ( ilen > 0 ) { + memcpy( (void *) (ctx->buffer + left), input + len, ilen - len ); + } + + return 0; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha512_update( mbedtls_sha512_context *ctx, + const unsigned char *input, + size_t ilen ) +{ + mbedtls_sha512_update_ret( ctx, input, ilen ); +} +#endif + + +static const unsigned char sha512_padding[128] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* + * SHA-512 final digest + */ +int mbedtls_sha512_finish_ret( mbedtls_sha512_context *ctx, unsigned char output[64] ) +{ + int ret; + size_t last, padn; + uint64_t high, low; + unsigned char msglen[16]; + + high = ( ctx->total[0] >> 61 ) + | ( ctx->total[1] << 3 ); + low = ( ctx->total[0] << 3 ); + + PUT_UINT64_BE( high, msglen, 0 ); + PUT_UINT64_BE( low, msglen, 8 ); + + last = (size_t)( ctx->total[0] & 0x7F ); + padn = ( last < 112 ) ? ( 112 - last ) : ( 240 - last ); + + if ( ( ret = mbedtls_sha512_update_ret( ctx, sha512_padding, padn ) ) != 0 ) { + return ret; + } + + if ( ( ret = mbedtls_sha512_update_ret( ctx, msglen, 16 ) ) != 0 ) { + return ret; + } + + if (ctx->mode == SHA2_384) { + memcpy(output, ctx->state, 48); + } else { + memcpy(output, ctx->state, 64); + } + + return ret; +} + +#if !defined(MBEDTLS_DEPRECATED_REMOVED) +void mbedtls_sha512_finish( mbedtls_sha512_context *ctx, + unsigned char output[64] ) +{ + mbedtls_sha512_finish_ret( ctx, output ); +} +#endif + +#endif /* MBEDTLS_SHA512_C && MBEDTLS_SHA512_ALT */ diff --git a/components/mbedtls/port/esp32s2/sha.c b/components/mbedtls/port/esp32s2/sha.c index d854aa98c..0d024a13c 100644 --- a/components/mbedtls/port/esp32s2/sha.c +++ b/components/mbedtls/port/esp32s2/sha.c @@ -3,7 +3,7 @@ * based on mbedTLS FIPS-197 compliant version. * * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved - * Additions Copyright (C) 2016, Espressif Systems (Shanghai) PTE Ltd + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE Ltd * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -28,34 +28,38 @@ #include #include #include -#include -#include "soc/soc.h" -#include "esp32s2/crypto_dma.h" -#include "esp32s2/sha.h" -#include "soc/crypto_dma_reg.h" + +#include "esp_log.h" #include "esp32s2/rom/ets_sys.h" #include "soc/dport_reg.h" #include "soc/hwcrypto_reg.h" -#include "esp32s2/rom/lldesc.h" + #include "esp32s2/rom/cache.h" -#include "esp_intr_alloc.h" -#include "esp_log.h" -#include "soc/periph_defs.h" + +#include "soc/cache_memory.h" #include "freertos/FreeRTOS.h" -#include "freertos/task.h" #include "freertos/semphr.h" -/* Single lock for SHA engine +#include "esp32s2/sha.h" +#include "esp32s2/crypto_dma.h" +#include "esp32s2/rom/lldesc.h" +#include "soc/periph_defs.h" +#include "soc/crypto_dma_reg.h" +#include "driver/periph_ctrl.h" +#include "sys/param.h" + + +/* Max amount of bytes in a single DMA operation is 4095, + for SHA this means that the biggest safe amount of bytes is + 31 blocks of 128 bytes = 3968 */ +#define SHA_DMA_MAX_BYTES 3968 + +/* Lock for SHA engine */ static _lock_t s_sha_lock; -/* Enable if want to use SHA interrupt */ -//#define CONFIG_MBEDTLS_SHA_USE_INTERRUPT - -#if defined(CONFIG_MBEDTLS_SHA_USE_INTERRUPT) -static SemaphoreHandle_t op_complete_sem; -#endif +const static char *TAG = "esp-sha"; /* Return block size (in bytes) for a given SHA type */ inline static size_t block_length(esp_sha_type type) @@ -96,190 +100,265 @@ inline static size_t state_length(esp_sha_type type) } } -/* This API was designed for ESP32, which has seperate - engines for SHA1,256,512. ESP32C has a single engine. -*/ -static void esp_sha_lock_engine_inner(void); - -bool esp_sha_try_lock_engine(esp_sha_type sha_type) -{ - if (_lock_try_acquire(&s_sha_lock) != 0) { - /* SHA engine is already in use */ - return false; - } else { - esp_sha_lock_engine_inner(); - return true; - } -} - -void esp_sha_lock_engine(esp_sha_type sha_type) -{ - _lock_acquire(&s_sha_lock); - esp_sha_lock_engine_inner(); -} - -/* Enable SHA block and then lock it */ -static void esp_sha_lock_engine_inner(void) +/* Enable SHA peripheral and then lock it */ +void esp_sha_acquire_hardware() { /* Need to lock DMA since it is shared with AES block */ - portENTER_CRITICAL(&crypto_dma_spinlock); + _lock_acquire(&crypto_dma_lock); + _lock_acquire(&s_sha_lock); - REG_SET_BIT(DPORT_PERIP_CLK_EN1_REG, DPORT_CRYPTO_SHA_CLK_EN | DPORT_CRYPTO_DMA_CLK_EN); - REG_CLR_BIT(DPORT_PERIP_RST_EN1_REG, DPORT_CRYPTO_SHA_RST | DPORT_CRYPTO_HMAC_RST | - DPORT_CRYPTO_DMA_RST | DPORT_CRYPTO_DS_RST); + /* Enable SHA and DMA hardware */ + periph_module_enable(PERIPH_SHA_DMA_MODULE); /* DMA for SHA */ REG_WRITE(CRYPTO_DMA_AES_SHA_SELECT_REG, 1); } -/* Disable SHA block and then unlock it */ -void esp_sha_unlock_engine(esp_sha_type sha_type) +/* Disable SHA peripheral block and then release it */ +void esp_sha_release_hardware() { - REG_WRITE(CRYPTO_DMA_AES_SHA_SELECT_REG, 0); - - REG_SET_BIT(DPORT_PERIP_RST_EN1_REG, DPORT_CRYPTO_SHA_RST | DPORT_CRYPTO_DMA_RST | - DPORT_CRYPTO_DS_RST); - REG_CLR_BIT(DPORT_PERIP_CLK_EN1_REG, DPORT_CRYPTO_SHA_CLK_EN | DPORT_CRYPTO_DMA_CLK_EN); - - portEXIT_CRITICAL(&crypto_dma_spinlock); + /* Disable SHA and DMA hardware */ + periph_module_disable(PERIPH_SHA_DMA_MODULE); + /* Need to lock DMA since it is shared with AES block */ _lock_release(&s_sha_lock); + _lock_release(&crypto_dma_lock); + } -#if defined (CONFIG_MBEDTLS_SHA_USE_INTERRUPT) -static IRAM_ATTR void esp_sha_dma_isr(void *arg) + +/* Busy wait until SHA is idle */ +static void esp_sha_wait_idle(void) { - BaseType_t higher_woken; - REG_WRITE(SHA_CLEAR_IRQ_REG, 1); - xSemaphoreGiveFromISR(op_complete_sem, &higher_woken); - if (higher_woken) { - portYIELD_FROM_ISR(); + while (DPORT_REG_READ(SHA_BUSY_REG) != 0) { } } -#endif -/* Check if SHA operation completed */ -static int esp_sha_dma_complete(void) + +void esp_sha_write_digest_state(esp_sha_type sha_type, void *digest_state) { -#if defined (CONFIG_MBEDTLS_SHA_USE_INTERRUPT) - if (!xSemaphoreTake(op_complete_sem, 2000 / portTICK_PERIOD_MS)) { - ESP_LOGE("SHA", "Timed out waiting for completion of SHA Interrupt"); - return -1; + uint32_t *digest_state_words = (uint32_t *)digest_state; + uint32_t *reg_addr_buf = (uint32_t *)(SHA_H_BASE); + + for (int i = 0; i < state_length(sha_type) / 4; i++) { + REG_WRITE(®_addr_buf[i], digest_state_words[i]); } -#else - esp_sha_wait_idle(); -#endif - return 0; -} - -/* Wait until SHA is busy */ -void esp_sha_wait_idle(void) -{ - while (DPORT_REG_READ(SHA_BUSY_REG) != 0) { } } /* Read the SHA digest from hardware */ void esp_sha_read_digest_state(esp_sha_type sha_type, void *digest_state) { + uint32_t *digest_state_words = (uint32_t *)digest_state; + int word_len = state_length(sha_type) / 4; + + esp_dport_access_read_buffer(digest_state_words, SHA_H_BASE, word_len); + + /* Fault injection check: verify SHA engine actually ran, + state is not all zeroes. + */ + for (int i = 0; i < word_len; i++) { + if (digest_state_words[i] != 0) { + return; + } + } + abort(); // SHA peripheral returned all zero state, probably due to fault injection +} + +/* The initial hash value for SHA512/t is generated according to the + algorithm described in the TRM, chapter SHA-Accelerator +*/ +int esp_sha_512_t_init_hash(uint16_t t) +{ + uint32_t t_string = 0; + uint8_t t0, t1, t2, t_len; + + if (t == 384) { + ESP_LOGE(TAG, "Invalid t for SHA512/t, t = %u,cannot be 384", t); + return -1; + } + + if (t <= 9) { + t_string = (uint32_t)((1 << 23) | ((0x30 + t) << 24)); + t_len = 0x48; + } else if (t <= 99) { + t0 = t % 10; + t1 = (t / 10) % 10; + t_string = (uint32_t)((1 << 15) | ((0x30 + t0) << 16) | + (((0x30 + t1) << 24))); + t_len = 0x50; + } else if (t <= 512) { + t0 = t % 10; + t1 = (t / 10) % 10; + t2 = t / 100; + t_string = (uint32_t)((1 << 7) | ((0x30 + t0) << 8) | + (((0x30 + t1) << 16) + ((0x30 + t2) << 24))); + t_len = 0x58; + } else { + ESP_LOGE(TAG, "Invalid t for SHA512/t, t = %u, must equal or less than 512", t); + return -1; + } + + REG_WRITE(SHA_T_LENGTH_REG, t_len); + REG_WRITE(SHA_T_STRING_REG, t_string); + REG_WRITE(SHA_MODE_REG, SHA2_512T); + REG_WRITE(SHA_START_REG, 1); + esp_sha_wait_idle(); - memcpy(digest_state, (void *)SHA_H_BASE, state_length(sha_type)); + + return 0; } -/* Internally calls DMA API for single block */ -void esp_sha_block(esp_sha_type sha_type, const void *data_block, bool is_first_block) -{ - esp_sha_dma(sha_type, data_block, block_length(sha_type), is_first_block); -} -/* Performs SHA on multiple blocks at a time */ -int esp_sha_dma(esp_sha_type sha_type, const void *data_block, uint32_t ilen, bool is_first_block) +static int esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen, + const void *buf, uint32_t buf_len, bool is_first_block); + +/* Performs SHA on multiple blocks at a time using DMA + splits up into smaller operations for inputs that exceed a single DMA list + */ +int esp_sha_dma(esp_sha_type sha_type, const void *input, uint32_t ilen, + const void *buf, uint32_t buf_len, bool is_first_block) { - size_t blk_len = 0; - const uint8_t *local_buf = data_block; int ret = 0; - volatile lldesc_t dma_descr; + const void *dma_input; + unsigned char *non_icache_input = NULL; + unsigned char *non_icache_buf = NULL; + int dma_op_num = ( ilen / (SHA_DMA_MAX_BYTES + 1) ) + 1; - if (ilen == 0) { - return ret; + if (buf_len > 128) { + ESP_LOGE(TAG, "SHA DMA buf_len cannot exceed max size for a single block"); + return -1; } - blk_len = block_length(sha_type); - - REG_WRITE(SHA_MODE_REG, sha_type); - if ((sha_type == SHA2_512T) && (is_first_block == true)) { - REG_WRITE(SHA_START_REG, 1); - } - - REG_WRITE(SHA_BLOCK_NUM_REG, (ilen / blk_len)); - - if ((sha_type == SHA2_512T) && (is_first_block == true)) { - esp_sha_wait_idle(); - is_first_block = false; - } - - bzero( (void *)&dma_descr, sizeof( dma_descr ) ); - - /* DMA descriptor for Memory to DMA-AES transfer */ - dma_descr.length = ilen; - dma_descr.size = ilen; - dma_descr.owner = 1; - dma_descr.eof = 1; - dma_descr.buf = local_buf; - dma_descr.sosf = 0; - dma_descr.empty = 0; - #if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) - if ((unsigned int)data_block >= SOC_EXTRAM_DATA_LOW && (unsigned int)data_block <= SOC_EXTRAM_DATA_HIGH) { + if (esp_ptr_external_ram(input) || esp_ptr_external_ram(buf)) { Cache_WriteBack_All(); } #endif + /* DMA cannot access memory in the iCache range, copy data to temporary buffers before transfer */ + if (!esp_ptr_dma_ext_capable(input) && !esp_ptr_dma_capable(input)) { + non_icache_input = malloc(sizeof(unsigned char) * MIN(ilen, SHA_DMA_MAX_BYTES)); + if (non_icache_input == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory"); + ret = ESP_ERR_NO_MEM; + goto cleanup; + } + } + + if (!esp_ptr_dma_ext_capable(buf) && !esp_ptr_dma_capable(buf)) { + non_icache_buf = malloc(sizeof(unsigned char) * buf_len); + if (non_icache_buf == NULL) { + ESP_LOGE(TAG, "Failed to allocate memory"); + ret = ESP_ERR_NO_MEM; + goto cleanup; + } + memcpy(non_icache_buf, buf, buf_len); + buf = non_icache_buf; + } + + /* The max amount of blocks in a single hardware operation is 2^6 - 1 = 63 + Thus we only do a single DMA input list + dma buf list, + which is max 3968/64 + 64/64 = 63 blocks */ + for (int i = 0; i < dma_op_num; i++) { + int dma_chunk_len = MIN(ilen, SHA_DMA_MAX_BYTES); + + + /* Input depends on if it's a temp alloc buffer or supplied by user */ + if (non_icache_input != NULL) { + memcpy(non_icache_input, input, dma_chunk_len); + dma_input = non_icache_input; + } else { + dma_input = input; + } + + ret = esp_sha_dma_process(sha_type, dma_input, dma_chunk_len, buf, buf_len, is_first_block); + + + if (ret != 0) { + return ret; + } + + is_first_block = false; + + + ilen -= dma_chunk_len; + input += dma_chunk_len; + + // Only append buf to the first operation + buf_len = 0; + } + +cleanup: + free(non_icache_input); + free(non_icache_buf); + return ret; +} + +static void esp_sha_dma_init(lldesc_t *input) +{ /* Reset DMA */ SET_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_AHBM_RST | CONF0_REG_OUT_RST | CONF0_REG_AHBM_FIFO_RST); CLEAR_PERI_REG_MASK(CRYPTO_DMA_CONF0_REG, CONF0_REG_AHBM_RST | CONF0_REG_OUT_RST | CONF0_REG_AHBM_FIFO_RST); /* Set descriptors */ CLEAR_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_ADDR); - SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, ((uint32_t)(&dma_descr))&OUT_LINK_REG_OUTLINK_ADDR); + SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, ((uint32_t)(input))&OUT_LINK_REG_OUTLINK_ADDR); /* Start transfer */ SET_PERI_REG_MASK(CRYPTO_DMA_OUT_LINK_REG, OUT_LINK_REG_OUTLINK_START); +} -#if defined (CONFIG_MBEDTLS_SHA_USE_INTERRUPT) - REG_WRITE(SHA_CLEAR_IRQ_REG, 1); - if (op_complete_sem == NULL) { - op_complete_sem = xSemaphoreCreateBinary(); - esp_intr_alloc(ETS_SHA_INTR_SOURCE, 0, esp_sha_dma_isr, 0, 0); +/* Performs SHA on multiple blocks at a time */ +static esp_err_t esp_sha_dma_process(esp_sha_type sha_type, const void *input, uint32_t ilen, + const void *buf, uint32_t buf_len, bool is_first_block) +{ + size_t blk_len = 0; + int ret = 0; + lldesc_t dma_descr_input = {}; + lldesc_t dma_descr_buf = {}; + lldesc_t *dma_descr_head; + + blk_len = block_length(sha_type); + + REG_WRITE(SHA_MODE_REG, sha_type); + REG_WRITE(SHA_BLOCK_NUM_REG, ((ilen + buf_len) / blk_len)); + + + /* DMA descriptor for Memory to DMA-SHA transfer */ + if (ilen) { + dma_descr_input.length = ilen; + dma_descr_input.size = ilen; + dma_descr_input.owner = 1; + dma_descr_input.eof = 1; + dma_descr_input.buf = input; + dma_descr_head = &dma_descr_input; + } + /* Check after input to overide head if there is any buf*/ + if (buf_len) { + dma_descr_buf.length = buf_len; + dma_descr_buf.size = buf_len; + dma_descr_buf.owner = 1; + dma_descr_buf.eof = 1; + dma_descr_buf.buf = buf; + dma_descr_head = &dma_descr_buf; } - REG_WRITE(SHA_INT_ENA_REG, 1); -#endif + /* Link DMA lists */ + if (buf_len && ilen) { + dma_descr_buf.eof = 0; + dma_descr_buf.empty = (uint32_t)(&dma_descr_input); + } + + esp_sha_dma_init(dma_descr_head); + + /* Start hashing */ if (is_first_block) { REG_WRITE(SHA_DMA_START_REG, 1); } else { REG_WRITE(SHA_DMA_CONTINUE_REG, 1); } - ret = esp_sha_dma_complete(); - -#if (CONFIG_SPIRAM_USE_CAPS_ALLOC || CONFIG_SPIRAM_USE_MALLOC) - if ((unsigned int)data_block >= SOC_EXTRAM_DATA_LOW && (unsigned int)data_block <= SOC_EXTRAM_DATA_HIGH) { - Cache_Invalidate_DCache_All(); - } -#endif + esp_sha_wait_idle(); return ret; } -void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, unsigned char *output) -{ - SHA_CTX ctx; - - esp_sha_lock_engine(sha_type); - - ets_sha_init(&ctx, sha_type); - ets_sha_starts(&ctx, 0); - ets_sha_update(&ctx, input, ilen, false); - ets_sha_finish(&ctx, output); - - esp_sha_unlock_engine(sha_type); -} diff --git a/components/mbedtls/port/esp_aes_xts.c b/components/mbedtls/port/esp_aes_xts.c new file mode 100644 index 000000000..18d001b80 --- /dev/null +++ b/components/mbedtls/port/esp_aes_xts.c @@ -0,0 +1,288 @@ +/** + * \brief AES block cipher, ESP32-S2 hardware accelerated version + * Based on mbedTLS FIPS-197 compliant version. + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * Additions Copyright (C) 2016-2020, Espressif Systems (Shanghai) PTE Ltd + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/* + * The AES block cipher was designed by Vincent Rijmen and Joan Daemen. + * + * http://csrc.nist.gov/encryption/aes/rijndael/Rijndael.pdf + * http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf + */ + + +/* Below XTS implementation is copied aes.c of mbedtls library. + * When MBEDTLS_AES_ALT is defined mbedtls expects alternate + * definition of XTS functions to be available. Even if this + * could have been avoided, it is done for consistency reason. + */ + +#include +#include +#include +#include "mbedtls/aes.h" + +#if CONFIG_IDF_TARGET_ESP32 +#include "esp32/aes.h" +#elif CONFIG_IDF_TARGET_ESP32S2 +#include "esp32s2/aes.h" +#endif + +void esp_aes_xts_init( esp_aes_xts_context *ctx ) +{ + esp_aes_init( &ctx->crypt ); + esp_aes_init( &ctx->tweak ); +} + +void esp_aes_xts_free( esp_aes_xts_context *ctx ) +{ + esp_aes_free( &ctx->crypt ); + esp_aes_free( &ctx->tweak ); +} + +static int esp_aes_xts_decode_keys( const unsigned char *key, + unsigned int keybits, + const unsigned char **key1, + unsigned int *key1bits, + const unsigned char **key2, + unsigned int *key2bits ) +{ + const unsigned int half_keybits = keybits / 2; + const unsigned int half_keybytes = half_keybits / 8; + + switch ( keybits ) { + case 256: break; + case 512: break; + default : return ( MBEDTLS_ERR_AES_INVALID_KEY_LENGTH ); + } + + *key1bits = half_keybits; + *key2bits = half_keybits; + *key1 = &key[0]; + *key2 = &key[half_keybytes]; + + return 0; +} + +int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx, + const unsigned char *key, + unsigned int keybits) +{ + int ret; + const unsigned char *key1, *key2; + unsigned int key1bits, key2bits; + + ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, + &key2, &key2bits ); + if ( ret != 0 ) { + return ( ret ); + } + + /* Set the tweak key. Always set tweak key for the encryption mode. */ + ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); + if ( ret != 0 ) { + return ( ret ); + } + + /* Set crypt key for encryption. */ + return esp_aes_setkey( &ctx->crypt, key1, key1bits ); +} + +int esp_aes_xts_setkey_dec( esp_aes_xts_context *ctx, + const unsigned char *key, + unsigned int keybits) +{ + int ret; + const unsigned char *key1, *key2; + unsigned int key1bits, key2bits; + + ret = esp_aes_xts_decode_keys( key, keybits, &key1, &key1bits, + &key2, &key2bits ); + if ( ret != 0 ) { + return ( ret ); + } + + /* Set the tweak key. Always set tweak key for encryption. */ + ret = esp_aes_setkey( &ctx->tweak, key2, key2bits ); + if ( ret != 0 ) { + return ( ret ); + } + + /* Set crypt key for decryption. */ + return esp_aes_setkey( &ctx->crypt, key1, key1bits ); +} + +/* Endianess with 64 bits values */ +#ifndef GET_UINT64_LE +#define GET_UINT64_LE(n,b,i) \ +{ \ + (n) = ( (uint64_t) (b)[(i) + 7] << 56 ) \ + | ( (uint64_t) (b)[(i) + 6] << 48 ) \ + | ( (uint64_t) (b)[(i) + 5] << 40 ) \ + | ( (uint64_t) (b)[(i) + 4] << 32 ) \ + | ( (uint64_t) (b)[(i) + 3] << 24 ) \ + | ( (uint64_t) (b)[(i) + 2] << 16 ) \ + | ( (uint64_t) (b)[(i) + 1] << 8 ) \ + | ( (uint64_t) (b)[(i) ] ); \ +} +#endif + +#ifndef PUT_UINT64_LE +#define PUT_UINT64_LE(n,b,i) \ +{ \ + (b)[(i) + 7] = (unsigned char) ( (n) >> 56 ); \ + (b)[(i) + 6] = (unsigned char) ( (n) >> 48 ); \ + (b)[(i) + 5] = (unsigned char) ( (n) >> 40 ); \ + (b)[(i) + 4] = (unsigned char) ( (n) >> 32 ); \ + (b)[(i) + 3] = (unsigned char) ( (n) >> 24 ); \ + (b)[(i) + 2] = (unsigned char) ( (n) >> 16 ); \ + (b)[(i) + 1] = (unsigned char) ( (n) >> 8 ); \ + (b)[(i) ] = (unsigned char) ( (n) ); \ +} +#endif + +/* + * GF(2^128) multiplication function + * + * This function multiplies a field element by x in the polynomial field + * representation. It uses 64-bit word operations to gain speed but compensates + * for machine endianess and hence works correctly on both big and little + * endian machines. + */ +static void esp_gf128mul_x_ble( unsigned char r[16], + const unsigned char x[16] ) +{ + uint64_t a, b, ra, rb; + + GET_UINT64_LE( a, x, 0 ); + GET_UINT64_LE( b, x, 8 ); + + ra = ( a << 1 ) ^ 0x0087 >> ( 8 - ( ( b >> 63 ) << 3 ) ); + rb = ( a >> 63 ) | ( b << 1 ); + + PUT_UINT64_LE( ra, r, 0 ); + PUT_UINT64_LE( rb, r, 8 ); +} + +/* + * AES-XTS buffer encryption/decryption + */ +int esp_aes_crypt_xts( esp_aes_xts_context *ctx, + int mode, + size_t length, + const unsigned char data_unit[16], + const unsigned char *input, + unsigned char *output ) +{ + int ret; + size_t blocks = length / 16; + size_t leftover = length % 16; + unsigned char tweak[16]; + unsigned char prev_tweak[16]; + unsigned char tmp[16]; + + /* Sectors must be at least 16 bytes. */ + if ( length < 16 ) { + return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; + } + + /* NIST SP 80-38E disallows data units larger than 2**20 blocks. */ + if ( length > ( 1 << 20 ) * 16 ) { + return MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH; + } + + /* Compute the tweak. */ + ret = esp_aes_crypt_ecb( &ctx->tweak, MBEDTLS_AES_ENCRYPT, + data_unit, tweak ); + if ( ret != 0 ) { + return ( ret ); + } + + while ( blocks-- ) { + size_t i; + + if ( leftover && ( mode == MBEDTLS_AES_DECRYPT ) && blocks == 0 ) { + /* We are on the last block in a decrypt operation that has + * leftover bytes, so we need to use the next tweak for this block, + * and this tweak for the lefover bytes. Save the current tweak for + * the leftovers and then update the current tweak for use on this, + * the last full block. */ + memcpy( prev_tweak, tweak, sizeof( tweak ) ); + esp_gf128mul_x_ble( tweak, tweak ); + } + + for ( i = 0; i < 16; i++ ) { + tmp[i] = input[i] ^ tweak[i]; + } + + ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); + if ( ret != 0 ) { + return ( ret ); + } + + for ( i = 0; i < 16; i++ ) { + output[i] = tmp[i] ^ tweak[i]; + } + + /* Update the tweak for the next block. */ + esp_gf128mul_x_ble( tweak, tweak ); + + output += 16; + input += 16; + } + + if ( leftover ) { + /* If we are on the leftover bytes in a decrypt operation, we need to + * use the previous tweak for these bytes (as saved in prev_tweak). */ + unsigned char *t = mode == MBEDTLS_AES_DECRYPT ? prev_tweak : tweak; + + /* We are now on the final part of the data unit, which doesn't divide + * evenly by 16. It's time for ciphertext stealing. */ + size_t i; + unsigned char *prev_output = output - 16; + + /* Copy ciphertext bytes from the previous block to our output for each + * byte of cyphertext we won't steal. At the same time, copy the + * remainder of the input for this final round (since the loop bounds + * are the same). */ + for ( i = 0; i < leftover; i++ ) { + output[i] = prev_output[i]; + tmp[i] = input[i] ^ t[i]; + } + + /* Copy ciphertext bytes from the previous block for input in this + * round. */ + for ( ; i < 16; i++ ) { + tmp[i] = prev_output[i] ^ t[i]; + } + + ret = esp_aes_crypt_ecb( &ctx->crypt, mode, tmp, tmp ); + if ( ret != 0 ) { + return ret; + } + + /* Write the result back to the previous block, overriding the previous + * output we copied. */ + for ( i = 0; i < 16; i++ ) { + prev_output[i] = tmp[i] ^ t[i]; + } + } + + return ( 0 ); +} \ No newline at end of file diff --git a/components/mbedtls/port/esp_sha.c b/components/mbedtls/port/esp_sha.c index be9447379..9265033c5 100644 --- a/components/mbedtls/port/esp_sha.c +++ b/components/mbedtls/port/esp_sha.c @@ -26,7 +26,6 @@ #include -#if CONFIG_IDF_TARGET_ESP32 void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, unsigned char *output) { int ret; @@ -83,4 +82,3 @@ void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, uns } } -#endif diff --git a/components/mbedtls/port/include/esp32s2/aes.h b/components/mbedtls/port/include/esp32s2/aes.h index b89da1350..5f55fdce4 100644 --- a/components/mbedtls/port/include/esp32s2/aes.h +++ b/components/mbedtls/port/include/esp32s2/aes.h @@ -3,7 +3,7 @@ * Based on mbedTLS FIPS-197 compliant version. * * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved - * Additions Copyright (C) 2016, Espressif Systems (Shanghai) PTE Ltd + * Additions Copyright (C) 2016-20, Espressif Systems (Shanghai) PTE Ltd * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -38,6 +38,7 @@ extern "C" { #define ERR_ESP_AES_INVALID_KEY_LENGTH -0x0020 /**< Invalid key length. */ #define ERR_ESP_AES_INVALID_INPUT_LENGTH -0x0022 /**< Invalid data input length. */ + /** * \brief AES context structure * @@ -52,11 +53,11 @@ typedef struct { uint8_t key[32]; } esp_aes_context; + /** * \brief The AES XTS context-type definition. */ -typedef struct -{ +typedef struct { esp_aes_context crypt; /*!< The AES context to use for AES block encryption or decryption. */ esp_aes_context tweak; /*!< The AES context used for tweak @@ -312,8 +313,8 @@ int esp_aes_crypt_ofb( esp_aes_context *ctx, * \return #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH on failure. */ int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits ); + const unsigned char *key, + unsigned int keybits ); /** * \brief Internal AES block encryption function @@ -325,8 +326,8 @@ int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx, * \param output Output (ciphertext) block */ int esp_aes_xts_setkey_dec( esp_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits ); + const unsigned char *key, + unsigned int keybits ); /** diff --git a/components/mbedtls/port/include/esp32s2/crypto_dma.h b/components/mbedtls/port/include/esp32s2/crypto_dma.h index b3eead844..ce7d67efa 100644 --- a/components/mbedtls/port/include/esp32s2/crypto_dma.h +++ b/components/mbedtls/port/include/esp32s2/crypto_dma.h @@ -31,8 +31,8 @@ extern "C" { /* Since crypto DMA is shared between DMA-AES and SHA blocks * Needs to be taken by respective blocks before using Crypto DMA */ -extern portMUX_TYPE crypto_dma_spinlock; - +extern _lock_t crypto_dma_lock; + #ifdef __cplusplus } #endif diff --git a/components/mbedtls/port/include/esp32s2/gcm.h b/components/mbedtls/port/include/esp32s2/gcm.h index 44568c24d..edd51384b 100644 --- a/components/mbedtls/port/include/esp32s2/gcm.h +++ b/components/mbedtls/port/include/esp32s2/gcm.h @@ -3,7 +3,7 @@ * Based on mbedTLS FIPS-197 compliant version. * * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved - * Additions Copyright (C) 2016, Espressif Systems (Shanghai) PTE Ltd + * Additions Copyright (C) 2019-2020, Espressif Systems (Shanghai) PTE Ltd * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -30,27 +30,39 @@ extern "C" { #endif -#define MBEDTLS_ERR_GCM_AUTH_FAILED -0x0012 /**< Authenticated decryption failed. */ -#define MBEDTLS_ERR_GCM_BAD_INPUT -0x0014 /**< Bad input parameters to function.*/ +#define MBEDTLS_ERR_GCM_AUTH_FAILED -0x0012 /**< Authenticated decryption failed. */ +#define MBEDTLS_ERR_GCM_BAD_INPUT -0x0014 /**< Bad input parameters to function.*/ + +typedef enum { + ESP_AES_GCM_STATE_INIT, + ESP_AES_GCM_STATE_UPDATE, + ESP_AES_GCM_STATE_FINISH +} esp_aes_gcm_state; /** * \brief The GCM context structure. */ typedef struct { uint8_t H[16]; /*!< H */ + uint8_t S[16]; + uint8_t J0[16]; + uint8_t ori_j0[16]; + const uint8_t *iv; size_t iv_len; /*!< The length of IV. */ - uint64_t aad_len; /*!< The total length of the additional data. */ + uint64_t aad_len; /*!< The total length of the additional data. */ + size_t data_len; + int mode; const unsigned char *aad; /*!< The additional data. */ - esp_aes_context aes_ctx; -} -esp_aes_gcm_context; + esp_aes_context aes_ctx; + esp_aes_gcm_state gcm_state; +} esp_gcm_context; /** * \brief This function initializes the specified GCM context * * \param ctx The GCM context to initialize. */ -void esp_aes_gcm_init( esp_aes_gcm_context *ctx); +void esp_aes_gcm_init( esp_gcm_context *ctx); /** * \brief This function associates a GCM context with a @@ -67,7 +79,7 @@ void esp_aes_gcm_init( esp_aes_gcm_context *ctx); * \return \c 0 on success. * \return A cipher-specific error code on failure. */ -int esp_aes_gcm_setkey( esp_aes_gcm_context *ctx, +int esp_aes_gcm_setkey( esp_gcm_context *ctx, mbedtls_cipher_id_t cipher, const unsigned char *key, unsigned int keybits ); @@ -88,12 +100,12 @@ int esp_aes_gcm_setkey( esp_aes_gcm_context *ctx, * * \return \c 0 on success. */ -int esp_aes_gcm_starts( esp_aes_gcm_context *ctx, - int mode, - const unsigned char *iv, - size_t iv_len, - const unsigned char *aad, - size_t aad_len ); +int esp_aes_gcm_starts( esp_gcm_context *ctx, + int mode, + const unsigned char *iv, + size_t iv_len, + const unsigned char *aad, + size_t aad_len ); /** * \brief This function feeds an input buffer into an ongoing GCM @@ -116,7 +128,7 @@ int esp_aes_gcm_starts( esp_aes_gcm_context *ctx, * \return \c 0 on success. * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. */ -int esp_aes_gcm_update( esp_aes_gcm_context *ctx, +int esp_aes_gcm_update( esp_gcm_context *ctx, size_t length, const unsigned char *input, unsigned char *output ); @@ -135,16 +147,16 @@ int esp_aes_gcm_update( esp_aes_gcm_context *ctx, * \return \c 0 on success. * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure. */ -int esp_aes_gcm_finish( esp_aes_gcm_context *ctx, - unsigned char *tag, - size_t tag_len ); +int esp_aes_gcm_finish( esp_gcm_context *ctx, + unsigned char *tag, + size_t tag_len ); /** * \brief This function clears a GCM context * * \param ctx The GCM context to clear. */ -void esp_aes_gcm_free( esp_aes_gcm_context *ctx); +void esp_aes_gcm_free( esp_gcm_context *ctx); /** * \brief This function performs GCM encryption or decryption of a buffer. @@ -170,17 +182,17 @@ void esp_aes_gcm_free( esp_aes_gcm_context *ctx); * * \return \c 0 on success. */ -int esp_aes_gcm_crypt_and_tag( esp_aes_gcm_context *ctx, - int mode, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *input, - unsigned char *output, - size_t tag_len, - unsigned char *tag ); +int esp_aes_gcm_crypt_and_tag( esp_gcm_context *ctx, + int mode, + size_t length, + const unsigned char *iv, + size_t iv_len, + const unsigned char *add, + size_t add_len, + const unsigned char *input, + unsigned char *output, + size_t tag_len, + unsigned char *tag ); /** @@ -206,16 +218,16 @@ int esp_aes_gcm_crypt_and_tag( esp_aes_gcm_context *ctx, * \return 0 if successful and authenticated. * \return #MBEDTLS_ERR_GCM_AUTH_FAILED if the tag does not match. */ -int esp_aes_gcm_auth_decrypt( esp_aes_gcm_context *ctx, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *tag, - size_t tag_len, - const unsigned char *input, - unsigned char *output ); +int esp_aes_gcm_auth_decrypt( esp_gcm_context *ctx, + size_t length, + const unsigned char *iv, + size_t iv_len, + const unsigned char *add, + size_t add_len, + const unsigned char *tag, + size_t tag_len, + const unsigned char *input, + unsigned char *output ); #ifdef __cplusplus } diff --git a/components/mbedtls/port/include/esp32s2/sha.h b/components/mbedtls/port/include/esp32s2/sha.h index 70acad343..58bd43941 100644 --- a/components/mbedtls/port/include/esp32s2/sha.h +++ b/components/mbedtls/port/include/esp32s2/sha.h @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// Copyright 2019-2020 Espressif Systems (Shanghai) PTE LTD // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,9 +16,8 @@ #define _ESP_SHA_H_ #include "esp32s2/rom/sha.h" -#include "esp_types.h" -/** @brief Low-level support functions for the hardware SHA engine +/** @brief Low-level support functions for the hardware SHA engine using DMA * * @note If you're looking for a SHA API to use, try mbedtls component * mbedtls/shaXX.h. That API supports hardware acceleration. @@ -28,15 +27,8 @@ * * Some technical details about the hardware SHA engine: * - * - SHA accelerator engine calculates one digest at a time, per SHA - * algorithm type. It initialises and maintains the digest state - * internally. It is possible to read out an in-progress SHA digest - * state, but it is not possible to restore a SHA digest state - * into the engine. - * - * - The memory block SHA_TEXT_BASE is shared between all SHA digest - * engines, so all engines must be idle before this memory block is - * modified. + * - The crypto DMA is shared between the SHA and AES engine, it is not + * possible for them to run calcalutions in parallel. * */ @@ -50,17 +42,11 @@ typedef SHA_TYPE esp_sha_type; /** @brief Calculate SHA1 or SHA2 sum of some data, using hardware SHA engine * * @note For more versatile SHA calculations, where data doesn't need - * to be passed all at once, try the mbedTLS mbedtls/shaX.h APIs. The - * hardware-accelerated mbedTLS implementation is also faster when - * hashing large amounts of data. + * to be passed all at once, try the mbedTLS mbedtls/shaX.h APIs. * * @note It is not necessary to lock any SHA hardware before calling * this function, thread safety is managed internally. * - * @note If a TLS connection is open then this function may block - * indefinitely waiting for a SHA engine to become available. Use the - * mbedTLS SHA API to avoid this problem. - * * @param sha_type SHA algorithm to use. * * @param input Input data buffer. @@ -73,157 +59,106 @@ typedef SHA_TYPE esp_sha_type; */ void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, unsigned char *output); -/* @brief Begin to execute a single SHA block operation +/** @brief Execute SHA block operation using DMA * * @note This is a piece of a SHA algorithm, rather than an entire SHA * algorithm. * - * @note Call esp_sha_try_lock_engine() before calling this - * function. Do not call esp_sha_lock_memory_block() beforehand, this - * is done inside the function. + * @note Call esp_sha_aquire_hardware() before calling this + * function. * * @param sha_type SHA algorithm to use. * - * @param data_block Pointer to block of data. Block size is - * determined by algorithm (SHA1/SHA2_256 = 64 bytes, - * SHA2_384/SHA2_512 = 128 bytes) - * - * @param is_first_block If this parameter is true, the SHA state will - * be initialised (with the initial state of the given SHA algorithm) - * before the block is calculated. If false, the existing state of the - * SHA engine will be used. - * - * @return As a performance optimisation, this function returns before - * the SHA block operation is complete. Both this function and - * esp_sha_read_state() will automatically wait for any previous - * operation to complete before they begin. If using the SHA registers - * directly in another way, call esp_sha_wait_idle() after calling this - * function but before accessing the SHA registers. - */ -void esp_sha_block(esp_sha_type sha_type, const void *data_block, bool is_first_block); - -/* @brief Begin to execute SHA block operation using DMA - * - * @note This is a piece of a SHA algorithm, rather than an entire SHA - * algorithm. - * - * @note Call esp_sha_try_lock_engine() before calling this - * function. Do not call esp_sha_lock_memory_block() beforehand, this - * is done inside the function. - * - * @param sha_type SHA algorithm to use. - * - * @param data_block Pointer to block of data. Block size is + * @param input Pointer to the input data. Block size is * determined by algorithm (SHA1/SHA2_256 = 64 bytes, * SHA2_384/SHA2_512 = 128 bytes) * * @param ilen length of input data should be multiple of block length. * + * @param buf Pointer to blocks of data that will be prepended + * to data_block before hashing. Useful when there is two sources of + * data that need to be efficiently calculated in a single SHA DMA + * operation. + * + * @param buf_len length of buf data should be multiple of block length. + * Should not be longer than the maximum amount of bytes in a single block + * (128 bytes) + * * @param is_first_block If this parameter is true, the SHA state will * be initialised (with the initial state of the given SHA algorithm) * before the block is calculated. If false, the existing state of the * SHA engine will be used. * + * @param t The number of bits for the SHA512/t hash function, with + * output truncated to t bits. Used for calculating the inital hash. + * t is any positive integer between 1 and 512, except 384. + * + * @return 0 if successful */ -int esp_sha_dma(esp_sha_type sha_type, const void *data_block, uint32_t ilen, bool is_first_block); +int esp_sha_dma(esp_sha_type sha_type, const void *input, uint32_t ilen, + const void *buf, uint32_t buf_len, bool is_first_block); -/** @brief Read out the current state of the SHA digest loaded in the engine. +/** + * @brief Read out the current state of the SHA digest * * @note This is a piece of a SHA algorithm, rather than an entire SHA algorithm. * - * @note Call esp_sha_try_lock_engine() before calling this - * function. Do not call esp_sha_lock_memory_block() beforehand, this - * is done inside the function. + * @note Call esp_sha_aquire_hardware() before calling this + * function. * * If the SHA suffix padding block has been executed already, the - * value that is read is the SHA digest (in big endian - * format). Otherwise, the value that is read is an interim SHA state. - * - * @note If sha_type is SHA2_384, only 48 bytes of state will be read. - * This is enough for the final SHA2_384 digest, but if you want the - * interim SHA-384 state (to continue digesting) then pass SHA2_512 instead. + * value that is read is the SHA digest. + * Otherwise, the value that is read is an interim SHA state. * * @param sha_type SHA algorithm in use. - * - * @param state Pointer to a memory buffer to hold the SHA state. Size - * is 20 bytes (SHA1), 32 bytes (SHA2_256), 48 bytes (SHA2_384) or 64 bytes (SHA2_512). - * + * @param digest_state Pointer to a memory buffer to hold the SHA state. Size + * is 20 bytes (SHA1), 32 bytes (SHA2_256), or 64 bytes (SHA2_384, SHA2_512). */ void esp_sha_read_digest_state(esp_sha_type sha_type, void *digest_state); /** - * @brief Obtain exclusive access to a particular SHA engine + * @brief Set the current state of the SHA digest * - * @param sha_type Type of SHA engine to use. + * @note Call esp_sha_aquire_hardware() before calling this + * function. * - * Blocks until engine is available. Note: Can block indefinitely - * while a TLS connection is open, suggest using - * esp_sha_try_lock_engine() and failing over to software SHA. + * When resuming a + * + * @param sha_type SHA algorithm in use. + * @param digest_state */ -void esp_sha_lock_engine(esp_sha_type sha_type); +void esp_sha_write_digest_state(esp_sha_type sha_type, void *digest_state); + /** - * @brief Try and obtain exclusive access to a particular SHA engine - * - * @param sha_type Type of SHA engine to use. - * - * @return Returns true if the SHA engine is locked for exclusive - * use. Call esp_sha_unlock_sha_engine() when done. Returns false if - * the SHA engine is already in use, caller should use software SHA - * algorithm for this digest. + * @brief Enables the SHA and crypto DMA peripheral and takes the + * locks for both of them. */ -bool esp_sha_try_lock_engine(esp_sha_type sha_type); +void esp_sha_acquire_hardware(void); /** - * @brief Unlock an engine previously locked with esp_sha_lock_engine() or esp_sha_try_lock_engine() - * - * @param sha_type Type of engine to release. + * @brief Disables the SHA and crypto DMA peripheral and releases the + * locks. */ -void esp_sha_unlock_engine(esp_sha_type sha_type); +void esp_sha_release_hardware(void); + +/* +*/ /** - * @brief Acquire exclusive access to the SHA shared memory block at SHA_TEXT_BASE + * @brief Sets the initial hash value for SHA512/t. * - * This memory block is shared across all the SHA algorithm types. + * @note Is generated according to the algorithm described in the TRM, + * chapter SHA-Accelerator * - * Caller should have already locked a SHA engine before calling this function. + * @note The engine must be locked until the value is used for an operation + * or read out. Else you risk another operation overwriting it. * - * Note that it is possible to obtain exclusive access to the memory block even - * while it is in use by the SHA engine. Caller should use esp_sha_wait_idle() - * to ensure the SHA engine is not reading from the memory block in hardware. + * @param t * - * @note You do not need to lock the memory block before calling esp_sha_block() or esp_sha_read_digest_state(), these functions handle memory block locking internally. - * - * Call esp_sha_unlock_memory_block() when done. + * @return 0 if successful */ -void esp_sha_lock_memory_block(void); - -/** - * @brief Release exclusive access to the SHA register memory block at SHA_TEXT_BASE - * - * Caller should have already locked a SHA engine before calling this function. - * - * Call following esp_sha_lock_memory_block(). - */ -void esp_sha_unlock_memory_block(void); - -/** @brief Wait for the SHA engine to finish any current operation - * - * @note This function does not ensure exclusive access to any SHA - * engine. Caller should use esp_sha_try_lock_engine() and - * esp_sha_lock_memory_block() as required. - * - * @note Functions declared in this header file wait for SHA engine - * completion automatically, so you don't need to use this API for - * these. However if accessing SHA registers directly, you will need - * to call this before accessing SHA registers if using the - * esp_sha_block() function. - * - * @note This function busy-waits, so wastes CPU resources. - * Best to delay calling until you are about to need it. - * - */ -void esp_sha_wait_idle(void); +int esp_sha_512_t_init_hash(uint16_t t); #ifdef __cplusplus } diff --git a/components/mbedtls/port/include/gcm_alt.h b/components/mbedtls/port/include/gcm_alt.h new file mode 100644 index 000000000..c3d886a7c --- /dev/null +++ b/components/mbedtls/port/include/gcm_alt.h @@ -0,0 +1,55 @@ +/** + * \file gcm_alt.h + * + * \brief AES block cipher + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + */ +#ifndef GCM_ALT_H +#define GCM_ALT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(MBEDTLS_GCM_ALT) + +#if CONFIG_IDF_TARGET_ESP32S2 +#include "esp32s2/gcm.h" + + +typedef esp_gcm_context mbedtls_gcm_context; + +#define mbedtls_gcm_init esp_aes_gcm_init +#define mbedtls_gcm_free esp_aes_gcm_free +#define mbedtls_gcm_setkey esp_aes_gcm_setkey +#define mbedtls_gcm_starts esp_aes_gcm_starts +#define mbedtls_gcm_update esp_aes_gcm_update +#define mbedtls_gcm_finish esp_aes_gcm_finish +#define mbedtls_gcm_auth_decrypt esp_aes_gcm_auth_decrypt +#define mbedtls_gcm_crypt_and_tag esp_aes_gcm_crypt_and_tag + +#endif // CONFIG_IDF_TARGET_ESP32S2 + +#endif /* MBEDTLS_GCM_ALT */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index f702ae1d5..5519396f9 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -118,6 +118,12 @@ #undef MBEDTLS_AES_ALT #endif +#ifdef CONFIG_MBEDTLS_HARDWARE_GCM +#define MBEDTLS_GCM_ALT +#else +#undef MBEDTLS_GCM_ALT +#endif + /* MBEDTLS_SHAxx_ALT to enable hardware SHA support with software fallback. */ diff --git a/components/mbedtls/port/include/sha1_alt.h b/components/mbedtls/port/include/sha1_alt.h index 54b774087..a3f05a84c 100644 --- a/components/mbedtls/port/include/sha1_alt.h +++ b/components/mbedtls/port/include/sha1_alt.h @@ -29,6 +29,30 @@ extern "C" { #if defined(MBEDTLS_SHA1_ALT) +#if CONFIG_IDF_TARGET_ESP32S2 + +#include "esp32s2/sha.h" +typedef enum { + ESP_SHA1_STATE_INIT, + ESP_SHA1_STATE_IN_PROCESS +} esp_sha1_state; + +/** + * \brief SHA-1 context structure + */ +typedef struct { + uint32_t total[2]; /*!< number of bytes processed */ + uint32_t state[5]; /*!< intermediate digest state */ + unsigned char buffer[64]; /*!< data block being processed */ + int first_block; /*!< if first then true else false */ + esp_sha_type mode; + esp_sha1_state sha_state; +} mbedtls_sha1_context; + +#endif //CONFIG_IDF_TARGET_ESP32S2 + +#if CONFIG_IDF_TARGET_ESP32 + typedef enum { ESP_MBEDTLS_SHA1_UNUSED, /* first block hasn't been processed yet */ ESP_MBEDTLS_SHA1_HARDWARE, /* using hardware SHA engine */ @@ -38,8 +62,7 @@ typedef enum { /** * \brief SHA-1 context structure */ -typedef struct -{ +typedef struct { uint32_t total[2]; /*!< number of bytes processed */ uint32_t state[5]; /*!< intermediate digest state */ unsigned char buffer[64]; /*!< data block being processed */ @@ -47,6 +70,8 @@ typedef struct } mbedtls_sha1_context; +#endif //CONFIG_IDF_TARGET_ESP32 + #endif #ifdef __cplusplus diff --git a/components/mbedtls/port/include/sha256_alt.h b/components/mbedtls/port/include/sha256_alt.h index 436f5324c..db82fb645 100644 --- a/components/mbedtls/port/include/sha256_alt.h +++ b/components/mbedtls/port/include/sha256_alt.h @@ -29,6 +29,33 @@ extern "C" { #if defined(MBEDTLS_SHA256_ALT) + +#if CONFIG_IDF_TARGET_ESP32S2 + +#include "esp32s2/sha.h" + +typedef enum { + ESP_SHA256_STATE_INIT, + ESP_SHA256_STATE_IN_PROCESS +} esp_sha256_state; + +/** + * \brief SHA-256 context structure + */ +typedef struct { + uint32_t total[2]; /*!< number of bytes processed */ + uint32_t state[8]; /*!< intermediate digest state */ + unsigned char buffer[64]; /*!< data block being processed */ + int first_block; /*!< if first then true, else false */ + esp_sha_type mode; + esp_sha256_state sha_state; +} +mbedtls_sha256_context; + +#endif //CONFIG_IDF_TARGET_ESP32S2 + +#if CONFIG_IDF_TARGET_ESP32 + typedef enum { ESP_MBEDTLS_SHA256_UNUSED, /* first block hasn't been processed yet */ ESP_MBEDTLS_SHA256_HARDWARE, /* using hardware SHA engine */ @@ -38,8 +65,7 @@ typedef enum { /** * \brief SHA-256 context structure */ -typedef struct -{ +typedef struct { uint32_t total[2]; /*!< number of bytes processed */ uint32_t state[8]; /*!< intermediate digest state */ unsigned char buffer[64]; /*!< data block being processed */ @@ -48,6 +74,8 @@ typedef struct } mbedtls_sha256_context; +#endif //CONFIG_IDF_TARGET_ESP32 + #endif #ifdef __cplusplus diff --git a/components/mbedtls/port/include/sha512_alt.h b/components/mbedtls/port/include/sha512_alt.h index 36b8fc9d2..da58e8a71 100644 --- a/components/mbedtls/port/include/sha512_alt.h +++ b/components/mbedtls/port/include/sha512_alt.h @@ -23,12 +23,51 @@ #ifndef _SHA512_ALT_H_ #define _SHA512_ALT_H_ + #ifdef __cplusplus extern "C" { #endif #if defined(MBEDTLS_SHA512_ALT) +#if CONFIG_IDF_TARGET_ESP32S2 +#include "esp32s2/sha.h" + +typedef enum { + ESP_SHA512_STATE_INIT, + ESP_SHA512_STATE_IN_PROCESS +} esp_sha512_state; + +/** + * \brief SHA-512 context structure + */ +typedef struct { + uint64_t total[2]; /*!< number of bytes processed */ + uint64_t state[8]; /*!< intermediate digest state */ + unsigned char buffer[128]; /*!< data block being processed */ + int first_block; + esp_sha_type mode; + uint32_t t_val; /*!< t_val for 512/t mode */ + esp_sha512_state sha_state; +} mbedtls_sha512_context; + +/** + * @brief Sets the specfic algorithm for SHA512 + * + * @param ctx The mbedtls sha512 context + * + * @param type The mode, used for setting SHA2_512224 and SHA2_512256: + * + */ +void esp_sha512_set_mode(mbedtls_sha512_context *ctx, esp_sha_type type); + +/* For SHA512/t mode the intial hash value will depend on t */ +void esp_sha512_set_t( mbedtls_sha512_context *ctx, uint16_t t_val); + +#endif //CONFIG_IDF_TARGET_ESP32S2 + +#if CONFIG_IDF_TARGET_ESP32 + typedef enum { ESP_MBEDTLS_SHA512_UNUSED, /* first block hasn't been processed yet */ ESP_MBEDTLS_SHA512_HARDWARE, /* using hardware SHA engine */ @@ -38,8 +77,7 @@ typedef enum { /** * \brief SHA-512 context structure */ -typedef struct -{ +typedef struct { uint64_t total[2]; /*!< number of bytes processed */ uint64_t state[8]; /*!< intermediate digest state */ unsigned char buffer[128]; /*!< data block being processed */ @@ -48,6 +86,8 @@ typedef struct } mbedtls_sha512_context; +#endif //CONFIG_IDF_TARGET_ESP32 + #endif #ifdef __cplusplus diff --git a/components/mbedtls/test/test_aes.c b/components/mbedtls/test/test_aes.c new file mode 100644 index 000000000..27ce6d357 --- /dev/null +++ b/components/mbedtls/test/test_aes.c @@ -0,0 +1,1025 @@ +/* mbedTLS AES test +*/ +#include +#include +#include +#include +#include "mbedtls/aes.h" +#include "mbedtls/gcm.h" +#include "unity.h" +#include "sdkconfig.h" +#include "esp_timer.h" +#include "esp_heap_caps.h" +#include "test_utils.h" + +TEST_CASE("mbedtls CTR stream test", "[aes]") +{ + const unsigned SZ = 100; + mbedtls_aes_context ctx; + uint8_t nonce[16]; + uint8_t key[16]; + uint8_t stream_block[16]; + + /* Cipher produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + nonce = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.CTR(nonce), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 100) + encryptor.finalize() + ct_arr = "" + for idx, b in enumerate(ct): + if idx % 8 == 0: + ct_arr += '\n' + ct_arr += "0x{}, ".format(binascii.hexlify(b)) + print(ct_arr) + */ + const uint8_t expected_cipher[] = { + 0xc5, 0x78, 0xa7, 0xb4, 0xf3, 0xb9, 0xcb, 0x8b, + 0x09, 0xe0, 0xd6, 0x89, 0x14, 0x6a, 0x19, 0x09, + 0xde, 0xaf, 0x37, 0x19, 0x32, 0x4d, 0xca, 0xf6, + 0xff, 0x6e, 0xd2, 0x5d, 0x87, 0x51, 0xaa, 0x8c, + 0x1c, 0xe3, 0x3b, 0xbb, 0x18, 0xf5, 0xa0, 0x1b, + 0xdc, 0x29, 0x52, 0x63, 0xf6, 0x5d, 0x49, 0x85, + 0x29, 0xf1, 0xf0, 0x69, 0x8f, 0xa6, 0x9f, 0x38, + 0x5c, 0xdd, 0x26, 0xf8, 0x9d, 0x40, 0xa1, 0xff, + 0x52, 0x46, 0xe1, 0x72, 0x70, 0x39, 0x73, 0xff, + 0xd0, 0x5e, 0xe5, 0x3f, 0xc5, 0xed, 0x5c, 0x18, + 0xa7, 0x84, 0xd8, 0xdf, 0x9d, 0xb5, 0x06, 0xb1, + 0xa7, 0xcf, 0x2e, 0x7a, 0x51, 0xfc, 0x44, 0xc5, + 0xb9, 0x5f, 0x22, 0x47, + }; + + + memset(nonce, 0xEE, 16); + memset(key, 0x44, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *plaintext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + memset(plaintext, 0xAA, SZ); + + /* Test that all the end results are the same + no matter how many bytes we encrypt each call + */ + for (int bytes_to_process = 1; bytes_to_process < SZ; bytes_to_process++) { + + memset(nonce, 0xEE, 16); + memset(chipertext, 0x0, SZ); + memset(decryptedtext, 0x0, SZ); + + size_t offset = 0; + + // Encrypt + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = (idx + bytes_to_process > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_ctr(&ctx, length, &offset, nonce, + stream_block, plaintext+idx, chipertext+idx ); + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher, chipertext, SZ); + + // Decrypt + memset(nonce, 0xEE, 16); + offset = 0; + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = (idx + bytes_to_process > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_ctr(&ctx, length, &offset, nonce, + stream_block, chipertext+idx, decryptedtext+idx ); + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + } + free(plaintext); + free(chipertext); + free(decryptedtext); +} + +TEST_CASE("mbedtls OFB stream test", "[aes]") +{ + const unsigned SZ = 100; + mbedtls_aes_context ctx; + uint8_t iv[16]; + uint8_t key[16]; + + /* Cipher produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + iv = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.OFB(iv), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 100) + encryptor.finalize() + ct_arr = "" + for idx, b in enumerate(ct): + if idx % 8 == 0: + ct_arr += '\n' + ct_arr += "0x{}, ".format(binascii.hexlify(b)) + print(ct_arr) + */ + const uint8_t expected_cipher[] = { + 0xc5, 0x78, 0xa7, 0xb4, 0xf3, 0xb9, 0xcb, 0x8b, + 0x09, 0xe0, 0xd6, 0x89, 0x14, 0x6a, 0x19, 0x09, + 0x0a, 0x33, 0x8b, 0xab, 0x82, 0xcb, 0x20, 0x8f, + 0x74, 0x2a, 0x6c, 0xb3, 0xc6, 0xe8, 0x18, 0x89, + 0x09, 0xb6, 0xaf, 0x20, 0xcd, 0xea, 0x74, 0x14, + 0x48, 0x61, 0xe8, 0x4d, 0x50, 0x12, 0x9f, 0x5e, + 0xb8, 0x10, 0x53, 0x3b, 0x74, 0xd9, 0xd0, 0x95, + 0x13, 0xdc, 0x14, 0xcf, 0x0c, 0xa1, 0x90, 0xfd, + 0xa2, 0x58, 0x12, 0xb2, 0x00, 0x2c, 0x5b, 0x7a, + 0x2a, 0x76, 0x80, 0x20, 0x82, 0x39, 0xa2, 0x21, + 0xf8, 0x7a, 0xec, 0xae, 0x82, 0x6a, 0x5c, 0xd3, + 0x04, 0xd9, 0xbd, 0xe4, 0x53, 0xc9, 0xdf, 0x67, + 0xaa, 0x5c, 0xaf, 0xa6, + }; + + + memset(key, 0x44, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *plaintext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + memset(plaintext, 0xAA, SZ); + + /* Test that all the end results are the same + no matter how many bytes we encrypt each call + */ + + for (int bytes_to_process = 1; bytes_to_process < SZ; bytes_to_process++) { + // Encrypt + memset(iv, 0xEE, 16); + size_t offset = 0; + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = ( (idx + bytes_to_process) > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_ofb(&ctx, length, &offset, iv, plaintext + idx, chipertext + idx); + + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher, chipertext, SZ); + + // Decrypt + memset(iv, 0xEE, 16); + offset = 0; + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = (idx + bytes_to_process > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_ofb(&ctx, length, &offset, iv, chipertext + idx, decryptedtext + idx); + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + } + free(plaintext); + free(chipertext); + free(decryptedtext); +} + +TEST_CASE("mbedtls CFB8 stream test", "[aes]") +{ + const unsigned SZ = 32; + mbedtls_aes_context ctx; + uint8_t iv[16]; + uint8_t key[16]; + + /* Cipher produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + iv = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.CFB8(iv), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 100) + encryptor.finalize() + ct_arr = "" + for idx, b in enumerate(ct): + if idx % 8 == 0: + ct_arr += '\n' + ct_arr += "0x{}, ".format(binascii.hexlify(b)) + print(ct_arr) + */ + const uint8_t expected_cipher[] = { + 0xc5, 0x2f, 0xb0, 0x9b, 0x94, 0x9c, 0xa4, 0x5c, + 0x0f, 0x4d, 0xa1, 0x9d, 0xd1, 0x19, 0xfc, 0x04, + 0xe2, 0x7f, 0x04, 0x82, 0x6a, 0xa3, 0x61, 0xbb, + 0x07, 0x6f, 0xac, 0xb9, 0xdf, 0x00, 0xf9, 0xa8, + 0xc4, 0xbe, 0x9d, 0x4d, 0xd9, 0x42, 0x8a, 0x83, + 0x12, 0x8b, 0xeb, 0xd7, 0x88, 0x70, 0x8a, 0xed, + 0x46, 0x81, 0x5b, 0x4c, 0x14, 0x67, 0xe0, 0xfb, + 0xab, 0x34, 0x90, 0x85, 0x24, 0xd2, 0x6b, 0x64, + 0xdf, 0x1d, 0x04, 0xfd, 0x69, 0xf6, 0x30, 0xbe, + 0xa6, 0xac, 0x0b, 0x54, 0x25, 0x24, 0x67, 0xd6, + 0x09, 0xb1, 0x8f, 0x91, 0x63, 0xbd, 0xdf, 0xa1, + 0x8a, 0xa3, 0x2e, 0xeb, 0x15, 0x7d, 0xe5, 0x37, + 0xe5, 0x5a, 0x9f, 0xa5, + }; + + + memset(key, 0x44, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *plaintext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + memset(plaintext, 0xAA, SZ); + + /* Test that all the end results are the same + no matter how many bytes we encrypt each call + */ + + for (int bytes_to_process = 1; bytes_to_process < SZ; bytes_to_process++) { + memset(iv, 0xEE, 16); + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = ( (idx + bytes_to_process) > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_cfb8(&ctx, MBEDTLS_AES_ENCRYPT, length, iv, plaintext + idx, chipertext + idx); + + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher, chipertext, SZ); + + memset(iv, 0xEE, 16); + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = ( (idx + bytes_to_process) > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_cfb8(&ctx, MBEDTLS_AES_DECRYPT, length, iv, chipertext + idx, decryptedtext + idx); + + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + } + free(plaintext); + free(chipertext); + free(decryptedtext); +} + +TEST_CASE("mbedtls CFB128 stream test", "[aes]") +{ + const unsigned SZ = 32; + mbedtls_aes_context ctx; + uint8_t iv[16]; + uint8_t key[16]; + + /* Cipher produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + iv = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.CFB(iv), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 100) + encryptor.finalize() + ct_arr = "" + for idx, b in enumerate(ct): + if idx % 8 == 0: + ct_arr += '\n' + ct_arr += "0x{}, ".format(binascii.hexlify(b)) + print(ct_arr) + */ + const uint8_t expected_cipher[] = { + 0xc5, 0x78, 0xa7, 0xb4, 0xf3, 0xb9, 0xcb, 0x8b, + 0x09, 0xe0, 0xd6, 0x89, 0x14, 0x6a, 0x19, 0x09, + 0xf9, 0x08, 0x7e, 0xe1, 0x92, 0x8a, 0x7c, 0xa4, + 0x25, 0xa5, 0xa7, 0x43, 0x24, 0x8d, 0x85, 0x3e, + 0x99, 0x28, 0xeb, 0x36, 0x59, 0x74, 0x69, 0x0e, + 0x09, 0x9f, 0x4e, 0xc0, 0x6d, 0xc3, 0x2b, 0x80, + 0x01, 0xad, 0xa1, 0x0c, 0x99, 0x90, 0x8b, 0x07, + 0xd6, 0x00, 0xf0, 0x32, 0xd7, 0x6b, 0xa1, 0xf1, + 0x4d, 0x14, 0xd0, 0x28, 0xde, 0x64, 0x23, 0x71, + 0xf4, 0x23, 0x61, 0x12, 0x71, 0xbe, 0x03, 0x74, + 0x99, 0x81, 0x9d, 0x65, 0x48, 0xd9, 0xd4, 0x67, + 0xd1, 0x31, 0xe8, 0x44, 0x27, 0x17, 0xd4, 0x2d, + 0x3d, 0x59, 0xf7, 0xd3, + }; + + + memset(key, 0x44, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *plaintext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + memset(plaintext, 0xAA, SZ); + + /* Test that all the end results are the same + no matter how many bytes we encrypt each call + */ + + //for (int bytes_to_process = 1; bytes_to_process < SZ; bytes_to_process++) { + int bytes_to_process = 17; + size_t offset = 0; + memset(iv, 0xEE, 16); + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = ( (idx + bytes_to_process) > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_cfb128(&ctx, MBEDTLS_AES_ENCRYPT, length, &offset, iv, plaintext + idx, chipertext + idx); + + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher, chipertext, SZ); + + offset = 0; + memset(iv, 0xEE, 16); + for (int idx = 0; idx < SZ; idx = idx + bytes_to_process) { + // Limit length of last call to avoid exceeding buffer size + size_t length = ( (idx + bytes_to_process) > SZ) ? (SZ - idx) : bytes_to_process; + mbedtls_aes_crypt_cfb128(&ctx, MBEDTLS_AES_DECRYPT, length, &offset, iv, chipertext + idx, decryptedtext + idx); + + } + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + + free(plaintext); + free(chipertext); + free(decryptedtext); +} + +/* Cipher produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + nonce = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.CTR(nonce), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 100) + encryptor.finalize() + ct_arr = "" + for idx, b in enumerate(ct): + if idx % 8 == 0: + ct_arr += '\n' + ct_arr += "0x{}, ".format(binascii.hexlify(b)) + print(ct_arr) +*/ + + + + +#ifdef CONFIG_SPIRAM_USE_MALLOC + +const uint8_t expected_cipher_psram_end[] = { + 0x7e, 0xdf, 0x13, 0xf3, 0x56, 0xef, 0x67, 0x01, + 0xfc, 0x08, 0x49, 0x62, 0xfa, 0xfe, 0x0c, 0x8b, + 0x99, 0x39, 0x09, 0x51, 0x2c, 0x9a, 0xd5, 0x48, + 0x4f, 0x76, 0xa2, 0x19, 0x2c, 0x08, 0x9d, 0x6a, + }; + + +void aes_psram_ctr_test(uint32_t input_buf_caps, uint32_t output_buf_caps) +{ + mbedtls_aes_context ctx; + uint8_t nonce[16]; + uint8_t key[16]; + uint8_t stream_block[16]; + size_t SZ = 6000; + size_t ALIGNMENT_SIZE_BYTES = 16; + memset(nonce, 0x2F, 16); + memset(key, 0x1E, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ + ALIGNMENT_SIZE_BYTES, output_buf_caps); + uint8_t *plaintext = heap_caps_malloc(SZ + ALIGNMENT_SIZE_BYTES, input_buf_caps); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(plaintext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + memset(plaintext, 0x26, SZ + ALIGNMENT_SIZE_BYTES); + + size_t offset; + + /* Shift buffers and test for all different misalignments */ + for (int i = 0; i < ALIGNMENT_SIZE_BYTES; i++ ) { + // Encrypt with input buffer in external ram + offset = 0; + memset(nonce, 0x2F, 16); + mbedtls_aes_crypt_ctr(&ctx, SZ, &offset, nonce, stream_block, plaintext + i, chipertext + i); + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher_psram_end, chipertext + i + SZ - 32, 32); + + // Decrypt + offset = 0; + memset(nonce, 0x2F, 16); + // Decrypt with input buffer in instruction memory, the crypto DMA can't access this + mbedtls_aes_crypt_ctr(&ctx, SZ, &offset, nonce, stream_block, chipertext + i, decryptedtext); + + TEST_ASSERT_EQUAL_HEX8_ARRAY(plaintext, decryptedtext, SZ); + + } + + free(plaintext); + free(chipertext); + free(decryptedtext); +} + +const uint8_t long_input[] = { + 0xf7, 0xe6, 0x6b, 0x8d, 0x2e, 0xbf, 0x88, 0xd6, + 0xb0, 0x77, 0xdf, 0x72, 0xbf, 0xa8, 0x0, 0x55, + 0xd5, 0xd1, 0x49, 0xa3, 0x2c, 0xc, 0xfe, 0xdb, + 0x17, 0x37, 0xa4, 0x1d, 0x70, 0x6b, 0x99, 0xf5, + 0x9e, 0x6, 0xad, 0x6c, 0xe0, 0x3b, 0xfa, 0x50, + 0x28, 0xb2, 0x62, 0xf2, 0x99, 0x3a, 0xcc, 0xe4, + 0x86, 0x5f, 0x1, 0xf8, 0x69, 0xd7, 0xf5, 0xb2, + 0x8a, 0x5f, 0x5c, 0x38, 0x9f, 0x8a, 0xb8, 0x8c, + 0xea, 0x6, 0xe1, 0x68, 0xff, 0xaf, 0x5d, 0xd9, + 0x1f, 0xa5, 0x5c, 0x8c, 0x52, 0xa1, 0x5f, 0x45, + 0x55, 0xcb, 0x76, 0x59, 0x8f, 0xfe, 0x36, 0xd0, + 0x85, 0x1f, 0x8, 0x90, 0x6f, 0x62, 0xb1, 0x1a, + 0xde, 0x75, 0xab, 0x90, 0xb7, 0x75, 0xe9, 0xa0, + 0xa9, 0xb0, 0xac, 0x61, 0x5, 0x6d, 0x9a, 0xe3, + 0x3b, 0x43, 0x61, 0x13, 0x8c, 0x3a, 0xa0, 0xaa, + 0x91, 0xea, 0x3e, 0xe1, 0x87, 0x35, 0xff, 0x90, + 0xe2, 0x43, 0xa3, 0x70, 0x57, 0x65, 0x2d, 0xa2, + 0x65, 0xe6, 0xde, 0xb0, 0x52, 0x85, 0x5b, 0xb8, + 0x3, 0x8, 0x63, 0x8b, 0xa1, 0xc2, 0xe1, 0x35, + 0x2e, 0xba, 0xe0, 0x84, 0x56, 0x52, 0x5f, 0x12, + 0xd3, 0x22, 0x8d, 0xa5, 0xbb, 0xe1, 0xd3, 0xfc, + 0x18, 0x1c, 0x90, 0x3b, 0x79, 0xe, 0xab, 0x2d, + 0x5e, 0xb0, 0x7, 0xbb, 0x46, 0x73, 0x1d, 0x35, + 0xd9, 0xc5, 0xa7, 0x87, 0x80, 0xf7, 0xee, 0x29, + 0xb5, 0x17, 0xf3, 0xaf, 0x30, 0xe5, 0x19, 0x50, + 0xf9, 0x5d, 0x2b, 0xc3, 0xc0, 0xda, 0x8f, 0xca, + 0x3c, 0x4d, 0xd5, 0xd7, 0x6c, 0xd2, 0x36, 0xa4, + 0x22, 0x8, 0x66, 0x48, 0x31, 0xb4, 0x3d, 0xc2, + 0xf6, 0x6b, 0xce, 0xf0, 0x12, 0xe4, 0x38, 0x5c, + 0xd8, 0x71, 0xea, 0x30, 0x52, 0xdf, 0x34, 0x62, + 0xdc, 0xb4, 0x30, 0xe, 0x74, 0xc, 0x5, 0x14, + 0xf, 0x47, 0x25, 0x5, 0x72, 0xc9, 0x14, 0x7c, + 0x1f, 0x6e, 0xdb, 0x6f, 0x83, 0x6, 0xa0, 0xb2, + 0x7f, 0x29, 0xe6, 0xb6, 0xe3, 0x11, 0x23, 0x4b, + 0x68, 0x92, 0xa, 0x49, 0xb5, 0x9d, 0x5d, 0x39, + 0x90, 0xff, 0x9, 0xa0, 0xa, 0x69, 0x6b, 0x2, + 0x18, 0xfb, 0xca, 0x5a, 0x91, 0x1a, 0xd9, 0x19, + 0x6b, 0xd4, 0x92, 0xd3, 0xd9, 0x7, 0xce, 0xcb, + 0xc7, 0xf3, 0xa1, 0x33, 0xcd, 0xa9, 0xb1, 0x44, + 0x8c, 0x93, 0xcd, 0xac, 0xc1, 0x44, 0x12, 0x48, + 0x95, 0x3, 0xdf, 0xc, 0x2f, 0xfc, 0x34, 0x8d, + 0x3, 0xde, 0xc1, 0xed, 0xdc, 0xf0, 0xfa, 0xa5, + 0xb2, 0x62, 0xcd, 0xa2, 0xbf, 0xf7, 0x7e, 0x47, + 0xb6, 0xcc, 0xe4, 0xa6, 0x4e, 0x51, 0xc6, 0x34, + 0xee, 0x83, 0x21, 0xb7, 0xc2, 0xe3, 0x13, 0x92, + 0xfc, 0xc9, 0x6, 0x6b, 0x91, 0x76, 0x7b, 0x2e, + 0x1e, 0xa2, 0xe0, 0x17, 0xab, 0x10, 0xfa, 0xac, + 0xd1, 0x2, 0x33, 0xb0, 0xd3, 0x3d, 0xb9, 0xce, + 0xea, 0xe9, 0x93, 0x5c, 0x98, 0x14, 0x0, 0xc6, + 0x2c, 0xa6, 0xdb, 0x1f, 0xdc, 0x76, 0xfb, 0xeb, + 0x9d, 0x55, 0xa6, 0x5f, 0xd5, 0x8e, 0x13, 0x39, + 0x88, 0x58, 0xff, 0xe8, 0xb4, 0x98, 0x9e, 0x4b, + 0xe7, 0x46, 0xdc, 0x7a, 0x68, 0x5b, 0xa8, 0xc2, + 0xe5, 0xa9, 0x50, 0xe2, 0x8, 0x31, 0x6, 0x3e, + 0x8e, 0xaf, 0x80, 0x24, 0x4e, 0xbd, 0x73, 0x6d, + 0xd9, 0x4b, 0xb4, 0x3e, 0x84, 0x5e, 0x31, 0x8e, + 0xf7, 0xa8, 0x9b, 0x5e, 0x2c, 0xd5, 0xe9, 0x7c, + 0xca, 0xca, 0xfa, 0x8e, 0x87, 0xbf, 0xf5, 0xa3, + 0x2f, 0x73, 0x2f, 0xc0, 0x5f, 0x46, 0xf4, 0x2, + 0xfd, 0xd1, 0x23, 0x6f, 0xc2, 0xc1, 0xc0, 0x86, + 0x62, 0x43, 0xc3, 0x44, 0x3b, 0x2c, 0x3d, 0xc2, + 0xd5, 0xe0, 0x2, 0xae, 0x1, 0x5a, 0x9, 0x89, + 0x52, 0x34, 0xdf, 0xb1, 0x6c, 0x2b, 0x85, 0x77, + 0xa5, 0x83, 0xe3, 0xa5, 0x50, 0x13, 0x2f, 0xf3, + 0xa6, 0x83, 0x60, 0x33, 0xba, 0xd5, 0xd2, 0x96, + 0x8a, 0xcd, 0xee, 0xfa, 0x76, 0x2a, 0x63, 0xec, + 0x41, 0x3a, 0xf3, 0xe5, 0x9e, 0x1d, 0x5e, 0x46, + 0x8, 0xd7, 0xe2, 0x3a, 0x25, 0x6f, 0x37, 0x7e, + 0x0, 0x2d, 0x3d, 0x1b, 0x86, 0xf4, 0xbe, 0x0, + 0x3c, 0xda, 0x82, 0x4a, 0xa3, 0x8, 0x2a, 0x38, + 0x95, 0xe, 0x38, 0xf8, 0x18, 0x6c, 0x42, 0x6f, + 0x30, 0x19, 0x8e, 0x22, 0xf6, 0xb7, 0x18, 0xb7, + 0x93, 0xd, 0x54, 0x72, 0x4, 0x64, 0xc1, 0x19, + 0x76, 0x6e, 0xfc, 0x9e, 0xb0, 0x7c, 0x20, 0x37, + 0xb0, 0xcb, 0x82, 0x3a, 0x20, 0x1d, 0x12, 0x44, + 0xbf, 0x44, 0xc4, 0x4d, 0x33, 0x7e, 0x7b, 0xeb, + 0xd8, 0xb8, 0xa1, 0x75, 0x9e, 0x47, 0x99, 0x64, + 0x92, 0xd3, 0x21, 0x1d, 0x72, 0x63, 0xc7, 0xb3, + 0x3d, 0xfc, 0xb9, 0x4, 0x65, 0x18, 0x94, 0xcc, + 0x20, 0xfe, 0x6f, 0x66, 0x36, 0xba, 0x36, 0x2a, + 0x7, 0xf0, 0x5e, 0x8a, 0xf2, 0x7, 0x1e, 0x9e, + 0x47, 0x2a, 0xc3, 0x7d, 0x7a, 0x20, 0x3c, 0x30, + 0x6f, 0xbe, 0x43, 0x5e, 0x71, 0x6f, 0xd, 0xb8, + 0x3d, 0x1d, 0x3e, 0x18, 0x65, 0x62, 0x75, 0xe8, + 0x34, 0xfd, 0x72, 0xbb, 0xd9, 0x3f, 0xf0, 0xa2, + 0x55, 0xee, 0x91, 0x12, 0x88, 0xda, 0x7, 0x3d, + 0x44, 0x88, 0x70, 0x1f, 0xe0, 0xbe, 0x4b, 0x88, + 0xa8, 0x8e, 0x28, 0x7, 0x73, 0xfd, 0x3f, 0xff, + 0x3e, 0xb2, 0xb5, 0xdb, 0x18, 0x48, 0x9e, 0x73, + 0x6e, 0xd7, 0x24, 0xa9, 0x25, 0xdb, 0x4, 0xe0, + 0xe0, 0xf4, 0x45, 0xc0, 0x1b, 0x82, 0xdf, 0x4e, + 0x48, 0x60, 0x85, 0x9c, 0xd8, 0x90, 0x32, 0xca, + 0x4b, 0xf9, 0xb4, 0xb8, 0xe1, 0xfe, 0xd2, 0xe0, + 0xb2, 0xd6, 0xb8, 0x19, 0x38, 0x34, 0x17, 0x8d, + 0x5e, 0xdf, 0xf4, 0xf1, 0xac, 0x2c, 0x88, 0x7f, + 0x54, 0xbc, 0xf1, 0x39, 0xf2, 0xaf, 0x5a, 0xff, + 0xa7, 0x96, 0x0, 0xf0, 0x27, 0x79, 0x27, 0x2e, + 0x9c, 0xf1, 0x4b, 0xa3, 0xad, 0xdc, 0x8a, 0x2c, + 0x9, 0x4c, 0xd3, 0xcd, 0xd0, 0x2d, 0xb1, 0xec, + 0x4d, 0x68, 0x40, 0xb8, 0xc5, 0x5, 0xfa, 0xb2, + 0x61, 0xb8, 0x31, 0x5, 0xea, 0xb8, 0xa3, 0x34, + 0xa8, 0x8b, 0x3, 0x5b, 0x22, 0x93, 0xba, 0x91, + 0x33, 0x3f, 0x8b, 0x5e, 0xed, 0x86, 0x23, 0x95, + 0xbc, 0x9e, 0xdf, 0xa9, 0x8c, 0xca, 0xb9, 0x97, + 0x9b, 0xc5, 0xca, 0xf4, 0xff, 0x4d, 0x62, 0x52, + 0x1c, 0xd3, 0x4c, 0x42, 0xbf, 0x8a, 0x25, 0x47, + 0xc7, 0x9, 0x4e, 0xe0, 0xb1, 0x72, 0x7d, 0x2, + 0x8f, 0xca, 0x4f, 0x4, 0xc8, 0x74, 0x82, 0x8e, + 0x53, 0xfd, 0xa1, 0x37, 0xda, 0x29, 0x5c, 0xa3, + 0x83, 0xe9, 0xa8, 0xd8, 0x25, 0x27, 0xfe, 0xf7, + 0x41, 0xc4, 0xb0, 0xee, 0x1d, 0x89, 0x1c, 0xe7, + 0xef, 0x86, 0x68, 0xd8, 0x87, 0x4c, 0x4f, 0x49, + 0xeb, 0xbc, 0xb3, 0x81, 0xa7, 0xf4, 0xb4, 0x9b, + 0xc1, 0x52, 0x93, 0x7e, 0xdf, 0x75, 0x75, 0xfc, + 0x45, 0xb2, 0x86, 0xa9, 0x50, 0xb5, 0xa3, 0xf7, + 0x61, 0x60, 0xe4, 0x13, 0x99, 0xc0, 0xf8, 0x49, + 0x7b, 0x61, 0x8b, 0xa8, 0xfa, 0x77, 0x0, 0xe4, + 0x6, 0x9a, 0xc5, 0x51, 0xe4, 0xeb, 0xaf, 0x5f, + 0xb9, 0x5c, 0x74, 0xc8, 0xf8, 0x3e, 0x62, 0x26, + 0xe, 0xe5, 0x85, 0xca, 0x49, 0xa0, 0x2f, 0xf7, + 0x7, 0x99, 0x3e, 0x5c, 0xe0, 0x72, 0xfa, 0xd4, + 0x80, 0x2e, 0xd6, 0x40, 0x6, 0xde, 0x5f, 0xc5, + 0xc5, 0x1, 0xd, 0xbf, 0xdb, 0xb6, 0xb3, 0x92, + 0x76, 0xb3, 0x3f, 0x3d, 0x5d, 0x1, 0x23, 0xb8, + 0xa, 0xcb, 0x80, 0x17, 0x31, 0x19, 0xc7, 0x64, + 0x69, 0xf1, 0x99, 0x53, 0xe5, 0xf2, 0x9f, 0x9d, + 0x3c, 0xda, 0xcb, 0xa6, 0x94, 0x94, 0x44, 0xd3, + 0xc6, 0x8b, 0xb5, 0xae, 0x45, 0x25, 0xef, 0x2a, + 0x24, 0x1, 0x3a, 0xf6, 0xf, 0xe, 0xcb, 0x10, + 0xc4, 0xe0, 0xf4, 0x3d, 0xf4, 0xf5, 0xea, 0x9b, + 0xd1, 0x16, 0x1b, 0x62, 0x11, 0x3e, 0x20, 0x3a, + 0x68, 0xc8, 0xf0, 0xe, 0x55, 0xbe, 0x51, 0x4d, + 0xbe, 0x1f, 0x4f, 0xda, 0x84, 0xda, 0xc4, 0x9e, + 0x24, 0xd7, 0x46, 0x82, 0x56, 0x4e, 0x61, 0x63, + 0xda, 0x18, 0xea, 0xc6, 0xc3, 0x21, 0x89, 0x18, + 0xe, 0x87, 0xb7, 0x91, 0xfe, 0x8d, 0xe, 0xac, + 0x75, 0x58, 0xe5, 0x9f, 0x1f, 0x93, 0xa6, 0x49, + 0x24, 0xa2, 0xc6, 0xe8, 0x9d, 0x9c, 0x6d, 0xc1, + 0xf, 0xfc, 0xe3, 0x57, 0xd3, 0xc2, 0x10, 0x91, + 0x9a, 0xa8, 0xaa, 0xd7, 0xf, 0xaa, 0x75, 0x90, + 0x4a, 0x10, 0xef, 0xb6, 0xdd, 0x6c, 0xd5, 0x1a, + 0xe3, 0xbb, 0xe0, 0x64, 0x44, 0xc, 0x59, 0xa1, + 0xef, 0x3, 0x52, 0xac, 0xa4, 0x85, 0x3e, 0x40, + 0xee, 0x5c, 0xef, 0xcf, 0xb1, 0xaa, 0x88, 0xe5, + 0x56, 0xb8, 0xcd, 0x87, 0xc7, 0xc6, 0xd3, 0xb4, + 0x85, 0x8f, 0x2a, 0xc9, 0xcd, 0x8a, 0x8b, 0x25, + 0x12, 0x71, 0x76, 0xc9, 0xaa, 0x62, 0x75, 0x80, + 0x6e, 0xa3, 0xf9, 0xa5, 0xfc, 0x90, 0xac, 0x28, + 0x13, 0x82, 0xbb, 0x5d, 0xa6, 0x93, 0x47, 0xd4, + 0xf, 0x3b, 0x19, 0xf6, 0x81, 0xdb, 0x55, 0xb0, + 0x47, 0x75, 0x63, 0x93, 0xb4, 0xdd, 0xf0, 0xaf, + 0xb7, 0x44, 0xcb, 0x7, 0x7b, 0x35, 0xc5, 0xe4, + 0x45, 0xfe, 0xbb, 0x11, 0x1a, 0x90, 0x96, 0x3a, + 0x7, 0x2a, 0xef, 0x9c, 0xc, 0xae, 0x38, 0x26, + 0xef, 0xc2, 0xc3, 0x53, 0xfa, 0x54, 0xcf, 0x6f, + 0xf7, 0xa, 0xea, 0x19, 0xa8, 0xf, 0xbd, 0xa7, + 0x3f, 0xcd, 0x38, 0x2c, 0xf3, 0x97, 0xfb, 0xdb, + 0xcb, 0xc5, 0x83, 0x80, 0x91, 0x3d, 0xc7, 0x29, + 0x67, 0x16, 0xa5, 0xd1, 0x41, 0xd0, 0xa1, 0x9b, + 0xde, 0x13, 0x83, 0x12, 0x36, 0x75, 0x81, 0x71, + 0x6b, 0xbc, 0x72, 0xcb, 0x37, 0x4, 0x6, 0x7c, + 0x3a, 0x22, 0x2b, 0xa, 0x11, 0xd3, 0x33, 0x8f, + 0x3, 0x54, 0x8e, 0x79, 0xb6, 0x36, 0x93, 0x92, + 0xb8, 0xf, 0x24, 0x4a, 0xd3, 0xd5, 0x27, 0x66, + 0xd1, 0xde, 0xe3, 0xaa, 0x4b, 0x2a, 0xe9, 0x22, + 0x9b, 0xbf, 0x6e, 0x9a, 0xf7, 0xa, 0x2f, 0x24, + 0x13, 0xd5, 0xd5, 0xbb, 0xa3, 0xba, 0x8f, 0xfc, + 0x28, 0xa8, 0xbe, 0xe6, 0x9f, 0xea, 0xed, 0xb1, + 0xba, 0xaf, 0xf, 0x1c, 0x1e, 0x51, 0xf8, 0xd7, + 0x1b, 0xa5, 0xa6, 0x63, 0x40, 0x6e, 0x3f, 0xa2, + 0x57, 0x6f, 0x57, 0xe4, 0x27, 0xc2, 0x3c, 0x33, + 0xc6, 0x9c, 0x24, 0xd0, 0x53, 0xc4, 0xfc, 0xed, + 0x8e, 0x1d, 0xf, 0xc3, 0x86, 0x9, 0x3d, 0x1d, + 0xc2, 0xdb, 0x24, 0x1a, 0x65, 0xf4, 0x30, 0xa5, + 0xc, 0x48, 0x37, 0xc5, 0x53, 0x35, 0x3b, 0xab, + 0xd, 0x96, 0x30, 0xd7, 0x1d, 0x66, 0x18, 0xc2, + 0x47, 0x3a, 0xef, 0xbe, 0x2e, 0xe4, 0x54, 0x9d, + 0xc4, 0xa5, 0xb9, 0xb3, 0x4c, 0x12, 0x73, 0x35, + 0xf0, 0x7, 0xe, 0x36, 0x88, 0xb2, 0x4b, 0x29, + 0xb, 0x4e, 0x84, 0x11, 0xaa, 0x9a, 0x3e, 0xb1, + 0xd7, 0xec, 0xfb, 0x7f, 0x10, 0x70, 0x1f, 0x26, + 0xf0, 0x27, 0x46, 0x5d, 0x4, 0x51, 0x97, 0x29, + 0xb4, 0x66, 0x39, 0x1, 0x82, 0x47, 0xd8, 0x5f, + 0xa9, 0xb3, 0xa1, 0xb8, 0xde, 0x1, 0xe1, 0xc4, + 0x47, 0xc5, 0xe8, 0xe6, 0xbb, 0xc0, 0xb6, 0x41, + 0x55, 0x10, 0x79, 0xa8, 0xd0, 0xd, 0x1, 0x56, + 0x29, 0x6c, 0xa5, 0x96, 0x87, 0x59, 0x4b, 0xd, + 0xc8, 0x3, 0x5, 0xaa, 0xa9, 0x6a, 0xb1, 0x10, + 0xbc, 0x1, 0x68, 0xd3, 0xa5, 0x52, 0x41, 0xe1, + 0x1f, 0x53, 0x7, 0xc6, 0xad, 0xb8, 0xc4, 0xf0, + 0x28, 0xe9, 0x3, 0x3a, 0xee, 0xce, 0x2c, 0xe2, + 0xb0, 0xda, 0x78, 0x3d, 0x37, 0x7, 0x2d, 0x1f, + 0xf1, 0x47, 0x81, 0x4, 0x67, 0x6e, 0xd, 0xa1, + 0x2b, 0x4, 0xe8, 0xd9, 0xf4, 0xaf, 0x35, 0xca, + 0xa5, 0xd1, 0xe3, 0xec, 0xc5, 0x82, 0x50, 0x99, + 0x9a, 0xee, 0xea, 0x53, 0x41, 0x86, 0x97, 0x44, + 0xeb, 0x58, 0x43, 0x47, 0xe7, 0xa0, 0xd3, 0x28, + 0xfc, 0xe7, 0x13, 0x8b, 0x56, 0xe3, 0xdb, 0xa9, + 0xcd, 0x9, 0xc8, 0x7, 0x11, 0xeb, 0xbf, 0xac, + 0x76, 0x72, 0x60, 0xaf, 0x9c, 0xba, 0x8a, 0x64, + 0xfb, 0xf4, 0xab, 0x27, 0x29, 0xe7, 0xec, 0x69, + 0x21, 0xcb, 0x5b, 0x79, 0x56, 0x10, 0xc1, 0x8, + 0xd5, 0x5d, 0x93, 0xb1, 0x70, 0x88, 0xf2, 0x19, + 0x41, 0xc6, 0xc2, 0x84, 0xdd, 0xf0, 0xb3, 0x40, + 0x12, 0x71, 0x24, 0x54, 0xc4, 0x5e, 0xfb, 0x5f, + 0x47, 0x8c, 0xa9, 0x4, 0x5a, 0xd5, 0x61, 0x19, + 0xb5, 0x7f, 0xc9, 0xbd, 0x87, 0xb2, 0xcd, 0x57, + 0x99, 0x50, 0x67, 0x1d, 0xb0, 0x1d, 0x82, 0xdd, + 0xef, 0x32, 0x38, 0xb9, 0xc7, 0x86, 0xb4, 0xd2, + 0xd6, 0xe1, 0x33, 0xb2, 0xdb, 0x5e, 0xc2, 0xa3, + 0x49, 0xa6, 0x5f, 0x79, 0x32, 0x50, 0x41, 0x5b, + 0xd7, 0x87, 0x74, 0xf5, 0xc9, 0x9c, 0x78, 0xb7, + 0xb, 0x1f, 0x72, 0xba, 0xd9, 0x3a, 0x4d, 0x18, + 0x45, 0x1d, 0xad, 0xef, 0xc4, 0xdc, 0x30, 0xe8, + 0x2, 0xb1, 0x7f, 0x6c, 0x8f, 0xaa, 0xd0, 0x40, + 0x17, 0xe, 0x58, 0x93, 0x42, 0x49, 0x63, 0x77, + 0x48, 0x55, 0x90, 0x2f, 0x7c, 0x3b, 0xee, 0x3c, + 0xac, 0xd, 0xd8, 0x72, 0x23, 0xd7, 0xa5, 0x6e, + 0xb0, 0xd2, 0x91, 0x25, 0x60, 0x9a, 0x52, 0xab, + 0xbd, 0x63, 0xce, 0xba, 0xda, 0xb1, 0xd7, 0xc7, + 0x3d, 0x21, 0x4e, 0x9c, 0x5a, 0x1e, 0x8d, 0xf4, + 0xa, 0xdb, 0xd9, 0xf, 0x20, 0x7e, 0xfb, 0xbf, + 0x36, 0x9c, 0x4f, 0xbd, 0xf7, 0xdb, 0x5b, 0xa2, + 0x6, 0xb2, 0x0, 0xe2, 0xa2, 0x9e, 0x4e, 0x19, + 0xd4, 0x69, 0xa9, 0x51, 0x69, 0x8b, 0xf5, 0xe1, + 0xad, 0x89, 0x8, 0xc5, 0x4f, 0xac, 0x1b, 0x7d, + 0xe7, 0xa, 0x9, 0x7d, 0x34, 0xf5, 0x3f, 0x46, + 0x80, 0xb9, 0xb9, 0x45, 0x58, 0xcd, 0x6c, 0xb5, + 0x5f, 0x60, 0xeb, 0x5a, 0xe3, 0xa3, 0x8, 0x5e, + 0xb1, 0xc4, 0x73, 0xc5, 0xa5, 0x67, 0x56, 0xd3, + 0xc6, 0x8a, 0x55, 0x6b, 0xd7, 0xd7, 0xc, 0x20, + 0xe6, 0xc, 0x73, 0x8, 0x2, 0x4b, 0xfb, 0xdd, + 0x4d, 0x4e, 0xa8, 0xb8, 0xd8, 0x4b, 0x53, 0x2f, + 0xc2, 0xfb, 0x5d, 0xa1, 0x6a, 0x16, 0x6b, 0xe, + 0xf1, 0xa1, 0xa5, 0x5b, 0xdf, 0x9c, 0x23, 0xb5, + 0x94, 0x9c, 0xae, 0x7b, 0xbe, 0x42, 0xb5, 0x79, + 0x80, 0xc3, 0x43, 0x41, 0xa4, 0x1b, 0x18, 0xfc, + 0x52, 0xcf, 0x43, 0xc5, 0x80, 0x7b, 0xbd, 0xc1, + 0x20, 0x5e, 0x65, 0xec, 0xc5, 0xfc, 0x3, 0xec, + 0x8f, 0x61, 0x66, 0xf5, 0x15, 0x67, 0xc8, 0xb6, + 0xef, 0x9a, 0xba, 0xb7, 0xcb, 0x2c, 0xac, 0x1b, + 0x50, 0xda, 0xb6, 0x29, 0xa4, 0x37, 0xe9, 0x96, + 0xa0, 0x7, 0x7d, 0x49, 0xa6, 0xce, 0xf3, 0xf0, + 0x19, 0xdf, 0x61, 0xc7, 0xa4, 0x7b, 0x5a, 0xd4, + 0x99, 0xb2, 0x64, 0xe7, 0xd1, 0x6b, 0x7f, 0xe8, + 0xb8, 0xd3, 0x89, 0xee, 0x96, 0xc0, 0xed, 0x5d, + 0x7e, 0x48, 0x2, 0xd2, 0x25, 0xd0, 0x5, 0xef, + 0x93, 0x72, 0x7c, 0x8c, 0xbd, 0x6e, 0x49, 0xd3, + 0x38, 0x46, 0x1c, 0xff, 0x28, 0x4e, 0x1b, 0xad, + 0x39, 0x2f, 0x65, 0x26, 0xe2, 0x70, 0x3d, 0xb8, + 0x7a, 0xd3, 0x38, 0x38, 0xfc, 0x3a, 0x67, 0x78, + 0xdb, 0x9, 0xcb, 0xbf, 0xc9, 0xe1, 0xee, 0x69, + 0x2b, 0xd, 0xb1, 0x79, 0x13, 0xd0, 0xa5, 0x75, + 0x6, 0x8, 0x79, 0xa7, 0x7c, 0xc, 0xe7, 0x1b, + 0x9c, 0x36, 0x64, 0xbe, 0x20, 0x65, 0xa2, 0xd4, + 0xd9, 0xc, 0x68, 0xe, 0x88, 0x2b, 0x93, 0x60, + 0xf1, 0xa5, 0x82, 0xc5, 0x4d, 0x2b, 0x7d, 0x73, + 0xe9, 0x13, 0x8c, 0xc1, 0x8, 0xbd, 0x21, 0x65, + 0x77, 0x2f, 0x34, 0xb1, 0x97, 0x9f, 0xd8, 0x55, + 0xcf, 0x75, 0xc2, 0xf2, 0x41, 0x68, 0xc1, 0x9c, + 0x1c, 0xd7, 0x23, 0xbf, 0x83, 0x2a, 0x9, 0x66, + 0xce, 0x8f, 0xd2, 0x12, 0x79, 0x93, 0xef, 0x8, + 0x9b, 0xeb, 0x2f, 0xc, 0xe4, 0x5b, 0x71, 0x1a, + 0xef, 0x11, 0x65, 0xd8, 0x6d, 0x8c, 0x59, 0x53, + 0x70, 0x1d, 0xb5, 0x81, 0xff, 0xc0, 0x7d, 0x87, + 0xa5, 0x21, 0x5d, 0x9f, 0x63, 0xb2, 0xe7, 0xe9, + 0xd0, 0x49, 0x41, 0xc7, 0x3c, 0xe1, 0x2b, 0xb1, + 0xac, 0x15, 0xcd, 0xb0, 0xa8, 0xdc, 0xae, 0x3b, + 0xef, 0x32, 0x98, 0x8c, 0xc7, 0x40, 0xa6, 0x81, + 0x1, 0xa1, 0x7d, 0x89, 0x46, 0x99, 0x91, 0x24, + 0xce, 0xb2, 0x70, 0x82, 0x92, 0xf3, 0x60, 0x66, + 0x34, 0x6, 0x37, 0xad, 0x5c, 0xed, 0xc3, 0x27, + 0x68, 0x8c, 0x56, 0xe7, 0xf, 0x73, 0x5c, 0x7e, + 0x9e, 0xd0, 0x8c, 0x99, 0x5a, 0xb1, 0x15, 0x98, + 0xbb, 0x79, 0x9f, 0xd1, 0x69, 0xce, 0x76, 0x5, + 0xcb, 0x8e, 0x18, 0xb3, 0x84, 0x65, 0xa9, 0x2, + 0xbc, 0x43, 0x8b, 0x7e, 0xe9, 0xe2, 0xe6, 0x74, + 0x31, 0x8d, 0xe7, 0xa2, 0x42, 0x8f, 0xca, 0x38, + 0x59, 0x85, 0x25, 0x47, 0xd2, 0x86, 0x47, 0x9, + 0xc2, 0x11, 0x2, 0x91, 0xe6, 0xf3, 0x47, 0xc2, + 0x9c, 0x28, 0x2f, 0xbb, 0xac, 0xde, 0x9f, 0xd, + 0xc2, 0x96, 0x4f, 0x43, 0xca, 0x32, 0xed, 0x34, + 0xba, 0xad, 0xef, 0xbe, 0x68, 0xc7, 0xa2, 0x83, + 0xaf, 0xe, 0xd3, 0x72, 0x52, 0xd1, 0x76, 0x3d, + 0x9a, 0x98, 0x39, 0xf4, 0x3e, 0x14, 0x27, 0xff, + 0xb2, 0x37, 0x23, 0xc5, 0x6d, 0x66, 0xef, 0xaa, + 0xfe, 0xe7, 0xe4, 0x86, 0xa1, 0xe, 0x4e, 0x36, + 0x64, 0xb1, 0x67, 0xf, 0x94, 0x6f, 0x77, 0xd5, + 0xec, 0xe2, 0x5e, 0xc8, 0xe3, 0x64, 0x29, 0x92, + 0xd, 0x20, 0x34, 0x9f, 0x19, 0x6e, 0x85, 0xf8, + 0x48, 0x78, 0xb0, 0xf, 0x42, 0xb2, 0x8c, 0xea, + 0xc2, 0x4d, 0xd3, 0x23, 0xb, 0x4d, 0x20, 0x33, + 0xc7, 0x46, 0x0, 0x45, 0x37, 0xc6, 0xcb, 0xd0, + 0xec, 0x11, 0xc6, 0x74, 0x91, 0x7d, 0x6b, 0x54, + 0x56, 0x10, 0x8d, 0xd0, 0xce, 0xe8, 0x57, 0x3b, + 0x83, 0xd8, 0x25, 0x51, 0x79, 0x48, 0xa, 0xa5, + 0xc3, 0xe4, 0x65, 0x33, 0xb2, 0x89, 0xa6, 0x4c, + 0xe8, 0xc8, 0x9e, 0xce, 0xea, 0x2a, 0x55, 0x40, + 0xfc, 0x26, 0x29, 0xd4, 0x2d, 0x7e, 0xe1, 0xb1, + 0x4d, 0x65, 0x1, 0xe9, 0x98, 0xc9, 0xf4, 0x69, + 0x10, 0xd9, 0xa3, 0xf9, 0x34, 0xaf, 0x3c, 0x34, + 0x64, 0x23, 0xde, 0xb8, 0x1c, 0x33, 0x18, 0x74, + 0x67, 0xb4, 0x4a, 0x71, 0xa6, 0x89, 0x2, 0xfe, + 0xf7, 0xf1, 0x32, 0xc7, 0x98, 0xad, 0xe5, 0x10, + 0x98, 0x3c, 0x6c, 0xaf, 0x1f, 0x13, 0x3d, 0xcc, + 0xfc, 0x3b, 0x67, 0x33, 0x34, 0xc9, 0x31, 0xcd, + 0x3f, 0xd, 0x3c, 0x5a, 0xb6, 0xc2, 0x8, 0xea, + 0xe2, 0xae, 0xdd, 0xfc, 0x6f, 0xca, 0xb5, 0x67, + 0x11, 0xce, 0xd5, 0xda, 0x3a, 0x8b, 0x7, 0xf2, + 0xc0, 0x9e, 0x78, 0x18, 0x92, 0x9f, 0x64, 0x26, + 0x9f, 0x66, 0x62, 0x66, 0xa1, 0x7e, 0x3, 0xf5, + 0xb9, 0xe6, 0x74, 0x20, 0x88, 0xb7, 0x7e, 0x62, + 0x7a, 0x33, 0x21, 0x9, 0x9c, 0x91, 0x3b, 0x62, + 0x9, 0x46, 0xd3, 0xd1, 0x1f, 0xc5, 0x3a, 0x8f, + 0x69, 0x27, 0x2c, 0x7b, 0xec, 0xda, 0x79, 0xf1, + 0xc9, 0xe9, 0x98, 0xd0, 0xa, 0xc9, 0xf6, 0x37, + 0x28, 0xf8, 0xfc, 0xe, 0xdc, 0xf, 0xe9, 0x23, + 0xf6, 0x84, 0x25, 0x96, 0x2c, 0x24, 0x14, 0xd7, + 0xe2, 0x5e, 0x1c, 0x56, 0x7f, 0x99, 0x98, 0x62, + 0x76, 0xcc, 0x84, 0x44, 0xd6, 0xb9, 0x47, 0x2b, + 0x52, 0xfb, 0x42, 0x40, 0xf3, 0x63, 0xaf, 0xd4, + 0x10, 0x5, 0xf9, 0x3b, 0xc8, 0x53, 0xa9, 0x45, + 0xa4, 0x50, 0x41, 0x83, 0xe8, 0x4a, 0x9, 0xb6, + 0xf1, 0x77, 0x70, 0xe3, 0x61, 0x30, 0xd8, 0x90, + 0x49, 0x52, 0x4b, 0x4a, 0xf2, 0x66, 0x84, 0xaf, + 0x71, 0x1, 0x40, 0x66, 0xf6, 0x3, 0xc9, 0x23, + 0xb1, 0x1a, 0xc1, 0xb2, 0xf7, 0x35, 0x1a, 0xc9, + 0x3a, 0x75, 0xb1, 0xa7, 0x4, 0xff, 0x69, 0xa, + 0x90, 0x58, 0xd4, 0xf4, 0x16, 0x79, 0xe1, 0xae, + 0x39, 0x9d, 0xbb, 0x32, 0x6b, 0x3, 0xe2, 0xf5, + 0x73, 0x83, 0x7e, 0x3c, 0xf8, 0x29, 0xab, 0xcc, + 0xdc, 0xf0, 0x13, 0xdb, 0x86, 0x28, 0x88, 0x8e, + 0xde, 0x6a, 0x29, 0xf1, 0xea, 0x0, 0x83, 0x97, + 0x1, 0x32, 0x5f, 0xaa, 0x5b, 0x1b, 0xe4, 0x87, + 0xec, 0x90, 0x45, 0xc7, 0xc5, 0x6c, 0x11, 0x83, + 0x95, 0xab, 0xdd, 0x71, 0x69, 0x24, 0xc, 0x5c, + 0xc0, 0xf3, 0xc1, 0xb0, 0x5e, 0x1, 0x5e, 0x4, + 0xa1, 0x6e, 0x6e, 0x7d, 0x3f, 0x6f, 0xbd, 0x5d, + 0x9, 0x8f, 0x23, 0x53, 0x74, 0x4b, 0xa9, 0x53, + 0xd2, 0x10, 0xa1, 0xc0, 0x8e, 0x18, 0xa, 0x2f, + 0x88, 0x8d, 0x4b, 0xf8, 0xc2, 0x3d, 0xeb, 0x34, + 0x23, 0xa, 0x80, 0xc, 0x69, 0x21, 0x3, 0xc1, + 0x6f, 0xbe, 0xdf, 0xf6, 0x2c, 0x27, 0x77, 0xa2, + 0xc5, 0x5c, 0x9, 0x54, 0x5d, 0x4a, 0x4c, 0xb, + 0x6b, 0xb5, 0x88, 0x11, 0x42, 0x62, 0x39, 0x89, + 0x9e, 0x36, 0xd3, 0x91, 0xf6, 0x70, 0x18, 0x35, + 0x79, 0xaf, 0x73, 0xf3, 0x0, 0x75, 0x5a, 0xa3, + 0xce, 0xf1, 0x42, 0x80, 0x19, 0x5e, 0x42, 0x56, + 0x53, 0x85, 0xbb, 0xf4, 0x29, 0xac, 0x84, 0x1d, + 0x97, 0x1, 0x1c, 0xc4, 0x58, 0xcb, 0x33, 0xc4, + 0xdc, 0x1e, 0x59, 0x8f, 0x48, 0xa9, 0x59, 0xfd, + 0xaf, 0xa3, 0x5c, 0x19, 0x17, 0x6b, 0x46, 0x2d, + 0xab, 0x44, 0xa3, 0xcc, 0x1a, 0xaa, 0x23, 0x4e, + 0x58, 0x37, 0x7b, 0x11, 0x14, 0xc2, 0xf1, 0xc9, + 0x58, 0x99, 0xd3, 0x3c, 0xec, 0xb9, 0xbe, 0x17, + 0x3c, 0x8d, 0x1c, 0x87, 0x9d, 0xe1, 0xb9, 0xad, + 0x68, 0x36, 0xd5, 0xfc, 0x24, 0x9b, 0x34, 0x5, + 0x26, 0xac, 0x15, 0x9f, 0xd6, 0x70, 0x74, 0x6c, + 0x72, 0xf, 0x6, 0x6, 0x5a, 0xc, 0xc0, 0x78, + 0x47, 0x8e, 0xcf, 0xf2, 0xce, 0x8, 0xe2, 0xa4, + 0xc6, 0x7d, 0x2d, 0x70, 0x14, 0xe2, 0xc6, 0xfc, + 0x63, 0x7a, 0x42, 0x8c, 0x45, 0xae, 0xe8, 0x3b, + 0x30, 0x48, 0xda, 0x3e, 0x14, 0xb5, 0x8b, 0x10, + 0xae, 0x56, 0xbd, 0x17, 0xdf, 0xcb, 0x63, 0xf5, + 0xb, 0x2b, 0xd7, 0x34, 0x7c, 0x96, 0x43, 0xe9, + 0x17, 0xd4, 0x53, 0x2b, 0x4e, 0xba, 0x61, 0x57, + 0x92, 0xdb, 0xe8, 0x37, 0xf4, 0xa3, 0x59, 0x88, + 0x74, 0xc2, 0x3c, 0x5d, 0x54, 0x30, 0xb9, 0x6, + 0xbe, 0x75, 0x13, 0xe8, 0xf2, 0xe8, 0xcb, 0x45, + 0x73, 0x70, 0xaf, 0x94, 0xe6, 0xc5, 0xb0, 0xdf, + 0xd2, 0xd5, 0x57, 0x97, 0x7c, 0x97, 0xde, 0x55, + 0xaf, 0xbb, 0xed, 0x19, 0x35, 0x17, 0xf4, 0x23, + 0x38, 0x9c, 0xce, 0x37, 0xfe, 0xd8, 0x4e, 0xd8, + 0x99, 0xba, 0x33, 0x22, 0xf2, 0xeb, 0xab, 0x97, + 0xee, 0x9d, 0xab, 0x67, 0x95, 0x35, 0xdf, 0xc8, + 0xb6, 0xa0, 0xf, 0x15, 0x51, 0xa9, 0x76, 0x15, + 0xdd, 0xbd, 0xac, 0x12, 0xce, 0x51, 0xde, 0x68, + 0x15, 0xaf, 0x27, 0xcf, 0xd1, 0xba, 0x7c, 0x17, + 0xef, 0xbf, 0xbb, 0xc0, 0x6e, 0x58, 0x73, 0xf6, + 0x57, 0xe1, 0x8d, 0xb0, 0x9a, 0x5a, 0x9, 0x19, + 0xef, 0xdd, 0x4, 0xe1, 0x76, 0x94, 0x31, 0xd7, + 0x26, 0x9f, 0x9c, 0x27, 0xc4, 0x2b, 0x4b, 0xf6, + 0x3b, 0xa1, 0x8c, 0xf4, 0x21, 0xde, 0x39, 0x14, + 0x5a, 0x54, 0xac, 0x95, 0x2f, 0xa0, 0x60, 0x53, + 0x87, 0x5b, 0x71, 0x92, 0xae, 0xf9, 0x6c, 0x62, + 0x76, 0x7e, 0x91, 0x11, 0xa6, 0xf4, 0xf2, 0xa8, + 0xdf, 0xc1, 0xf6, 0x3a, 0xdb, 0x34, 0x96, 0x9, + 0x71, 0xb4, 0x4, 0xfa, 0xd4, 0x3, 0x46, 0x16, + 0x78, 0x41, 0x42, 0x7d, 0x15, 0x68, 0x63, 0x55, + 0x23, 0x4, 0x46, 0x5d, 0xe1, 0xd8, 0xe7, 0x5f, + 0x55, 0x39, 0xd2, 0x45, 0xb2, 0x0, 0x35, 0xde, + 0xd8, 0x9d, 0xc7, 0x3a, 0x8f, 0x37, 0x7e, 0xe5, + 0x9e, 0xcf, 0xd1, 0x6a, 0x22, 0xe1, 0x51, 0xb2, + 0xe6, 0x99, 0x3e, 0x83, 0xeb, 0x34, 0x9d, 0x34, + 0x7, 0x1c, 0xbe, 0x91, 0x69, 0x9e, 0xaa, 0xcb, + 0x86, 0xd2, 0xb6, 0xed, 0xa5, 0x4, 0xf9, 0x7d, + 0xf8, 0xba, 0x2a, 0x27, 0x38, 0xe1, 0xaa, 0x22, + 0x94, 0x46, 0x1f, 0x1b, 0xcf, 0xc4, 0x78, 0x88, + 0x3d, 0x50, 0x83, 0x30, 0x61, 0x87, 0xb6, 0x38, + 0x5b, 0x4f, 0x5a, 0x3, 0x2d, 0x5d, 0xa6, 0x33, + 0x38, 0xe7, 0x8b, 0x60, 0x1, 0x8e, 0xde, 0x69, + 0x8e, 0x4d, 0x60, 0x24, 0x3b, 0x47, 0x4b, 0x56, + 0xea, 0xf9, 0xc8, 0xfa, 0x2d, 0x65, 0x7b, 0xad, + 0xee, 0xe4, 0x91, 0x20, 0x6f, 0x64, 0x6e, 0x81, + 0x69, 0xda, 0xf5, 0x3c, 0x3d, 0xff, 0x4c, 0xe9, + 0x9b, 0x4d, 0xa8, 0x67, 0x9e, 0x67, 0x7f, 0x84, + 0xdb, 0x7a, 0xb7, 0x24, 0x32, 0xa0, 0x80, 0x16, + 0x55, 0x2d, 0x1d, 0xc1, 0x3a, 0x19, 0xd3, 0x17, + 0x74, 0x8e, 0x2a, 0x5c, 0xf6, 0x71, 0xf7, 0x25, + 0x3a, 0x54, 0x28, 0xef, 0x50, 0x78, 0x14, 0x5, + 0x49, 0x8a, 0xbb, 0x71, 0xb2, 0xed, 0xa2, 0x5b, + 0xff, 0x2, 0xe, 0xd8, 0x1a, 0x8b, 0x3c, 0xcc, + 0x58, 0x27, 0x71, 0x2d, 0xb, 0x11, 0x9f, 0x6, + 0xc3, 0xfd, 0x37, 0x19, 0xdb, 0xec, 0xa5, 0x4b, + 0x93, 0x81, 0xb6, 0xff, 0xd4, 0xf5, 0x7b, 0xf5, + 0x49, 0x5b, 0x95, 0x9, 0xa4, 0xca, 0xa5, 0x33, + 0x9a, 0xfc, 0x97, 0xec, 0x7b, 0xb, 0xb9, 0x2e, + 0x3b, 0x9d, 0x52, 0xc2, 0xa2, 0x9, 0xc8, 0xbf, + 0x39, 0x16, 0xce, 0x42, 0x3, 0x4b, 0xe3, 0xfc, + 0xfd, 0xc, 0x37, 0x96, 0x10, 0x36, 0xad, 0x44, + 0xda, 0xc5, 0x58, 0x3e, 0x78, 0x52, 0xa1, 0x65, + 0xed, 0x89, 0xe7, 0xea, 0xbf, 0xa8, 0x6a, 0xf2, + 0xa7, 0x8e, 0x9d, 0x1, 0x25, 0x83, 0x57, 0x5f, + 0x51, 0xe6, 0xe1, 0xa4, 0x4f, 0xf6, 0x81, 0xd7, + 0xe6, 0x98, 0x29, 0x98, 0x58, 0xfe, 0xda, 0x45, + 0xab, 0x38, 0x6, 0x91, 0x97, 0xb7, 0xa3, 0x4f, + 0x93, 0x8d, 0x8a, 0x8b, 0x5, 0xe9, 0x5, 0x98, + 0x3b, 0xc4, 0xb7, 0xe1, 0x68, 0x58, 0xa0, 0x3b, + 0x99, 0xea, 0x8a, 0xa9, 0xfb, 0x55, 0xe2, 0xc7, + 0x1d, 0x87, 0x3, 0x40, 0x24, 0x13, 0x28, 0x6a, + 0x34, 0x8a, 0xff, 0x62, 0x91, 0xb8, 0x7d, 0x28, + 0x1a, 0xd2, 0xfc, 0x4e, 0xa3, 0xda, 0x66, 0x69, + 0x15, 0xc0, 0xda, 0x15, 0x3e, 0x67, 0x12, 0x95, + 0x6, 0x1b, 0xf4, 0x60, 0xe4, 0x39, 0x82, 0xe9, + 0x2e, 0xbe, 0xab, 0x8c, 0x2c, 0x6e, 0xd6, 0x40, + 0x91, 0xc0, 0x68, 0xf7, 0xa2, 0x41, 0xd0, 0xa8, + 0x7, 0xab, 0x13, 0x34, 0x16, 0xf4, 0x73, 0x4f, + 0x1d, 0x21, 0x1a, 0x7d, 0xad, 0x43, 0x12, 0xf, + 0xb7, 0xfe, 0xa3, 0x81, 0xe9, 0xb5, 0x2d, 0xd3, + 0xa, 0x29, 0xb5, 0x32, 0xcb, 0x49, 0x6f, 0x1, + 0x90, 0x45, 0x62, 0xca, 0x1b, 0x66, 0x39, 0x88, + 0x1c, 0xee, 0x30, 0xa8, 0xb5, 0x37, 0xd0, 0xfa, + 0x46, 0x52, 0x16, 0x30, 0x17, 0xcf, 0x88, 0xd0, + 0x4, 0x5d, 0xde, 0x5e, 0x4f, 0xe7, 0xa9, 0xbf, + 0x3c, 0x29, 0x3a, 0x63, 0x67, 0x23, 0xb3, 0x7c, + 0x51, 0x17, 0xfe, 0x8d, 0xdb, 0xc8, 0x8d, 0x70, + 0xe9, 0x6f, 0x56, 0xe5, 0x44, 0xb2, 0x94, 0xeb, + 0x47, 0xca, 0x3a, 0xdc, 0xe3, 0x33, 0x87, 0x9c, + 0xe8, 0x89, 0x4b, 0x41, 0xb8, 0xb3, 0x69, 0xb0, + 0x7f, 0xc8, 0xc7, 0x74, 0xf5, 0xcb, 0x20, 0xad, + 0xea, 0xbb, 0x3d, 0x11, 0xc6, 0xc0, 0xd2, 0x88, + 0x8b, 0x16, 0xee, 0x62, 0x5a, 0x4d, 0x32, 0xe7, + 0x48, 0xae, 0xab, 0x5e, 0xc2, 0x83, 0xc4, 0xfc, + 0xd1, 0xb9, 0x71, 0xf2, 0x9, 0x7f, 0xdc, 0xbc, + 0x28, 0x74, 0xa0, 0x37, 0xa9, 0x5b, 0x6c, 0x7c, + 0x9b, 0x61, 0x94, 0x88, 0xf7, 0x40, 0x84, 0x75, + 0xa5, 0x50, 0xab, 0xb0, 0x92, 0x66, 0x10, 0x66, + 0xf6, 0xec, 0x6b, 0x5e, 0x31, 0x9b, 0xc4, 0xfa, + 0x95, 0x8b, 0xe7, 0xd4, 0xba, 0x81, 0xd2, 0x85, + 0x30, 0x4, 0x8b, 0x3d, 0xfa, 0x8a, 0x8f, 0x9b, + 0x54, 0x6a, 0x4d, 0x35, 0xa2, 0xe9, 0x58, 0x95, + 0xe3, 0xd1, 0x71, 0xcd, 0x3a, 0x54, 0xae, 0xd9, + 0x5c, 0x83, 0xd, 0x15, 0x64, 0x66, 0xee, 0x39, + 0xa1, 0x85, 0xe2, 0x28, 0xf5, 0x66, 0x5f, 0xec, + 0x39, 0x70, 0x96, 0x2c, 0x72, 0x9e, 0x57, 0xfd, + 0x57, 0x27, 0xb7, 0xda, 0x79, 0x39, 0xd8, 0x3b, + 0x2e, 0xa3, 0xb0, 0xde, 0xbf, 0x60, 0xb6, 0x42, + 0x78, 0x9d, 0x8f, 0xe8, 0x1c, 0x7c, 0x45, 0x72, + 0x3, 0xc4, 0xd5, 0x81, 0xf6, 0xe6, 0x9, 0x29, + 0x1e, 0xcd, 0xf3, 0xe, 0xd6, 0x65, 0xee, 0x6d, + 0x90, 0x17, 0x95, 0x20, 0x54, 0xf1, 0xd, 0x2f, + 0xa0, 0xac, 0xe3, 0x4b, 0xfc, 0xa4, 0xdc, 0xab, + 0x9d, 0x9e, 0x32, 0x63, 0x72, 0xd1, 0xb4, 0xef, + 0xf1, 0x83, 0xa7, 0xd7, 0x2b, 0x1a, 0x9a, 0x9e, + 0xfa, 0x1e, 0xb, 0x2b, 0xdc, 0x7b, 0x87, 0x96, + 0xf, 0xdb, 0x75, 0xb9, 0x6, 0x2b, 0xd3, 0x95, + 0xc5, 0xb3, 0x9, 0x53, 0x94, 0x54, 0x1f, 0xd0, + 0x75, 0x5a, 0x36, 0x6a, 0x7c, 0x82, 0xdb, 0xb1, + 0xa2, 0x17, 0xbc, 0xeb, 0x1f, 0xfa, 0x34, 0x3d, + 0xee, 0x68, 0xee, 0x93, 0x33, 0xfb, 0xcb, 0xd2, + 0xa3, 0xd1, 0x24, 0x5e, 0xf4, 0x9, 0xbe, 0x5a, + 0x68, 0x9e, 0x3e, 0xd4, 0x81, 0xcd, 0xa3, 0x1e, + 0x2, 0x13, 0xb4, 0x79, 0x94, 0xc9, 0xb2, 0xde, + 0x56, 0xf1, 0x7b, 0x2f, 0xe2, 0x56, 0xe1, 0x10, + 0xf4, 0x73, 0x2d, 0xc9, 0xca, 0x4d, 0x5f, 0x11, + 0x9e, 0xd6, 0x3c, 0x73, 0x12, 0x57, 0xe9, 0x14, + 0xe0, 0x8d, 0xdd, 0x4b, 0x8a, 0xbb, 0xb3, 0x78, + 0xbe, 0x16, 0x94, 0x93, 0x51, 0x33, 0x7a, 0xa5, + 0x41, 0x14, 0x60, 0x82, 0x94, 0x67, 0x70, 0xea, + 0xe6, 0x3, 0x7f, 0xc5, 0xa0, 0x20, 0x15, 0x88, + 0x53, 0xe3, 0x7e, 0x16, 0x52, 0xe4, 0xca, 0xa0, + 0x6f, 0xb9, 0x68, 0x4e, 0x30, 0xb9, 0x8c, 0xe6, + 0x9c, 0x5e, 0xc2, 0x93, 0xf9, 0xe1, 0x41, 0x4b, + 0x18, 0x42, 0x6f, 0x8f, 0x96, 0x3d, 0x2b, 0x28, + 0xd5, 0x53, 0x62, 0xdd, 0x6b, 0xd0, 0xf8, 0x2e, + 0xa6, 0x97, 0xe5, 0x87, 0xc5, 0xf6, 0x96, 0x7b, + 0xc4, 0x3e, 0x84, 0xc9, 0xf6, 0x34, 0x63, 0x46, + 0xe1, 0x10, 0xa5, 0x91, 0x6b, 0xff, 0x10, 0x3f, + 0x50, 0x2e, 0xd7, 0x39, 0x12, 0x7a, 0x15, 0x85, + 0xed, 0x99, 0xdb, 0x9b, 0x99, 0x6b, 0xfa, 0xfa, + 0x93, 0x7, 0x44, 0xbe, 0xbe, 0x60, 0x23, 0xc1, + 0xec, 0x5c, 0xf6, 0x93, 0x38, 0xf9, 0x89, 0x0, + 0xc5, 0x5f, 0x5b, 0xe2, 0x9d, 0x2b, 0xea, 0x6b, + 0x2e, 0xee, 0xb7, 0x4a, 0x4e, 0x8d, 0xd0, 0x35, + 0xe9, 0xc1, 0x5, 0x2b, 0x83, 0xb7, 0x72, 0x25, + 0xbb, 0xbe, 0xe8, 0x15, 0xf4, 0x74, 0x69, 0x69, + 0x67, 0x8c, 0x5c, 0x31, 0x79, 0x78, 0x2e, 0x43, + 0x83, 0xd1, 0xdd, 0x9, 0xc3, 0xa1, 0x0, 0x13, + 0x31, 0x4b, 0x86, 0xce, 0xee, 0xd7, 0xec, 0xb1, + 0x2c, 0x38, 0x46, 0x68, 0x62, 0xd9, 0x84, 0xdb, + 0x24, 0x62, 0x82, 0xc, 0x12, 0xb7, 0x4f, 0x86, + 0x54, 0x18, 0xc6, 0xd7, 0x94, 0x8b, 0xf2, 0x4c, + 0x17, 0x98, 0xaa, 0xe0, +}; + +const uint8_t expected_cipher_long_input_end[] = { + 0x05, 0x95, 0x58, 0x7b, 0xb4, 0x60, 0x15, + 0x32, 0x9f, 0x38, 0xcc, 0x98, 0x1b, 0xbe, 0x10, 0xa5, 0x06, 0x67, 0xae, 0x38, + 0xbd, 0x7d, 0xb5, 0xcd, 0x58, 0x32, 0xdd, 0x9e, + 0x6a, 0xde, 0xe3, 0x53, + }; + +void aes_icache_ctr_test(uint32_t output_buf_caps) +{ + mbedtls_aes_context ctx; + uint8_t nonce[16]; + uint8_t key[16]; + uint8_t stream_block[16]; + size_t SZ = sizeof(long_input); + memset(nonce, 0x2F, 16); + memset(key, 0x1E, 16); + + // allocate internal memory + uint8_t *chipertext = heap_caps_malloc(SZ, output_buf_caps); + uint8_t *decryptedtext = heap_caps_malloc(SZ, MALLOC_CAP_8BIT | MALLOC_CAP_DMA); + + TEST_ASSERT_NOT_NULL(chipertext); + TEST_ASSERT_NOT_NULL(decryptedtext); + + mbedtls_aes_init(&ctx); + mbedtls_aes_setkey_enc(&ctx, key, 128); + + size_t offset; + + // Encrypt with input buffer in external ram + offset = 0; + memset(nonce, 0x2F, 16); + mbedtls_aes_crypt_ctr(&ctx, SZ, &offset, nonce, stream_block, long_input, chipertext); + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_cipher_long_input_end, chipertext + SZ - 32, 32); + + // Decrypt + offset = 0; + memset(nonce, 0x2F, 16); + // Decrypt with input buffer in instruction memory, the crypto DMA can't access this + mbedtls_aes_crypt_ctr(&ctx, SZ, &offset, nonce, stream_block, chipertext, decryptedtext); + + TEST_ASSERT_EQUAL_HEX8_ARRAY(long_input, decryptedtext, SZ); + + free(chipertext); + free(decryptedtext); +} + +/* Tests how crypto DMA handles data in external memory */ +TEST_CASE("mbedtls AES PSRAM tests", "[aes]") +{ + aes_psram_ctr_test(MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM); + aes_psram_ctr_test(MALLOC_CAP_SPIRAM, MALLOC_CAP_INTERNAL); + aes_psram_ctr_test(MALLOC_CAP_SPIRAM, MALLOC_CAP_SPIRAM); +} + +/* Tests how crypto DMA handles data from iCache */ +TEST_CASE("mbedtls AES iCache tests", "[aes]") +{ + aes_icache_ctr_test(MALLOC_CAP_SPIRAM); + aes_icache_ctr_test(MALLOC_CAP_INTERNAL); +} +#endif // CONFIG_SPIRAM_USE_MALLOC + +TEST_CASE("mbedtls AES GCM self-tests", "[aes]") +{ + TEST_ASSERT_FALSE_MESSAGE(mbedtls_gcm_self_test(1), "AES GCM self-test should pass."); +} + diff --git a/components/mbedtls/test/test_aes_perf.c b/components/mbedtls/test/test_aes_perf.c index 1447e4b2b..ec6d9214d 100644 --- a/components/mbedtls/test/test_aes_perf.c +++ b/components/mbedtls/test/test_aes_perf.c @@ -5,6 +5,7 @@ #include #include #include "mbedtls/aes.h" +#include "mbedtls/gcm.h" #include "unity.h" #include "sdkconfig.h" #include "esp_timer.h" @@ -14,7 +15,7 @@ TEST_CASE("mbedtls AES performance", "[aes]") { const unsigned CALLS = 256; - const unsigned CALL_SZ = 32*1024; + const unsigned CALL_SZ = 32 * 1024; mbedtls_aes_context ctx; int64_t start, end; uint8_t iv[16]; @@ -24,7 +25,7 @@ TEST_CASE("mbedtls AES performance", "[aes]") memset(key, 0x44, 16); // allocate internal memory - uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL); TEST_ASSERT_NOT_NULL(buf); mbedtls_aes_init(&ctx); mbedtls_aes_setkey_enc(&ctx, key, 128); @@ -63,9 +64,65 @@ TEST_CASE("mbedtls AES performance", "[aes]") float mb_sec = (CALL_SZ * CALLS) / usecs; printf("Encryption rate %.3fMB/sec\n", mb_sec); #ifdef CONFIG_MBEDTLS_HARDWARE_AES - // Don't put a hard limit on software AES performance (software is approx 2.3MB/sec on Release config) + // Don't put a hard limit on software AES performance TEST_PERFORMANCE_GREATER_THAN(AES_CBC_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif } +TEST_CASE("mbedtls AES GCM performance", "[aes]") +{ + const unsigned CALLS = 1; + const unsigned CALL_SZ = 32 * 1024; + mbedtls_gcm_context ctx; + int64_t start, end; + unsigned char tag_buf[16]; + mbedtls_cipher_id_t cipher = MBEDTLS_CIPHER_ID_AES; + uint8_t iv[16]; + uint8_t key[16]; + + memset(iv, 0xEE, 16); + memset(key, 0x44, 16); + + // allocate internal memory + uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL); + TEST_ASSERT_NOT_NULL(buf); + mbedtls_gcm_init(&ctx); + mbedtls_gcm_setkey( &ctx, cipher, key, 128); + + start = esp_timer_get_time(); + for (int c = 0; c < CALLS; c++) { + memset(buf, 0xAA, CALL_SZ); + mbedtls_gcm_crypt_and_tag(&ctx, MBEDTLS_AES_ENCRYPT, CALL_SZ, iv, sizeof(iv), NULL, 0, buf, buf, 16, tag_buf); + } + end = esp_timer_get_time(); + + /* Sanity check: make sure the last ciphertext block matches + what we expect to see. + + Last block produced via this Python: + import os, binascii + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + key = b'\x44' * 16 + iv = b'\xee' * 16 + cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=default_backend()) + encryptor = cipher.encryptor() + ct = encryptor.update(b'\xaa' * 1 * 32 * 1024) + encryptor.finalize() + print(binascii.hexlify(ct[-16:])) + */ + const uint8_t expected_last_block[] = { + 0x7d, 0x3d, 0x16, 0x84, 0xd0, 0xb4, 0x38, 0x30, + 0xd1, 0x24, 0x6f, 0x7e, 0x9a, 0x9c, 0x81, 0x58, + }; + TEST_ASSERT_EQUAL_HEX8_ARRAY(expected_last_block, buf + CALL_SZ - 16, 16); + + free(buf); + + float usecs = end - start; + // bytes/usec = MB/sec + float mb_sec = (CALL_SZ * CALLS) / usecs; + printf("GCM encryption rate %.3fMB/sec\n", mb_sec); +} + + diff --git a/components/mbedtls/test/test_mbedtls_sha.c b/components/mbedtls/test/test_mbedtls_sha.c index 9408f1462..2882bf599 100644 --- a/components/mbedtls/test/test_mbedtls_sha.c +++ b/components/mbedtls/test/test_mbedtls_sha.c @@ -14,6 +14,7 @@ #include "unity.h" #include "sdkconfig.h" #include "test_apb_dport_access.h" +#include "sodium/utils.h" TEST_CASE("mbedtls SHA self-tests", "[mbedtls]") { @@ -21,19 +22,19 @@ TEST_CASE("mbedtls SHA self-tests", "[mbedtls]") TEST_ASSERT_FALSE_MESSAGE(mbedtls_sha1_self_test(1), "SHA1 self-tests should pass."); TEST_ASSERT_FALSE_MESSAGE(mbedtls_sha256_self_test(1), "SHA256 self-tests should pass."); TEST_ASSERT_FALSE_MESSAGE(mbedtls_sha512_self_test(1), "SHA512 self-tests should pass."); - TEST_ASSERT_FALSE_MESSAGE(mbedtls_sha512_self_test(1), "SHA512 self-tests should pass."); verify_apb_access_loop(); } static const unsigned char *one_hundred_as = (unsigned char *) - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; static const unsigned char *one_hundred_bs = (unsigned char *) - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; static const uint8_t sha256_thousand_as[32] = { 0x41, 0xed, 0xec, 0xe4, 0x2d, 0x63, 0xe8, 0xd9, 0xbf, 0x51, 0x5a, 0x9b, 0xa6, 0x93, 0x2e, 0x1c, - 0x20, 0xcb, 0xc9, 0xf5, 0xa5, 0xd1, 0x34, 0x64, 0x5a, 0xdb, 0x5d, 0xb1, 0xb9, 0x73, 0x7e, 0xa3 }; + 0x20, 0xcb, 0xc9, 0xf5, 0xa5, 0xd1, 0x34, 0x64, 0x5a, 0xdb, 0x5d, 0xb1, 0xb9, 0x73, 0x7e, 0xa3 +}; static const uint8_t sha256_thousand_bs[32] = { 0xf6, 0xf1, 0x18, 0xe1, 0x20, 0xe5, 0x2b, 0xe0, 0xbd, 0x0c, 0xfd, 0xf2, 0x79, 0x4c, 0xd1, 0x2c, 0x07, 0x68, 0x6c, 0xc8, 0x71, 0x23, 0x5a, 0xc2, 0xf1, 0x14, 0x59, 0x37, 0x8e, 0x6d, 0x23, 0x5b @@ -49,7 +50,9 @@ static const uint8_t sha384_thousand_bs[48] = { static const uint8_t sha1_thousand_as[20] = { 0x29, 0x1e, 0x9a, 0x6c, 0x66, 0x99, 0x49, 0x49, 0xb5, 0x7b, 0xa5, - 0xe6, 0x50, 0x36, 0x1e, 0x98, 0xfc, 0x36, 0xb1, 0xba }; + 0xe6, 0x50, 0x36, 0x1e, 0x98, 0xfc, 0x36, 0xb1, 0xba +}; + TEST_CASE("mbedtls SHA interleaving", "[mbedtls]") { @@ -129,8 +132,8 @@ TEST_CASE("mbedtls SHA multithreading", "[mbedtls]") xTaskCreate(tskRunSHA256Test, "SHA256Task1", SHA_TASK_STACK_SIZE, NULL, 3, NULL); xTaskCreate(tskRunSHA256Test, "SHA256Task2", SHA_TASK_STACK_SIZE, NULL, 3, NULL); - for(int i = 0; i < 4; i++) { - if(!xSemaphoreTake(done_sem, 10000/portTICK_PERIOD_MS)) { + for (int i = 0; i < 4; i++) { + if (!xSemaphoreTake(done_sem, 10000 / portTICK_PERIOD_MS)) { TEST_FAIL_MESSAGE("done_sem not released by test task"); } } @@ -140,31 +143,30 @@ TEST_CASE("mbedtls SHA multithreading", "[mbedtls]") void tskRunSHASelftests(void *param) { for (int i = 0; i < 5; i++) { - if(mbedtls_sha1_self_test(1)) { + if (mbedtls_sha1_self_test(1)) { printf("SHA1 self-tests failed.\n"); - while(1) {} + while (1) {} } - if(mbedtls_sha256_self_test(1)) { + if (mbedtls_sha256_self_test(1)) { printf("SHA256 self-tests failed.\n"); - while(1) {} + while (1) {} } - if(mbedtls_sha512_self_test(1)) { + if (mbedtls_sha512_self_test(1)) { printf("SHA512 self-tests failed.\n"); - while(1) {} + while (1) {} } - if(mbedtls_sha512_self_test(1)) { + if (mbedtls_sha512_self_test(1)) { printf("SHA512 self-tests failed.\n"); - while(1) {} + while (1) {} } } xSemaphoreGive(done_sem); vTaskDelete(NULL); } -#if !TEMPORARY_DISABLED_FOR_TARGETS(ESP32S2) TEST_CASE("mbedtls SHA self-tests multithreaded", "[mbedtls]") { done_sem = xSemaphoreCreateCounting(2, 0); @@ -173,14 +175,13 @@ TEST_CASE("mbedtls SHA self-tests multithreaded", "[mbedtls]") const int TIMEOUT_MS = 40000; - for(int i = 0; i < 2; i++) { - if(!xSemaphoreTake(done_sem, TIMEOUT_MS/portTICK_PERIOD_MS)) { + for (int i = 0; i < 2; i++) { + if (!xSemaphoreTake(done_sem, TIMEOUT_MS / portTICK_PERIOD_MS)) { TEST_FAIL_MESSAGE("done_sem not released by test task"); } } vSemaphoreDelete(done_sem); } -#endif //!TEMPORARY_DISABLED_FOR_TARGETS(ESP32S2) TEST_CASE("mbedtls SHA512 clone", "[mbedtls]") { @@ -207,10 +208,11 @@ TEST_CASE("mbedtls SHA512 clone", "[mbedtls]") TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha512_thousand_bs, sha512, 64, "SHA512 cloned calculation"); } -TEST_CASE("mbedtls SHA384 clone", "[mbedtls]") +TEST_CASE("mbedtls SHA384 clone", "[mbedtls][") { mbedtls_sha512_context ctx; mbedtls_sha512_context clone; + unsigned char sha384[48]; mbedtls_sha512_init(&ctx); @@ -220,12 +222,12 @@ TEST_CASE("mbedtls SHA384 clone", "[mbedtls]") } mbedtls_sha512_clone(&clone, &ctx); + for (int i = 0; i < 5; i++) { TEST_ASSERT_EQUAL(0, mbedtls_sha512_update_ret(&ctx, one_hundred_bs, 100)); TEST_ASSERT_EQUAL(0, mbedtls_sha512_update_ret(&clone, one_hundred_bs, 100)); } TEST_ASSERT_EQUAL(0, mbedtls_sha512_finish_ret(&ctx, sha384)); - TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha384_thousand_bs, sha384, 48, "SHA512 original calculation"); TEST_ASSERT_EQUAL(0, mbedtls_sha512_finish_ret(&clone, sha384)); @@ -258,7 +260,6 @@ TEST_CASE("mbedtls SHA256 clone", "[mbedtls]") TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha256_thousand_as, sha256, 32, "SHA256 cloned calculation"); } -#if !TEMPORARY_DISABLED_FOR_TARGETS(ESP32S2) typedef struct { mbedtls_sha256_context ctx; uint8_t result[32]; @@ -279,8 +280,8 @@ static void tskFinaliseSha(void *v_param) vTaskDelete(NULL); } -// No concurrent SHA sessions in esp32s2, only has one engine -TEST_CASE("mbedtls SHA session passed between tasks" , "[mbedtls]") + +TEST_CASE("mbedtls SHA session passed between tasks", "[mbedtls]") { finalise_sha_param_t param = { 0 }; @@ -303,4 +304,130 @@ TEST_CASE("mbedtls SHA session passed between tasks" , "[mbedtls]") TEST_ASSERT_EQUAL(0, param.ret); TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha256_thousand_as, param.result, 32, "SHA256 result from other task"); } -#endif + +/* ESP32 do not have SHA512/t functions */ +#if !DISABLED_FOR_TARGETS(ESP32) + +/* Function are not implemented in SW */ +#ifdef CONFIG_MBEDTLS_HARDWARE_SHA + +/* + * FIPS-180-2 test vectors + */ +static unsigned char sha512T_test_buf[2][113] = { + { "abc" }, + { + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn" + "hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu" + } +}; + +static const size_t sha512T_test_buflen[2] = { + 3, 112 +}; + +static const esp_sha_type sha512T_algo[4] = { + SHA2_512224, SHA2_512256, SHA2_512T, SHA2_512T +}; + +static const size_t sha512T_t_len[4] = { 224, 256, 224, 256 }; + +static const unsigned char sha512_test_sum[4][32] = { + /* SHA512-224 */ + { + 0x46, 0x34, 0x27, 0x0f, 0x70, 0x7b, 0x6a, 0x54, + 0xda, 0xae, 0x75, 0x30, 0x46, 0x08, 0x42, 0xe2, + 0x0e, 0x37, 0xed, 0x26, 0x5c, 0xee, 0xe9, 0xa4, + 0x3e, 0x89, 0x24, 0xaa + }, + { + 0x23, 0xfe, 0xc5, 0xbb, 0x94, 0xd6, 0x0b, 0x23, + 0x30, 0x81, 0x92, 0x64, 0x0b, 0x0c, 0x45, 0x33, + 0x35, 0xd6, 0x64, 0x73, 0x4f, 0xe4, 0x0e, 0x72, + 0x68, 0x67, 0x4a, 0xf9 + }, + + /* SHA512-256 */ + { + 0x53, 0x04, 0x8e, 0x26, 0x81, 0x94, 0x1e, 0xf9, + 0x9b, 0x2e, 0x29, 0xb7, 0x6b, 0x4c, 0x7d, 0xab, + 0xe4, 0xc2, 0xd0, 0xc6, 0x34, 0xfc, 0x6d, 0x46, + 0xe0, 0xe2, 0xf1, 0x31, 0x07, 0xe7, 0xaf, 0x23 + }, + { + 0x39, 0x28, 0xe1, 0x84, 0xfb, 0x86, 0x90, 0xf8, + 0x40, 0xda, 0x39, 0x88, 0x12, 0x1d, 0x31, 0xbe, + 0x65, 0xcb, 0x9d, 0x3e, 0xf8, 0x3e, 0xe6, 0x14, + 0x6f, 0xea, 0xc8, 0x61, 0xe1, 0x9b, 0x56, 0x3a + } + + /* For SHA512_T testing we use t=224 & t=256 + * so the hash digest should be same as above + */ +}; + +/* This will run total of 8 test cases, 2 for each of the below MODE + * SHA512/224, SHA512/256, SHA512/t with t=224 & SHA512/t with t=256 + * + * Test is disabled for ESP32 as there is no hardware for SHA512/t + */ +TEST_CASE("mbedtls SHA512/t", "[mbedtls]") +{ + mbedtls_sha512_context sha512_ctx; + unsigned char sha512[64], k; + + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 2; j++) { + k = i * 2 + j; + mbedtls_sha512_init(&sha512_ctx); + TEST_ASSERT_EQUAL(0, mbedtls_sha512_starts_ret(&sha512_ctx, false)); + esp_sha512_set_mode(&sha512_ctx, sha512T_algo[i]); + if (i > 1) { + k = (i - 2) * 2 + j; + esp_sha512_set_t(&sha512_ctx, sha512T_t_len[i]); + } + TEST_ASSERT_EQUAL(0, mbedtls_sha512_update_ret(&sha512_ctx, sha512T_test_buf[j], sha512T_test_buflen[j])); + TEST_ASSERT_EQUAL(0, mbedtls_sha512_finish_ret(&sha512_ctx, sha512)); + TEST_ASSERT_EQUAL_MEMORY_MESSAGE(sha512_test_sum[k], sha512, sha512T_t_len[i] / 8, "SHA512t calculation"); + } + } +} + +#ifdef CONFIG_SPIRAM +TEST_CASE("mbedtls SHA256 PSRAM DMA", "[mbedtls]") +{ + + const unsigned CALLS = 256; + const unsigned CALL_SZ = 16 * 1024; + mbedtls_sha256_context sha256_ctx; + unsigned char sha256[32]; + + // allocate internal memory + uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_SPIRAM); + TEST_ASSERT(esp_ptr_external_ram(buf)); + memset(buf, 0x54, CALL_SZ); + + mbedtls_sha256_init(&sha256_ctx); + TEST_ASSERT_EQUAL(0, mbedtls_sha256_starts_ret(&sha256_ctx, false)); + for (int c = 0; c < CALLS; c++) { + TEST_ASSERT_EQUAL(0, mbedtls_sha256_update_ret(&sha256_ctx, buf, CALL_SZ)); + } + TEST_ASSERT_EQUAL(0, mbedtls_sha256_finish_ret(&sha256_ctx, sha256)); + + free(buf); + mbedtls_sha256_free(&sha256_ctx); + + /* Check the result. Reference value can be calculated using: + * dd if=/dev/zero bs=$((16*1024)) count=256 | tr '\000' '\124' | sha256sum + */ + const char *expected_hash = "8d031167bd706ac337e07aa9129c34ae4ae792d0a79a2c70e7f012102e8adc3d"; + char hash_str[sizeof(sha256) * 2 + 1]; + sodium_bin2hex(hash_str, sizeof(hash_str), sha256, sizeof(sha256)); + + TEST_ASSERT_EQUAL_STRING(expected_hash, hash_str); + +} +#endif //CONFIG_SPIRAM + +#endif //CONFIG_MBEDTLS_HARDWARE_SHA +#endif //!DISABLED_FOR_TARGETS(ESP32S2) \ No newline at end of file diff --git a/components/mbedtls/test/test_sha_perf.c b/components/mbedtls/test/test_sha_perf.c index 1c69af193..1449e0a42 100644 --- a/components/mbedtls/test/test_sha_perf.c +++ b/components/mbedtls/test/test_sha_perf.c @@ -14,13 +14,13 @@ TEST_CASE("mbedtls SHA performance", "[aes]") { const unsigned CALLS = 256; - const unsigned CALL_SZ = 16*1024; + const unsigned CALL_SZ = 16 * 1024; mbedtls_sha256_context sha256_ctx; int64_t start, end; unsigned char sha256[32]; // allocate internal memory - uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_8BIT|MALLOC_CAP_INTERNAL); + uint8_t *buf = heap_caps_malloc(CALL_SZ, MALLOC_CAP_8BIT | MALLOC_CAP_INTERNAL); TEST_ASSERT_NOT_NULL(buf); memset(buf, 0x55, CALL_SZ); @@ -39,7 +39,7 @@ TEST_CASE("mbedtls SHA performance", "[aes]") /* Check the result. Reference value can be calculated using: * dd if=/dev/zero bs=$((16*1024)) count=256 | tr '\000' '\125' | sha256sum */ - const char* expected_hash = "c88df2638fb9699abaad05780fa5e0fdb6058f477069040eac8bed3231286275"; + const char *expected_hash = "c88df2638fb9699abaad05780fa5e0fdb6058f477069040eac8bed3231286275"; char hash_str[sizeof(sha256) * 2 + 1]; sodium_bin2hex(hash_str, sizeof(hash_str), sha256, sizeof(sha256)); @@ -49,7 +49,7 @@ TEST_CASE("mbedtls SHA performance", "[aes]") // bytes/usec = MB/sec float mb_sec = (CALL_SZ * CALLS) / usecs; printf("SHA256 rate %.3fMB/sec\n", mb_sec); -#ifdef CONFIG_MBEDTLS_HARDWARE +#ifdef CONFIG_MBEDTLS_HARDWARE_SHA // Don't put a hard limit on software SHA performance TEST_PERFORMANCE_GREATER_THAN(SHA256_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif diff --git a/components/soc/include/soc/soc_memory_layout.h b/components/soc/include/soc/soc_memory_layout.h index a393f0841..d88d356a1 100644 --- a/components/soc/include/soc/soc_memory_layout.h +++ b/components/soc/include/soc/soc_memory_layout.h @@ -145,6 +145,15 @@ inline static bool IRAM_ATTR esp_ptr_dma_capable(const void *p) return (intptr_t)p >= SOC_DMA_LOW && (intptr_t)p < SOC_DMA_HIGH; } +inline static bool IRAM_ATTR esp_ptr_dma_ext_capable(const void *p) +{ +#if CONFIG_IDF_TARGET_ESP32S2 + return (intptr_t)p >= SOC_DMA_EXT_LOW && (intptr_t)p < SOC_DMA_EXT_HIGH; +#else + return false; +#endif +} + inline static bool IRAM_ATTR esp_ptr_word_aligned(const void *p) { return ((intptr_t)p) % 4 == 0; diff --git a/components/soc/soc/esp32s2/include/soc/hwcrypto_reg.h b/components/soc/soc/esp32s2/include/soc/hwcrypto_reg.h index b3832eb6b..96a6f98ee 100644 --- a/components/soc/soc/esp32s2/include/soc/hwcrypto_reg.h +++ b/components/soc/soc/esp32s2/include/soc/hwcrypto_reg.h @@ -69,24 +69,6 @@ #define SHA_H_BASE ((DR_REG_SHA_BASE) + 0x40) #define SHA_TEXT_BASE ((DR_REG_SHA_BASE) + 0x80) -/* AES Block operation modes */ -#define AES_BLOCK_MODE_ECB 0 -#define AES_BLOCK_MODE_CBC 1 -#define AES_BLOCK_MODE_OFB 2 -#define AES_BLOCK_MODE_CTR 3 -#define AES_BLOCK_MODE_CFB8 4 -#define AES_BLOCK_MODE_CFB128 5 -#define AES_BLOCK_MODE_GCM 6 - -/* AES Block operation modes (used with DMA) */ -#define AES_BLOCK_MODE_ECB 0 -#define AES_BLOCK_MODE_CBC 1 -#define AES_BLOCK_MODE_OFB 2 -#define AES_BLOCK_MODE_CTR 3 -#define AES_BLOCK_MODE_CFB8 4 -#define AES_BLOCK_MODE_CFB128 5 -#define AES_BLOCK_MODE_GCM 6 - /* AES acceleration registers */ #define AES_MODE_REG ((DR_REG_AES_BASE) + 0x40) #define AES_ENDIAN_REG ((DR_REG_AES_BASE) + 0x44) diff --git a/components/soc/soc/esp32s2/include/soc/periph_defs.h b/components/soc/soc/esp32s2/include/soc/periph_defs.h index 5a98443dd..cbd9af557 100644 --- a/components/soc/soc/esp32s2/include/soc/periph_defs.h +++ b/components/soc/soc/esp32s2/include/soc/periph_defs.h @@ -48,6 +48,12 @@ typedef enum { PERIPH_WIFI_MODULE, PERIPH_WIFI_BT_COMMON_MODULE, PERIPH_SYSTIMER_MODULE, + PERIPH_AES_MODULE, + PERIPH_SHA_MODULE, + PERIPH_RSA_MODULE, + PERIPH_CRYPTO_DMA_MODULE, //this DMA is shared between AES and SHA + PERIPH_AES_DMA_MODULE, + PERIPH_SHA_DMA_MODULE, } periph_module_t; typedef enum { diff --git a/components/soc/soc/esp32s2/include/soc/soc.h b/components/soc/soc/esp32s2/include/soc/soc.h index ec1533045..53459679d 100644 --- a/components/soc/soc/esp32s2/include/soc/soc.h +++ b/components/soc/soc/esp32s2/include/soc/soc.h @@ -270,10 +270,14 @@ #define SOC_DIRAM_DRAM_LOW 0x3FFB0000 #define SOC_DIRAM_DRAM_HIGH 0x40000000 -// Region of memory accessible via DMA. See esp_ptr_dma_capable(). +// Region of memory accessible via DMA in internal memory. See esp_ptr_dma_capable(). #define SOC_DMA_LOW 0x3FFB0000 #define SOC_DMA_HIGH 0x40000000 +// Region of memory accessible via DMA in external memory. See esp_ptr_dma_ext_capable(). +#define SOC_DMA_EXT_LOW 0x3F500000 +#define SOC_DMA_EXT_HIGH 0x3FF80000 + // Region of memory that is byte-accessible. See esp_ptr_byte_accessible(). #define SOC_BYTE_ACCESSIBLE_LOW 0x3FF9E000 #define SOC_BYTE_ACCESSIBLE_HIGH 0x40000000 diff --git a/components/soc/src/esp32s2/include/hal/clk_gate_ll.h b/components/soc/src/esp32s2/include/hal/clk_gate_ll.h index e2aed3824..9225de99a 100644 --- a/components/soc/src/esp32s2/include/hal/clk_gate_ll.h +++ b/components/soc/src/esp32s2/include/hal/clk_gate_ll.h @@ -84,6 +84,18 @@ static inline uint32_t periph_ll_get_clk_en_mask(periph_module_t periph) return DPORT_WIFI_CLK_WIFI_BT_COMMON_M; case PERIPH_SYSTIMER_MODULE: return DPORT_SYSTIMER_CLK_EN; + case PERIPH_AES_MODULE: + return DPORT_CRYPTO_AES_CLK_EN; + case PERIPH_SHA_MODULE: + return DPORT_CRYPTO_SHA_CLK_EN; + case PERIPH_RSA_MODULE: + return DPORT_CRYPTO_RSA_CLK_EN; + case PERIPH_CRYPTO_DMA_MODULE: + return DPORT_CRYPTO_DMA_CLK_EN; + case PERIPH_SHA_DMA_MODULE: + return DPORT_CRYPTO_DMA_CLK_EN | DPORT_CRYPTO_SHA_CLK_EN; + case PERIPH_AES_DMA_MODULE: + return DPORT_CRYPTO_DMA_CLK_EN | DPORT_CRYPTO_AES_CLK_EN; default: return 0; } @@ -144,6 +156,49 @@ static inline uint32_t periph_ll_get_rst_en_mask(periph_module_t periph, bool en return DPORT_CAN_RST; case PERIPH_SYSTIMER_MODULE: return DPORT_SYSTIMER_RST; + case PERIPH_AES_MODULE: + if (enable == true) { + // Clear reset on digital signature, otherwise AES unit is held in reset also. + return (DPORT_CRYPTO_AES_RST | DPORT_CRYPTO_DS_RST); + } else { + //Don't return other units to reset, as this pulls reset on RSA & SHA units, respectively. + return DPORT_CRYPTO_AES_RST; + } + case PERIPH_SHA_MODULE: + if (enable == true) { + // Clear reset on digital signature and HMAC, otherwise SHA is held in reset + return (DPORT_CRYPTO_SHA_RST | DPORT_CRYPTO_DS_RST | DPORT_CRYPTO_HMAC_RST); + } else { + // Don't assert reset on secure boot, otherwise AES is held in reset + return DPORT_CRYPTO_SHA_RST; + } + case PERIPH_RSA_MODULE: + if (enable == true) { + // Also clear reset on digital signature, otherwise RSA is held in reset + return (DPORT_CRYPTO_RSA_RST | DPORT_CRYPTO_DS_RST); + } else { + // Don't reset digital signature unit, as this resets AES also + return DPORT_CRYPTO_RSA_RST; + } + case PERIPH_CRYPTO_DMA_MODULE: + return DPORT_CRYPTO_DMA_RST; + + case PERIPH_AES_DMA_MODULE: + if (enable == true) { + // Clear reset on digital signature, otherwise AES unit is held in reset also. + return (DPORT_CRYPTO_AES_RST | DPORT_CRYPTO_DS_RST | DPORT_CRYPTO_DMA_RST); + } else { + //Don't return other units to reset, as this pulls reset on RSA & SHA units, respectively. + return (DPORT_CRYPTO_AES_RST | DPORT_CRYPTO_DMA_RST); + } + case PERIPH_SHA_DMA_MODULE: + if (enable == true) { + // Clear reset on digital signature and HMAC, otherwise SHA is held in reset + return (DPORT_CRYPTO_SHA_RST | DPORT_CRYPTO_DS_RST | DPORT_CRYPTO_HMAC_RST | DPORT_CRYPTO_DMA_RST); + } else { + // Don't assert reset on secure boot, otherwise AES is held in reset + return (DPORT_CRYPTO_SHA_RST | DPORT_CRYPTO_DMA_RST); + } default: return 0; } @@ -156,6 +211,13 @@ static inline uint32_t periph_ll_get_clk_en_reg(periph_module_t periph) case PERIPH_WIFI_MODULE: case PERIPH_WIFI_BT_COMMON_MODULE: return DPORT_WIFI_CLK_EN_REG; + case PERIPH_AES_MODULE: + case PERIPH_SHA_MODULE: + case PERIPH_RSA_MODULE: + case PERIPH_CRYPTO_DMA_MODULE: + case PERIPH_AES_DMA_MODULE: + case PERIPH_SHA_DMA_MODULE: + return DPORT_PERIP_CLK_EN1_REG; default: return DPORT_PERIP_CLK_EN_REG; } @@ -168,6 +230,13 @@ static inline uint32_t periph_ll_get_rst_en_reg(periph_module_t periph) case PERIPH_WIFI_MODULE: case PERIPH_WIFI_BT_COMMON_MODULE: return DPORT_CORE_RST_EN_REG; + case PERIPH_AES_MODULE: + case PERIPH_SHA_MODULE: + case PERIPH_RSA_MODULE: + case PERIPH_CRYPTO_DMA_MODULE: + case PERIPH_AES_DMA_MODULE: + case PERIPH_SHA_DMA_MODULE: + return DPORT_PERIP_RST_EN1_REG; default: return DPORT_PERIP_RST_EN_REG; }