From dcd63f73f6380a75e73043b0d4fbebde8fa272c9 Mon Sep 17 00:00:00 2001 From: Shivani Tipnis Date: Wed, 17 Oct 2018 17:08:32 +0530 Subject: [PATCH] nvs_util: Add changes to nvs part gen util nvs_util: Add changes to nvs part gen util nvs_host_test: Add test case for keygen and keyfile paratmeters (cherry picked from commit 9db19878927233d8d15a3a7fd568bfcc380a41d3) --- .../nvs_partition_generator/README.rst | 146 +++++--- .../nvs_partition_gen.py | 240 +++++++----- .../testdata/encryption_keys.txt | 1 - .../testdata/sample_encryption_keys.bin | 1 + .../nvs_flash/test_nvs_host/test_nvs.cpp | 351 +++++++++--------- 5 files changed, 411 insertions(+), 328 deletions(-) delete mode 100644 components/nvs_flash/nvs_partition_generator/testdata/encryption_keys.txt create mode 100644 components/nvs_flash/nvs_partition_generator/testdata/sample_encryption_keys.bin diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index bea753864..e0103b829 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -56,7 +56,7 @@ When a new namespace entry is encountered in the CSV file, each follow-up entrie Multipage Blob Support ---------------------- -By default, binary blobs are allowed to span over multiple pages and written in the format mentioned in section :ref:`structure_of_entry`. +By default, binary blobs are allowed to span over multiple pages and written in the format mentioned in section :ref:`structure_of_entry`. If older format is intended to be used, the utility provides an option to disable this feature. Encryption Support @@ -66,76 +66,108 @@ This utility allows you to create an enrypted binary file also. Encryption used Running the utility ------------------- -*Usage*:: +**Usage**:: - python nvs_partition_gen.py [--version {v1,v2}] input output - -You can run this utility in two modes: - - Normal mode - Binary generated in this mode is an unencrypted binary file. - - Encryption mode - Binary generated in this mode is an encrypted binary file. - -*In normal mode:* - - A sample CSV file is provided with the utility. You can run the utility using below command:: - - python nvs_partition_generator.py sample.csv sample.bin - -*In encryption mode:* - - You can run the utility using below commands: - - - By taking encryption keys as an input file. A sample encryption keys file is provided with the utility:: - - python nvs_partition_gen.py sample.csv sample_encrypted.bin --encrypt True --keyfile testdata/keys.txt - - - By enabling generation of encryption keys:: - - python nvs_partition_gen.py sample.csv sample_encrypted.bin --encrypt True --keygen True - - -.. note:: In encryption mode, this utility creates a binary file named `encryption_keys.bin` containing the encryption keys used. This binary file is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. - - -You can also provide the format version number while running this utility: - - Multipage Blob Support Enabled (v2) - - Multipage Blob Support Disabled (v1) - - -*Multipage Blob Support Enabled (v2):* - -You can run the utility in this format by setting the version parameter to v2, as shown below. -A sample CSV file is provided with the utility:: - - python nvs_partition_gen.py sample_multipage_blob.csv partition_multipage_blob.bin --version v2 - - -*Multipage Blob Support Disabled (v1):* - -You can run the utility in this format by setting the version parameter to v1, as shown below. -A sample CSV file is provided with the utility:: - - python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1 + python nvs_partition_gen.py [-h] [--input INPUT] [--output OUTPUT] + [--size SIZE] [--version {v1,v2}] + [--keygen {true,false}] [--encrypt {true,false}] + [--keyfile KEYFILE] +------------------------+----------------------------------------------------------------------------------------------+ | Arguments | Description | +========================+==============================================================================================+ -| input | Path to CSV file to parse. Will use stdin if omitted | +| --input INPUT | Path to CSV file to parse. | +------------------------+----------------------------------------------------------------------------------------------+ -| output | Path to output converted binary file. Will use stdout if omitted | +| --output OUTPUT | Path to output generated binary file. | +------------------------+----------------------------------------------------------------------------------------------+ -| size | Size of NVS Partition in bytes (must be multiple of 4096) | +| --size SIZE | Size of NVS Partition in bytes (must be multiple of 4096) | +------------------------+----------------------------------------------------------------------------------------------+ -| --version {v1,v2} | Set version. Default: v2 | -+-------------------------------+---------------------------------------------------------------------------------------+ -| --keygen {True,False} | Generate keys for encryption. Default: False | -| | (Applicable only if encryption mode is true) | +| --version {v1,v2} | Set version. Default: v2 | +------------------------+----------------------------------------------------------------------------------------------+ -| --encrypt {True,False} | Set encryption mode. Default: False | +| --keygen {true,false} | Generate keys for encryption. Creates an `encryption_keys.bin` file (in current directory). | +| | Default: false | ++------------------------+----------------------------------------------------------------------------------------------+ +| --encrypt {true,false} | Set encryption mode. Default: false | +------------------------+----------------------------------------------------------------------------------------------+ | --keyfile KEYFILE | File having key for encryption (Applicable only if encryption mode is true) | +------------------------+----------------------------------------------------------------------------------------------+ + + +You can run this utility in two modes: + - Normal mode - Binary generated in this mode is an unencrypted binary file. + - Encryption mode - Binary generated in this mode is an encrypted binary file. + + +**In normal mode:** + +*Usage*:: + + python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT + --size SIZE [--version {v1,v2}] + [--keygen {true,false}] [--encrypt {true,false}] + [--keyfile KEYFILE] + +You can run the utility using below command:: + + python nvs_partition_gen.py --input sample.csv --output sample.bin --size 0x3000 + + + +**In encryption mode:** + +*Usage*:: + + python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT + --size SIZE --encrypt {true,false} + --keygen {true,false} | --keyfile KEYFILE + [--version {v1,v2}] + + +You can run the utility using below commands: + + - By taking encryption keys as an input file. A sample encryption keys binary file is provided with the utility:: + + python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keyfile testdata/sample_encryption_keys.bin + + - By enabling generation of encryption keys:: + + python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keygen true + + + +*To generate* **only** *encryption keys with this utility* ( Creates an `encryption_keys.bin` file in current directory ): :: + + python nvs_partition_gen.py --keygen true + +.. note:: This `encryption_keys.bin` file is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. + + + +You can also provide the format version number (in any of the two modes): + - Multipage Blob Support Enabled (v2) + - Multipage Blob Support Disabled (v1) + + +**Multipage Blob Support Enabled (v2):** + +You can run the utility in this format by setting the version parameter to v2, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py --input sample_multipage_blob.csv --output partition_multipage_blob.bin --size 0x3000 --version v2 + + +**Multipage Blob Support Disabled (v1):** + +You can run the utility in this format by setting the version parameter to v1, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py --input sample_singlepage_blob.csv --output partition_single_page.bin --size 0x3000 --version v1 + + +.. note:: *When flashing the binary onto the device, make sure it is consistent with the application's sdkconfig.* + Caveats ------- - Utility doesn't check for duplicate keys and will write data pertaining to both keys. User needs to make sure keys are distinct. diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index 416c146fc..a4d5f78d5 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -19,7 +19,7 @@ # from __future__ import division, print_function -from builtins import int, range +from builtins import int, range, bytes from io import open import sys import argparse @@ -34,6 +34,10 @@ import codecs from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend +VERSION1_PRINT = "v1 - Multipage Blob Support Disabled" +VERSION2_PRINT = "v2 - Multipage Blob Support Enabled" + + """ Class for standard NVS page structure """ class Page(object): PAGE_PARAMS = { @@ -153,7 +157,10 @@ class Page(object): # Extract encryption key and tweak key from given key input - encr_key_input = codecs.decode(self.encr_key, 'hex') + if len(self.encr_key) == key_len_needed: + encr_key_input = self.encr_key + else: + encr_key_input = codecs.decode(self.encr_key, 'hex') rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET @@ -230,7 +237,7 @@ class Page(object): return entry_struct - def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj): + def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, encoding, nvs_obj): chunk_start = 0 chunk_count = 0 chunk_index = Page.CHUNK_ANY @@ -273,7 +280,10 @@ class Page(object): # Compute CRC of data chunk struct.pack_into(' Page.PAGE_PARAMS["max_old_blob_size"]: - raise InputError("%s: Size exceeds max allowed length." % key) + raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key)) if version == Page.VERSION2: if encoding == "string": if datalen > Page.PAGE_PARAMS["max_new_blob_size"]: - raise InputError("%s: Size exceeds max allowed length." % key) + raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key)) # Calculate no. of entries data will require rounded_size = (datalen + 31) & ~31 @@ -398,7 +410,7 @@ class Page(object): if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]): entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\ - datalen,total_entry_count, nvs_obj) + datalen,total_entry_count, encoding, nvs_obj) else: self.write_single_page_entry(entry_struct, data, datalen, data_entry_count, nvs_obj) @@ -480,6 +492,10 @@ class NVS(object): break result = self.get_binary_data() + if version == Page.VERSION1: + print("Version: ", VERSION1_PRINT) + else: + print("Version: ", VERSION2_PRINT) self.fout.write(result) def create_new_page(self, is_rsrv_page=False): @@ -533,6 +549,7 @@ class NVS(object): encoding = encoding.lower() varlen_encodings = ["string", "binary", "hex2bin", "base64"] primitive_encodings = ["u8", "i8", "u16", "u32", "i32"] + if encoding in varlen_encodings: try: self.cur_page.write_varlen_data(key, value, encoding, self.namespace_idx,self) @@ -620,154 +637,193 @@ def nvs_close(nvs_instance): """ nvs_instance.__exit__(None, None, None) -def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key_gen=None, encrypt_mode=None, key_file=None, version_no=None): - """ Wrapper to generate nvs partition binary - :param input_filename: Name of input file containing data - :param output_filename: Name of output file to store generated binary - :param input_size: Size of partition in bytes (must be multiple of 4096) - :param key_gen: Enable encryption key generation in encryption mode - :param encrypt_mode: Enable/Disable encryption mode - :param key_file: Input file having encryption keys in encryption mode - :return: None - """ - global version, is_encrypt_data, key_input +def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None,\ +encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None): + + global version, is_encrypt_data, input_size, key_gen + version = version_no - key_input = None is_encrypt_data = encrypt_mode + key_gen = is_key_gen + input_size = input_part_size - # Set size - input_size = int(input_size, 0) + if is_encrypt_data.lower() == 'true': + is_encrypt_data = True + elif is_encrypt_data.lower() == 'false': + is_encrypt_data = False - if input_size % 4096 !=0: - sys.exit("Size of partition (must be multiple of 4096)") if version == 'v1': version = Page.VERSION1 elif version == 'v2': version = Page.VERSION2 - # Update size as a page needs to be reserved of size 4KB - input_size = input_size - Page.PAGE_PARAMS["max_size"] - - if input_size == 0: - sys.exit("Size parameter is insufficient.") - - if is_encrypt_data == 'True': - is_encrypt_data = True - elif is_encrypt_data == 'False': - is_encrypt_data = False - - if key_gen == 'True': + if key_gen.lower() == 'true': key_gen = True - elif key_gen == 'False': + elif key_gen.lower() == 'false': key_gen = False + + if key_gen: + if all(arg is not None for arg in [input_filename, output_filename, input_size]): + if not is_encrypt_data: + sys.exit("--encrypt argument is missing or set to false.") + elif any(arg is not None for arg in [input_filename, output_filename, input_size]): + sys.exit(print_arg_str) + else: + if not input_size: + if not all(arg is not None for arg in [input_filename, output_filename]): + sys.exit(print_arg_str) + + if is_encrypt_data and not key_gen and not key_file: - sys.exit("Missing parameter. Enter --keyfile or --keygen.") + sys.exit(print_encrypt_arg_str) if is_encrypt_data and key_gen and key_file: - sys.exit("Only one input allowed. Enter --keyfile or --keygen.") - - if not is_encrypt_data and key_gen: - sys.exit("Invalid. Cannot give --key_gen as --encrypt is set to False.") + sys.exit(print_encrypt_arg_str) if not is_encrypt_data and key_file: - sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.") + sys.exit("Invalid. Cannot give --keyfile as --encrypt is set to false.") + + if input_size: + # Set size + input_size = int(input_size, 0) + + if input_size % 4096 !=0: + sys.exit("Size of partition (must be multiple of 4096)") + + # Update size as a page needs to be reserved of size 4KB + input_size = input_size - Page.PAGE_PARAMS["max_size"] + + if input_size == 0: + sys.exit("Size parameter is insufficient.") + + + + +def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None): + """ Wrapper to generate nvs partition binary + + :param input_filename: Name of input file containing data + :param output_filename: Name of output file to store generated binary + :param input_part_size: Size of partition in bytes (must be multiple of 4096) + :param is_key_gen: Enable encryption key generation in encryption mode + :param encrypt_mode: Enable/Disable encryption mode + :param key_file: Input file having encryption keys in encryption mode + :param version_no: Format Version number + :return: None + """ + + global key_input, key_len_needed + + key_len_needed = 64 + key_input = bytearray() if key_gen: key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip() elif key_file: - with open(key_file, 'rt', encoding='utf8') as key_f: - key_input = key_f.readline() - key_input = key_input.strip() + with open(key_file, 'rb') as key_f: + key_input = key_f.read(64) - input_file = open(input_filename, 'rt', encoding='utf8') - output_file = open(output_filename, 'wb') + if all(arg is not None for arg in [input_filename, output_filename, input_size]): + input_file = open(input_filename, 'rt', encoding='utf8') + output_file = open(output_filename, 'wb') - with nvs_open(output_file, input_size) as nvs_obj: - reader = csv.DictReader(input_file, delimiter=',') - for row in reader: - try: - write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) - except (InputError) as e: - print(e) - input_file.close() - output_file.close() - sys.exit(-2) + with nvs_open(output_file, input_size) as nvs_obj: + reader = csv.DictReader(input_file, delimiter=',') + for row in reader: + try: + write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) + except (InputError) as e: + print(e) + input_file.close() + output_file.close() + sys.exit(-2) - input_file.close() - output_file.close() + input_file.close() + output_file.close() - if is_encrypt_data: + + if key_gen: keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"] key_bytes = bytearray() - - key_bytes = codecs.decode(key_input, 'hex') + if len(key_input) == key_len_needed: + key_bytes = key_input + else: + key_bytes = codecs.decode(key_input, 'hex') key_len = len(key_bytes) keys_page_buf[0:key_len] = key_bytes - crc_data = keys_page_buf[0:key_len] crc_data = bytes(crc_data) crc = zlib.crc32(crc_data, 0xFFFFFFFF) - struct.pack_into(' 0); int status; @@ -2315,77 +2279,108 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted.bin"); - nvs_handle handle; - - nvs_sec_cfg_t xts_cfg; - + + nvs_sec_cfg_t cfg; for(int count = 0; count < NVS_KEY_SIZE; count++) { - xts_cfg.eky[count] = 0x11; - xts_cfg.tky[count] = 0x22; + cfg.eky[count] = 0x11; + cfg.tky[count] = 0x22; } - TEST_ESP_OK(nvs_flash_secure_init_custom(NVS_DEFAULT_PART_NAME, 0, 3, &xts_cfg)); - - TEST_ESP_OK(nvs_open_from_partition(NVS_DEFAULT_PART_NAME, "dummyNamespace", NVS_READONLY, &handle)); - - uint8_t u8v; - TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); - CHECK(u8v == 127); + check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); - int8_t i8v; - TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v)); - CHECK(i8v == -128); - uint16_t u16v; - TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v)); - CHECK(u16v == 32768); - uint32_t u32v; - TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v)); - CHECK(u32v == 4294967295); - int32_t i32v; - TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v)); - CHECK(i32v == -2147483648); +} - char buf[64] = {0}; - size_t buflen = 64; - TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen)); - CHECK(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0); - uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; - buflen = 64; - TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); - CHECK(memcmp(buf, hexdata, buflen) == 0); +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keygen", "[nvs_part_gen]") +{ + int childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "--input", + "../nvs_partition_generator/sample_multipage_blob.csv", + "--output", + "../nvs_partition_generator/partition_encrypted_using_keygen.bin", + "--size", + "0x3000", + "--encrypt", + "True", + "--keygen", + "true",NULL)); - uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; - buflen = 64; - TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); - CHECK(memcmp(buf, base64data, buflen) == 0); + } else { + CHECK(childpid > 0); + int status; + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) != -1); + } - uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}; - buflen = 64; - TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen)); - CHECK(memcmp(buf, hexfiledata, buflen) == 0); - - uint8_t base64filedata[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xab, 0xcd, 0xef}; - buflen = 64; - TEST_ESP_OK( nvs_get_blob(handle, "base64FileKey", buf, &buflen)); - CHECK(memcmp(buf, base64filedata, buflen) == 0); - - uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0"; - buflen = 64; - TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen)); - CHECK(memcmp(buf, strfiledata, buflen) == 0); - - char bin_data[5120]; - size_t bin_len = sizeof(bin_data); - char binfiledata[5200]; - ifstream file; - file.open("../nvs_partition_generator/testdata/sample_multipage_blob.bin"); - file.read(binfiledata,5120); - TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); - CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); + SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keygen.bin"); - nvs_close(handle); TEST_ESP_OK(nvs_flash_deinit()); + + char buffer[64]; + FILE *fp; + + fp = fopen("encryption_keys.bin","rb"); + fread(buffer,sizeof(buffer),1,fp); + + fclose(fp); + + nvs_sec_cfg_t cfg; + + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = buffer[count] & 255; + cfg.tky[count] = buffer[count+32] & 255; + } + + check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); + + +} + +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keyfile", "[nvs_part_gen]") +{ + int childpid = fork(); + if (childpid == 0) { + exit(execlp("python", "python", + "../nvs_partition_generator/nvs_partition_gen.py", + "--input", + "../nvs_partition_generator/sample_multipage_blob.csv", + "--output", + "../nvs_partition_generator/partition_encrypted_using_keyfile.bin", + "--size", + "0x3000", + "--encrypt", + "True", + "--keyfile", + "encryption_keys.bin",NULL)); + + } else { + CHECK(childpid > 0); + int status; + waitpid(childpid, &status, 0); + CHECK(WEXITSTATUS(status) != -1); + } + + SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keyfile.bin"); + + char buffer[64]; + FILE *fp; + + fp = fopen("encryption_keys.bin","rb"); + fread(buffer,sizeof(buffer),1,fp); + + fclose(fp); + + nvs_sec_cfg_t cfg; + + for(int count = 0; count < NVS_KEY_SIZE; count++) { + cfg.eky[count] = buffer[count] & 255; + cfg.tky[count] = buffer[count+32] & 255; + } + + check_nvs_part_gen_args(NVS_DEFAULT_PART_NAME, "../nvs_partition_generator/testdata/sample_multipage_blob.bin", true, &cfg); } #endif