From 3636e75792447ef60e2f02a400615aaf64ce786f Mon Sep 17 00:00:00 2001 From: Shivani Tipnis Date: Tue, 7 May 2019 15:36:02 +0530 Subject: [PATCH] nvs_util: Add NVS decryption feature --- .../nvs_partition_generator/README.rst | 301 +++++--- .../nvs_partition_gen.py | 641 +++++++++++------- .../nvs_flash/test_nvs_host/test_nvs.cpp | 78 +-- 3 files changed, 626 insertions(+), 394 deletions(-) diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index bcb74f7cd..800344223 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -7,7 +7,6 @@ Introduction The utility :component_file:`nvs_flash/nvs_partition_generator/nvs_partition_gen.py` creates a binary file based on key-value pairs provided in a CSV file. The binary file is compatible with NVS architecture defined in :doc:`Non-Volatile Storage `. This utility is ideally suited for generating a binary blob, containing data specific to ODM/OEM, which can be flashed externally at the time of device manufacturing. This allows manufacturers to generate many instances of the same application firmware with customized parameters for each device, such as a serial number. - Prerequisites ------------- To use this utility in encryption mode, install the following packages: @@ -15,7 +14,6 @@ To use this utility in encryption mode, install the following packages: All the required packages are included in `requirements.txt` in the root of the esp-idf directory. - CSV file format --------------- @@ -51,7 +49,6 @@ Below is an example dump of such a CSV file:: key1,data,u8,1 key2,file,string,/path/to/file - .. note:: Make sure there are **no spaces**: @@ -80,124 +77,230 @@ Encryption Support The NVS Partition Generator utility also allows you to create an encrypted binary file. The utility uses the AES-XTS encryption. Please refer to :ref:`nvs_encryption` for more details. +Decryption Support +------------------- +This utility allows you to decrypt an encrypted NVS binary file. The utility uses an NVS binary file encrypted using AES-XTS encryption. Please refer to :ref:`nvs_encryption` for more details. + Running the utility ------------------- **Usage**:: - python nvs_partition_gen.py [-h] [--input INPUT] [--output OUTPUT] - [--size SIZE] [--version {v1,v2}] - [--keygen {true,false}] [--encrypt {true,false}] - [--keyfile KEYFILE] [--outdir OUTDIR] + python nvs_partition_gen.py [-h] {generate,generate-key,encrypt,decrypt} ... + + Optional Arguments: + +-----+------------+----------------------------------------------------------------------+ + | No. | Parameter | Description | + +=====+============+======================================================================+ + | 1 | -h, --help | show this help message and exit | + +-----+------------+----------------------------------------------------------------------+ + + Commands: + Run nvs_partition_gen.py {command} -h for additional help + +-----+--------------+--------------------------------------------------------------------+ + | No. | Parameter | Description | + +=====+==============+====================================================================+ + | 1 | generate | Generate NVS partition | + +-----+--------------+--------------------------------------------------------------------+ + | 2 | generate-key | Generate keys for encryption | + +-----+--------------+--------------------------------------------------------------------+ + | 3 | encrypt | Generate NVS encrypted partition | + +-----+--------------+--------------------------------------------------------------------+ + | 4 | decrypt | Decrypt NVS encrypted partition | + +-----+--------------+--------------------------------------------------------------------+ + + +To generate NVS partition (Default): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py generate [-h] [--version {1,2}] [--outdir OUTDIR] + input output size + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to CSV file to parse | + +--------------+----------------------------------------------------------------------+ + | output | Path to output NVS binary file | + +--------------+----------------------------------------------------------------------+ + | size | Size of NVS partition in bytes (must be multiple of 4096) | + +--------------+----------------------------------------------------------------------+ -+------------------------+---------------------------------------------------+-------------------+ -| Arguments | Description | Default Value | -+========================+===================================================+===================+ -| --input INPUT | Path to a CSV file to parse. | | -+------------------------+---------------------------------------------------+-------------------+ -| --output OUTPUT | Path to the generated binary file. | | -+------------------------+---------------------------------------------------+-------------------+ -| --size SIZE | Size of NVS Partition in bytes | | -| | (must be multiple of 4096). | | -+------------------------+---------------------------------------------------+-------------------+ -| --version {v1,v2} | Set version. | v2 | -+------------------------+---------------------------------------------------+-------------------+ -| --keygen {true,false} | Generate keys for encryption. | | -+------------------------+---------------------------------------------------+-------------------+ -| --encrypt {true,false} | Set encryption mode. Default: false. | false | -+------------------------+---------------------------------------------------+-------------------+ -| --keyfile KEYFILE | File containing the key for encryption | | -| | (Applicable only if encryption mode is true). | | -+------------------------+---------------------------------------------------+-------------------+ -| --outdir OUTDIR | The output directory to store the created files. | current directory | -+------------------------+---------------------------------------------------+-------------------+ - -You can run this utility in two modes: - - - **Default mode**: You get an unencrypted binary file. - - **Encryption mode**: You get an encrypted binary file. + Optional Arguments: + +-----------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=================+====================================================================+ + | -h, --help | show this help message and exit | + +-----------------+--------------------------------------------------------------------+ + | --version {1,2} | Set multipage blob version. | + | | Version 1 - Multipage blob support disabled. | + | | Version 2 - Multipage blob support enabled. | + | | Default: Version 2 | + | | | + +-----------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +-----------------+--------------------------------------------------------------------+ -**In default mode:** --------------------- - -*Usage*:: - - python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT - --size SIZE [--version {v1,v2}] - [--keygen {true,false}] [--encrypt {true,false}] - [--keyfile KEYFILE] [--outdir OUTDIR] - -You can run the utility using the command below:: - - python nvs_partition_gen.py --input sample.csv --output sample.bin --size 0x3000 - - -**In encryption mode:** ------------------------ - -*Usage*:: - - python nvs_partition_gen.py [-h] --input INPUT --output OUTPUT - --size SIZE --encrypt {true,false} - --keygen {true,false} --keyfile KEYFILE - [--version {v1,v2}] [--outdir OUTDIR] - - -You can run the utility using one of the commands below: - - - By enabling generation of encryption keys:: - - python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keygen true - - - By taking encryption keys as an input file. A sample binary file containing encryption keys is provided with the utility:: - - python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keyfile testdata/sample_encryption_keys.bin - - - By enabling generation of encryption keys and storing the keys in a binary file with a custom filename:: - - python nvs_partition_gen.py --input sample.csv --output sample_encrypted.bin --size 0x3000 --encrypt true --keygen true --keyfile encryption_keys_generated.bin - -.. note:: If `--keygen` is given with the `--keyfile` argument, generated keys will be stored in the `--keyfile` file. If `--keygen` argument is absent, `--keyfile` is taken as input file containing encryption keys. - - -*To generate* **only** *encryption keys with this utility*:: - - python nvs_partition_gen.py --keygen true - -This creates an `encryption_keys_.bin` file. - -.. note:: This newly created file having encryption keys in `keys/` directory is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. - - -You can also provide the format version number (in any of the two modes): - - Multipage Blob Support Enabled (v2) - - Multipage Blob Support Disabled (v1) - - -**Multipage Blob Support Enabled (v2):** ----------------------------------------- - -You can run the utility in this format by setting the version parameter to v2, as shown below. +You can run the utility to generate NVS partition using the command below: A sample CSV file is provided with the utility:: - python nvs_partition_gen.py --input sample_multipage_blob.csv --output partition_multipage_blob.bin --size 0x4000 --version v2 + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 -**Multipage Blob Support Disabled (v1):** ------------------------------------------ +To generate only encryption keys: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: -You can run the utility in this format by setting the version parameter to v1, as shown below. + python nvs_partition_gen.py generate-key [-h] [--keyfile KEYFILE] + [--outdir OUTDIR] + + Optional Arguments: + +--------------------+----------------------------------------------------------------------+ + | Parameter | Description | + +====================+======================================================================+ + | -h, --help | show this help message and exit | + +--------------------+----------------------------------------------------------------------+ + | --keyfile KEYFILE | Path to output encryption keys file | + +--------------------+----------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created. | + | | (Default: current directory) | + +--------------------+----------------------------------------------------------------------+ + +You can run the utility to generate only encryption keys using the command below:: + + python nvs_partition_gen.py generate-key + + +To generate encrypted NVS partition: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py encrypt [-h] [--version {1,2}] [--keygen] + [--keyfile KEYFILE] [--inputkey INPUTKEY] + [--outdir OUTDIR] + input output size + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to CSV file to parse | + +--------------+----------------------------------------------------------------------+ + | output | Path to output NVS binary file | + +--------------+----------------------------------------------------------------------+ + | size | Size of NVS partition in bytes (must be multiple of 4096) | + +--------------+----------------------------------------------------------------------+ + + + Optional Arguments: + +---------------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=====================+====================================================================+ + | -h, --help | show this help message and exit | + | | | + +---------------------+--------------------------------------------------------------------+ + | --version {1,2} | Set multipage blob version. | + | | Version 1 - Multipage blob support disabled. | + | | Version 2 - Multipage blob support enabled. | + | | Default: Version 2 | + +---------------------+--------------------------------------------------------------------+ + | --keygen | Generates key for encrypting NVS partition | + +---------------------+--------------------------------------------------------------------+ + | --keyfile KEYFILE | Path to output encryption keys file | + +---------------------+--------------------------------------------------------------------+ + | --inputkey INPUTKEY | File having key for encrypting NVS partition | + +---------------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +---------------------+--------------------------------------------------------------------+ + + +You can run the utility to encrypt NVS partition using the command below: +A sample CSV file is provided with the utility: + +- Encrypt by allowing the utility to generate encryption keys:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen + +.. note:: Encryption key of the following format ``/keys/keys-.bin`` is created. + +- Encrypt by allowing the utility to generate encryption keys and store it in provided custom filename:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --keygen --keyfile sample_keys.bin + +.. note:: Encryption key of the following format ``/keys/sample_keys.bin`` is created. +.. note:: This newly created file having encryption keys in ``keys/`` directory is compatible with NVS key-partition structure. Refer to :ref:`nvs_key_partition` for more details. + +- Encrypt by providing the encryption keys as input binary file:: + + python nvs_partition_gen.py encrypt sample_singlepage_blob.csv sample_encr.bin 0x3000 --inputkey sample_keys.bin + +To decrypt encrypted NVS partition: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + **Usage**:: + + python nvs_partition_gen.py decrypt [-h] [--outdir OUTDIR] input key output + + Positional Arguments: + +--------------+----------------------------------------------------------------------+ + | Parameter | Description | + +==============+======================================================================+ + | input | Path to encrypted NVS partition file to parse | + +--------------+----------------------------------------------------------------------+ + | key | Path to file having keys for decryption | + +--------------+----------------------------------------------------------------------+ + | output | Path to output decrypted binary file | + +--------------+----------------------------------------------------------------------+ + + + Optional Arguments: + +---------------------+--------------------------------------------------------------------+ + | Parameter | Description | + +=====================+====================================================================+ + | -h, --help | show this help message and exit | + +---------------------+--------------------------------------------------------------------+ + | --outdir OUTDIR | Output directory to store files created | + | | (Default: current directory) | + +---------------------+--------------------------------------------------------------------+ + + +You can run the utility to decrypt encrypted NVS partition using the command below:: + + python nvs_partition_gen.py decrypt sample_encr.bin sample_keys.bin sample_decr.bin + +You can also provide the format version number: + - Multipage Blob Support Disabled (Version 1) + - Multipage Blob Support Enabled (Version 2) + + +Multipage Blob Support Disabled (Version 1): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can run the utility in this format by setting the version parameter to 1, as shown below. A sample CSV file is provided with the utility:: - python nvs_partition_gen.py --input sample_singlepage_blob.csv --output partition_single_page.bin --size 0x3000 --version v1 + python nvs_partition_gen.py generate sample_singlepage_blob.csv sample.bin 0x3000 --version 1 + + +Multipage Blob Support Enabled (Version 2): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can run the utility in this format by setting the version parameter to 2, as shown below. +A sample CSV file is provided with the utility:: + + python nvs_partition_gen.py generate sample_multipage_blob.csv sample.bin 0x4000 --version 2 .. note:: *Minimum NVS Partition Size needed is 0x3000 bytes.* .. note:: *When flashing the binary onto the device, make sure it is consistent with the application's sdkconfig.* + Caveats ------- - Utility does not check for duplicate keys and will write data pertaining to both keys. You need to make sure that the keys are distinct. diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index e2cefa4d9..68bf657e3 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -33,6 +33,7 @@ import zlib import codecs import datetime import distutils.dir_util + try: from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend @@ -42,8 +43,18 @@ except ImportError: 'setting up the required packages.') raise -VERSION1_PRINT = "v1 - Multipage Blob Support Disabled" -VERSION2_PRINT = "v2 - Multipage Blob Support Enabled" +VERSION1_PRINT = "V1 - Multipage Blob Support Disabled" +VERSION2_PRINT = "V2 - Multipage Blob Support Enabled" + + +def reverse_hexbytes(addr_tmp): + addr = [] + reversed_bytes = "" + for i in range(0, len(addr_tmp), 2): + addr.append(addr_tmp[i:i + 2]) + reversed_bytes = "".join(reversed(addr)) + + return reversed_bytes """ Class for standard NVS page structure """ @@ -81,20 +92,16 @@ class Page(object): VERSION1 = 0xFF VERSION2 = 0xFE - def __init__(self, page_num, is_rsrv_page=False): + def __init__(self, page_num, version, is_rsrv_page=False): self.entry_num = 0 - self.is_encrypt = False - self.encr_key = None self.bitmap_array = array.array('B') - self.version = Page.VERSION2 + self.version = version self.page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"] if not is_rsrv_page: self.bitmap_array = self.create_bitmap_array() - self.set_header(page_num) - - def set_header(self, page_num): - global page_header + self.set_header(page_num, version) + def set_header(self, page_num, version): # set page state to active page_header = bytearray(b'\xff') * 32 page_state_active_seq = Page.ACTIVE @@ -141,30 +148,22 @@ class Page(object): return encrypted_data - def reverse_hexbytes(self, addr_tmp): - addr = [] - reversed_bytes = "" - for i in range(0, len(addr_tmp), 2): - addr.append(addr_tmp[i:i + 2]) - reversed_bytes = "".join(reversed(addr)) - - return reversed_bytes - def encrypt_data(self, data_input, no_of_entries, nvs_obj): # Set values needed for encryption and encrypt data byte wise encr_data_to_write = bytearray() data_len_needed = 64 # in hex tweak_len_needed = 32 # in hex + key_len_needed = 64 init_tweak_val = '0' init_data_val = 'f' tweak_tmp = '' encr_key_input = None # Extract encryption key and tweak key from given key input - if len(self.encr_key) == key_len_needed: - encr_key_input = self.encr_key + if len(nvs_obj.encr_key) == key_len_needed: + encr_key_input = nvs_obj.encr_key else: - encr_key_input = codecs.decode(self.encr_key, 'hex') + encr_key_input = codecs.decode(nvs_obj.encr_key, 'hex') rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET @@ -187,12 +186,10 @@ class Page(object): if addr_len > 2: if not addr_len % 2: addr_tmp = addr - tweak_tmp = self.reverse_hexbytes(addr_tmp) - tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) else: addr_tmp = init_tweak_val + addr - tweak_tmp = self.reverse_hexbytes(addr_tmp) - tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) + tweak_tmp = reverse_hexbytes(addr_tmp) + tweak_val = tweak_tmp + (init_tweak_val * (tweak_len_needed - (len(tweak_tmp)))) else: tweak_val = addr + (init_tweak_val * (tweak_len_needed - len(addr))) @@ -214,7 +211,7 @@ class Page(object): def write_entry_to_buf(self, data, entrycount,nvs_obj): encr_data = bytearray() - if self.is_encrypt: + if nvs_obj.encrypt: encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj) encr_data[0:len(encr_data_ret)] = encr_data_ret data = encr_data @@ -360,11 +357,11 @@ class Page(object): datalen = len(data) if datalen > Page.PAGE_PARAMS["max_old_blob_size"]: - if version == Page.VERSION1: - raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key)) + if self.version == Page.VERSION1: + raise InputError(" Input File: Size exceeds max allowed length `%s` bytes for key `%s`." % (Page.PAGE_PARAMS["max_old_blob_size"], key)) else: if encoding == "string": - raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key)) + raise InputError(" Input File: Size exceeds max allowed length `%s` bytes for key `%s`." % (Page.PAGE_PARAMS["max_old_blob_size"], key)) # Calculate no. of entries data will require rounded_size = (datalen + 31) & ~31 @@ -375,7 +372,7 @@ class Page(object): if self.entry_num >= Page.PAGE_PARAMS["max_entries"]: raise PageFullError() elif (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]: - if not (version == Page.VERSION2 and encoding in ["hex2bin", "binary", "base64"]): + if not (self.version == Page.VERSION2 and encoding in ["hex2bin", "binary", "base64"]): raise PageFullError() # Entry header @@ -383,7 +380,7 @@ class Page(object): # Set Namespace Index entry_struct[0] = ns_index # Set Span - if version == Page.VERSION2: + if self.version == Page.VERSION2: if encoding == "string": entry_struct[2] = data_entry_count + 1 # Set Chunk Index @@ -403,7 +400,7 @@ class Page(object): elif encoding in ["hex2bin", "binary", "base64"]: entry_struct[1] = Page.BLOB - if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]): + if self.version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]): entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data, datalen,total_entry_count, encoding, nvs_obj) else: @@ -465,13 +462,18 @@ Binary can later be flashed onto device via a flashing utility. class NVS(object): - def __init__(self, fout, input_size): + def __init__(self, fout, input_size, version, encrypt=False, key_input=None): self.size = input_size + self.encrypt = encrypt + self.encr_key = None self.namespace_idx = 0 self.page_num = -1 self.pages = [] - self.cur_page = self.create_new_page() + self.version = version self.fout = fout + if self.encrypt: + self.encr_key = key_input + self.cur_page = self.create_new_page(version) def __enter__(self): return self @@ -487,32 +489,26 @@ class NVS(object): # Creating the last reserved page self.create_new_page(is_rsrv_page=True) break - result = self.get_binary_data() - if version == Page.VERSION1: - print("Version: ", VERSION1_PRINT) - else: - print("Version: ", VERSION2_PRINT) self.fout.write(result) - def create_new_page(self, is_rsrv_page=False): + def create_new_page(self, version=None, is_rsrv_page=False): # Set previous page state to FULL before creating new page if self.pages: curr_page_state = struct.unpack('/ + :param outdir: Target output dir to store files + :param filepath: Path of target file + ''' + bin_ext = '.bin' + # Expand if tilde(~) provided in path + outdir = os.path.expanduser(outdir) - :param input_filename: Name of input file containing data - :param output_filename: Name of output file to store generated binary - :param input_part_size: Size of partition in bytes (must be multiple of 4096) - :param is_key_gen: Enable encryption key generation in encryption mode - :param encrypt_mode: Enable/Disable encryption mode - :param key_file: Input file having encryption keys in encryption mode - :param version_no: Format Version number - :return: None - """ + if filepath: + key_file_name, ext = os.path.splitext(filepath) + if not ext: + filepath = key_file_name + bin_ext + elif bin_ext not in ext: + sys.exit('Error: `%s`. Only `%s` extension allowed.' % (filepath, bin_ext)) - global key_input, key_len_needed - encr_key_bin_file = None - encr_keys_dir = None - backslash = ['/','\\'] + # Create dir if does not exist + if not (os.path.isdir(outdir)): + distutils.dir_util.mkpath(outdir) - key_len_needed = 64 - key_input = bytearray() + filedir, filename = os.path.split(filepath) + filedir = os.path.join(outdir,filedir,'') + if filedir and not os.path.isdir(filedir): + distutils.dir_util.mkpath(filedir) - if key_gen: - key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip() - elif key_file: - with open(key_file, 'rb') as key_f: - key_input = key_f.read(64) + if os.path.isabs(filepath): + if not outdir == os.getcwd(): + print("\nWarning: `%s` \n\t==> absolute path given so outdir is ignored for this file." % filepath) + # Set to empty as outdir is ignored here + outdir = '' - if all(arg is not None for arg in [input_filename, output_filename, input_size]): - if not os.path.isabs(output_filename) and not any(ch in output_filename for ch in backslash): - output_filename = os.path.join(output_dir, '') + output_filename - input_file = open(input_filename, 'rt', encoding='utf8') - output_file = open(output_filename, 'wb') + # Set full path - outdir + filename + filepath = os.path.join(outdir, '') + filepath - with nvs_open(output_file, input_size) as nvs_obj: - reader = csv.DictReader(filter(lambda row: row[0] != '#',input_file), delimiter=',') - for row in reader: - try: - write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) - except (InputError) as e: - print(e) - input_file.close() - output_file.close() - sys.exit(-2) + return outdir, filepath - input_file.close() - output_file.close() - print("NVS binary created: " + output_filename) +def encrypt(args): + ''' + Generate encrypted NVS Partition + :param args: Command line arguments given + ''' + key = None + bin_ext = '.bin' - if key_gen: - keys_page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"] - key_bytes = bytearray() - if len(key_input) == key_len_needed: - key_bytes = key_input + check_size(args.size) + if (args.keygen is False) and (not args.inputkey): + sys.exit("Error. --keygen or --inputkey argument needed.") + elif args.keygen and args.inputkey: + sys.exit("Error. --keygen and --inputkey both are not allowed.") + elif not args.keygen and args.keyfile: + print("\nWarning:","--inputkey argument is given. --keyfile argument will be ignored...") + + if args.inputkey: + # Check if key file has .bin extension + filename, ext = os.path.splitext(args.inputkey) + if bin_ext not in ext: + sys.exit('Error: `%s`. Only `%s` extension allowed.' % (args.inputkey, bin_ext)) + key = bytearray() + with open(args.inputkey, 'rb') as key_f: + key = key_f.read(64) + + # Generate encrypted NVS Partition + generate(args, is_encr_enabled=True, encr_key=key) + + +def decrypt_data(data_input, decr_key, page_num, entry_no, entry_size): + ''' + Decrypt NVS data entry + ''' + page_max_size = 4096 + first_entry_offset = 64 + init_tweak_val = '0' + tweak_len_needed = 32 # in hex + tweak_tmp = '' + + data_input = binascii.hexlify(data_input) + rel_addr = page_num * page_max_size + first_entry_offset + + # Set tweak value + offset = entry_no * entry_size + addr = hex(rel_addr + offset)[2:] + addr_len = len(addr) + if addr_len > 2: + if not addr_len % 2: + addr_tmp = addr else: - key_bytes = codecs.decode(key_input, 'hex') - key_len = len(key_bytes) - keys_page_buf[0:key_len] = key_bytes - crc_data = keys_page_buf[0:key_len] - crc_data = bytes(crc_data) - crc = zlib.crc32(crc_data, 0xFFFFFFFF) - struct.pack_into('", args.output) + + +def generate_key(args): + ''' + Generate encryption keys + :param args: Command line arguments given + ''' + page_max_size = 4096 + keys_dir = 'keys' + output_keyfile = None + bin_ext = '.bin' + + if not args.keyfile: + timestamp = datetime.datetime.now().strftime('%m-%d_%H-%M') + args.keyfile = "keys-" + timestamp + bin_ext + + keys_outdir = os.path.join(args.outdir,keys_dir, '') + # Create keys/ dir in if does not exist + if not (os.path.isdir(keys_outdir)): + distutils.dir_util.mkpath(keys_outdir) + keys_outdir, output_keyfile = set_target_filepath(keys_outdir, args.keyfile) + + key = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip() + encr_key_bytes = codecs.decode(key, 'hex') + key_len = len(encr_key_bytes) + + keys_buf = bytearray(b'\xff') * page_max_size + keys_buf[0:key_len] = encr_key_bytes + crc_data = keys_buf[0:key_len] + crc_data = bytes(crc_data) + crc = zlib.crc32(crc_data, 0xFFFFFFFF) + struct.pack_into(' ", output_keyfile) + + return key + + +def generate(args, is_encr_enabled=False, encr_key=None): + ''' + Generate NVS Partition + :param args: Command line arguments given + :param is_encr_enabled: Encryption enabled/disabled + :param encr_key: Key to encrypt NVS partition + ''' + is_dir_new = False + bin_ext = '.bin' + + input_size = check_size(args.size) + if args.version == 1: + args.version = Page.VERSION1 + elif args.version == 2: + args.version = Page.VERSION2 + + # Check if key file has .bin extension + filename, ext = os.path.splitext(args.output) + if bin_ext not in ext: + sys.exit('Error: `%s`. Only `.bin` extension allowed.' % args.output) + args.outdir, args.output = set_target_filepath(args.outdir, args.output) + + if is_encr_enabled and not encr_key: + encr_key = generate_key(args) + + input_file = open(args.input, 'rt', encoding='utf8') + output_file = open(args.output, 'wb') + + with open(args.input, 'rt', encoding='utf8') as input_file,\ + open(args.output, 'wb') as output_file,\ + nvs_open(output_file, input_size, args.version, is_encrypt=is_encr_enabled, key=encr_key) as nvs_obj: + # Comments are skipped + reader = csv.DictReader(filter(lambda row: row[0] != '#',input_file), delimiter=',') + if nvs_obj.version == Page.VERSION1: + version_set = VERSION1_PRINT else: - if encr_key_prefix: - encr_key_bin_file = encr_keys_dir + encr_key_prefix + "-keys" + ".bin" - else: - encr_key_bin_file = encr_keys_dir + "encryption_keys_" + timestamp + ".bin" + version_set = VERSION2_PRINT + print("\nCreating NVS binary with version:", version_set) + for row in reader: + try: + write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) + except InputError as e: + print(e) + filedir, filename = os.path.split(args.output) + if filename: + print("\nWarning: NVS binary not created...") + os.remove(args.output) + if is_dir_new and not filedir == os.getcwd(): + print("\nWarning: Output dir not created...") + os.rmdir(filedir) + sys.exit(-2) - with open(encr_key_bin_file,'wb') as output_keys_file: - output_keys_file.write(keys_page_buf) - - print("Encryption keys binary created: " + encr_key_bin_file) + print("\nCreated NVS binary: ===>", args.output) def main(): - parser = argparse.ArgumentParser(description="ESP32 NVS partition generation utility") - nvs_part_gen_group = parser.add_argument_group('To generate NVS partition') - nvs_part_gen_group.add_argument("--input", - help="Path to CSV file to parse.", - default=None) - - nvs_part_gen_group.add_argument("--output", - help='Path to output converted binary file.', - default=None) - - nvs_part_gen_group.add_argument("--size", - help='Size of NVS Partition in bytes (must be multiple of 4096)') - - nvs_part_gen_group.add_argument("--version", - help='Set version. Default: v2', - choices=['v1','v2'], - default='v2', - type=str.lower) - - keygen_action_key = nvs_part_gen_group.add_argument("--keygen", - help='Generate keys for encryption.', - choices=['true','false'], - default='false', - type=str.lower) - - nvs_part_gen_group.add_argument("--encrypt", - help='Set encryption mode. Default: false', - choices=['true','false'], - default='false', - type=str.lower) - - keygen_action_file = nvs_part_gen_group.add_argument("--keyfile", - help='File having key for encryption (Applicable only if encryption mode is true).', - default=None) - - keygen_action_dir = nvs_part_gen_group.add_argument('--outdir', - dest='outdir', - default=os.getcwd(), - help='the output directory to store the files created\ - (Default: current directory)') - - key_gen_group = parser.add_argument_group('To generate encryption keys') - key_gen_group._group_actions.append(keygen_action_key) - key_gen_group._group_actions.append(keygen_action_file) - key_gen_group._group_actions.append(keygen_action_dir) + parser = argparse.ArgumentParser(description="\nESP NVS partition generation utility", formatter_class=argparse.RawTextHelpFormatter) + subparser = parser.add_subparsers(title='Commands', + dest='command', + help='\nRun nvs_partition_gen.py {command} -h for additional help\n\n') + parser_gen = subparser.add_parser('generate', + help='Generate NVS partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_gen.set_defaults(func=generate) + parser_gen.add_argument('input', + default=None, + help='Path to CSV file to parse') + parser_gen.add_argument('output', + default=None, + help='Path to output NVS binary file') + parser_gen.add_argument('size', + default=None, + help='Size of NVS partition in bytes\ + \n(must be multiple of 4096)') + parser_gen.add_argument('--version', + choices=[1,2], + default=2, + type=int, + help='''Set multipage blob version.\ + \nVersion 1 - Multipage blob support disabled.\ + \nVersion 2 - Multipage blob support enabled.\ + \nDefault: Version 2''') + parser_gen.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created\ + \n(Default: current directory)') + parser_gen_key = subparser.add_parser('generate-key', + help='Generate keys for encryption', + formatter_class=argparse.RawTextHelpFormatter) + parser_gen_key.set_defaults(func=generate_key) + parser_gen_key.add_argument('--keyfile', + default=None, + help='Path to output encryption keys file') + parser_gen_key.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') + parser_encr = subparser.add_parser('encrypt', + help='Generate NVS encrypted partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_encr.set_defaults(func=encrypt) + parser_encr.add_argument('input', + default=None, + help='Path to CSV file to parse') + parser_encr.add_argument('output', + default=None, + help='Path to output NVS binary file') + parser_encr.add_argument('size', + default=None, + help='Size of NVS partition in bytes\ + \n(must be multiple of 4096)') + parser_encr.add_argument('--version', + choices=[1,2], + default=2, + type=int, + help='''Set multipage blob version.\ + \nVersion 1 - Multipage blob support disabled.\ + \nVersion 2 - Multipage blob support enabled.\ + \nDefault: Version 2''') + parser_encr.add_argument('--keygen', + action="store_true", + default=False, + help='Generates key for encrypting NVS partition') + parser_encr.add_argument('--keyfile', + default=None, + help='Path to output encryption keys file') + parser_encr.add_argument('--inputkey', + default=None, + help='File having key for encrypting NVS partition') + parser_encr.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') + parser_decr = subparser.add_parser('decrypt', + help='Decrypt NVS encrypted partition', + formatter_class=argparse.RawTextHelpFormatter) + parser_decr.set_defaults(func=decrypt) + parser_decr.add_argument('input', + default=None, + help='Path to encrypted NVS partition file to parse') + parser_decr.add_argument('key', + default=None, + help='Path to file having keys for decryption') + parser_decr.add_argument('output', + default=None, + help='Path to output decrypted binary file') + parser_decr.add_argument('--outdir', + default=os.getcwd(), + help='Output directory to store files created.\ + \n(Default: current directory)') args = parser.parse_args() - input_filename = args.input - output_filename = args.output - part_size = args.size - version_no = args.version - is_key_gen = args.keygen - is_encrypt_data = args.encrypt - key_file = args.keyfile - output_dir_path = args.outdir - encr_keys_prefix = None - print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory.\ - \nTo generate encryption keys --keygen argument is mandatory." - print_encrypt_arg_str = "Missing parameter. Enter --keyfile or --keygen." - - check_input_args(input_filename,output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no, - print_arg_str, print_encrypt_arg_str, output_dir_path) - nvs_part_gen(input_filename, output_filename, part_size, is_key_gen, is_encrypt_data, key_file, - encr_keys_prefix, version_no, output_dir_path) + args.func(args) if __name__ == "__main__": diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp index bbd4f8ee6..a92a20bc6 100644 --- a/components/nvs_flash/test_nvs_host/test_nvs.cpp +++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp @@ -2375,14 +2375,14 @@ TEST_CASE("check and read data from partition generated via partition generation if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", - "--input", + "generate", "../nvs_partition_generator/sample_singlepage_blob.csv", - "--output", - "../nvs_partition_generator/partition_single_page.bin", - "--size", + "partition_single_page.bin", "0x3000", "--version", - "v1",NULL)); + "1", + "--outdir", + "../nvs_partition_generator",NULL)); } else { CHECK(childpid > 0); int status; @@ -2429,14 +2429,14 @@ TEST_CASE("check and read data from partition generated via partition generation if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", - "--input", + "generate", "../nvs_partition_generator/sample_multipage_blob.csv", - "--output", - "../nvs_partition_generator/partition_multipage_blob.bin", - "--size", + "partition_multipage_blob.bin", "0x4000", "--version", - "v2",NULL)); + "2", + "--outdir", + "../nvs_partition_generator",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0); @@ -2461,6 +2461,7 @@ TEST_CASE("check and read data from partition generated via partition generation } } +#if false TEST_CASE("check and read data from partition generated via manufacturing utility with multipage blob support disabled", "[mfg_gen]") { int childpid = fork(); @@ -2631,6 +2632,7 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit } } +#endif #if CONFIG_NVS_ENCRYPTION TEST_CASE("check underlying xts code for 32-byte size sector encryption", "[nvs]") @@ -2790,16 +2792,14 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", - "--input", + "encrypt", "../nvs_partition_generator/sample_multipage_blob.csv", - "--output", - "../nvs_partition_generator/partition_encrypted.bin", - "--size", + "partition_encrypted.bin", "0x4000", - "--encrypt", - "True", - "--keyfile", - "../nvs_partition_generator/testdata/sample_encryption_keys.bin",NULL)); + "--inputkey", + "../nvs_partition_generator/testdata/sample_encryption_keys.bin", + "--outdir", + "../nvs_partition_generator",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0); @@ -2831,7 +2831,6 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } - TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keygen", "[nvs_part_gen]") { int childpid = fork(); @@ -2852,7 +2851,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena if (childpid == 0) { exit(execlp("rm", " rm", "-rf", - "keys",NULL)); + "../nvs_partition_generator/keys",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0); @@ -2862,16 +2861,13 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", - "--input", + "encrypt", "../nvs_partition_generator/sample_multipage_blob.csv", - "--output", - "../nvs_partition_generator/partition_encrypted_using_keygen.bin", - "--size", + "partition_encrypted_using_keygen.bin", "0x4000", - "--encrypt", - "True", "--keygen", - "true",NULL)); + "--outdir", + "../nvs_partition_generator",NULL)); } else { CHECK(childpid > 0); @@ -2889,7 +2885,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena char *files; char *file_ext; - dir = opendir("keys"); + dir = opendir("../nvs_partition_generator/keys"); while ((file = readdir(dir)) != NULL) { filename = file->d_name; @@ -2904,7 +2900,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } } - std::string encr_file = std::string("keys/") + std::string(filename); + std::string encr_file = std::string("../nvs_partition_generator/keys/") + std::string(filename); SpiFlashEmulator emu("../nvs_partition_generator/partition_encrypted_using_keygen.bin"); char buffer[64]; @@ -2927,7 +2923,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } -TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using keyfile", "[nvs_part_gen]") +TEST_CASE("test nvs apis for nvs partition generator utility with encryption enabled using inputkey", "[nvs_part_gen]") { int childpid = fork(); int status; @@ -2938,7 +2934,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena char *files; char *file_ext; - dir = opendir("keys"); + dir = opendir("../nvs_partition_generator/keys"); while ((file = readdir(dir)) != NULL) { filename = file->d_name; @@ -2953,21 +2949,19 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } } - std::string encr_file = std::string("keys/") + std::string(filename); + std::string encr_file = std::string("../nvs_partition_generator/keys/") + std::string(filename); if (childpid == 0) { exit(execlp("python", "python", "../nvs_partition_generator/nvs_partition_gen.py", - "--input", + "encrypt", "../nvs_partition_generator/sample_multipage_blob.csv", - "--output", - "../nvs_partition_generator/partition_encrypted_using_keyfile.bin", - "--size", + "partition_encrypted_using_keyfile.bin", "0x4000", - "--encrypt", - "True", - "--keyfile", - encr_file.c_str(),NULL)); + "--inputkey", + encr_file.c_str(), + "--outdir", + "../nvs_partition_generator",NULL)); } else { CHECK(childpid > 0); @@ -2999,7 +2993,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena if (childpid == 0) { exit(execlp("rm", " rm", "-rf", - "keys",NULL)); + "../nvs_partition_generator/keys",NULL)); } else { CHECK(childpid > 0); waitpid(childpid, &status, 0); @@ -3020,6 +3014,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena } +#if false TEST_CASE("check and read data from partition generated via manufacturing utility with encryption enabled using sample keyfile", "[mfg_gen]") { int childpid = fork(); @@ -3260,6 +3255,7 @@ TEST_CASE("check and read data from partition generated via manufacturing utilit } #endif +#endif /* Add new tests above */ /* This test has to be the final one */