From 8369bd244b1b878486a713f6137c7c3df5185e68 Mon Sep 17 00:00:00 2001 From: Piyush Shah Date: Thu, 14 Jun 2018 16:07:26 +0530 Subject: [PATCH 1/2] nvs_partition_generator: Modifications to enable using the utility as a Python library And also use directly as an executable --- .../nvs_partition_gen.py | 45 ++++++++++++------- tools/ci/executable-list.txt | 1 + 2 files changed, 30 insertions(+), 16 deletions(-) mode change 100644 => 100755 components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py old mode 100644 new mode 100755 index ecbe9b5db..b895fc1ca --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -333,29 +333,42 @@ def nvs_close(nvs_instance): """ nvs_instance.__exit__(None, None, None) -def main(): - parser = argparse.ArgumentParser(description="ESP32 NVS partition generation utility") - parser.add_argument( - "input", - help="Path to CSV file to parse. Will use stdin if omitted", - type=argparse.FileType('rb'), - default=sys.stdin) +def nvs_part_gen(input_filename=None, output_filename=None): + input_file = open(input_filename, 'rb') + output_file = open(output_filename, 'wb') - parser.add_argument( - "output", - help='Path to output converted binary file. Will use stdout if omitted', - type=argparse.FileType('wb'), - default=sys.stdout) - - args = parser.parse_args() - with nvs_open(args.output) as nvs_obj: - reader = csv.DictReader(args.input, delimiter=',') + with nvs_open(output_file) as nvs_obj: + reader = csv.DictReader(input_file, delimiter=',') for row in reader: try: write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"]) except InputError as e: print(e) + input_file.close() + output_file.close() exit(-2) + input_file.close() + output_file.close() + +def main(): + parser = argparse.ArgumentParser(description="ESP32 NVS partition generation utility") + parser.add_argument( + "input", + help="Path to CSV file to parse. Will use stdin if omitted", + default=sys.stdin) + + parser.add_argument( + "output", + help='Path to output converted binary file. Will use stdout if omitted', + default=sys.stdout) + + args = parser.parse_args() + input_filename = args.input + output_filename = args.output + nvs_part_gen(input_filename, output_filename) + + + if __name__ == "__main__": main() diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 6fc884a00..3fe4158f9 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -7,6 +7,7 @@ components/partition_table/gen_esp32part.py components/partition_table/parttool.py components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py components/ulp/esp32ulp_mapgen.py +components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py docs/check_doc_warnings.sh docs/check_lang_folder_sync.sh docs/gen-kconfig-doc.py From e7ddd47716faa26a95f8df50c857205f82c8c9c1 Mon Sep 17 00:00:00 2001 From: Piyush Shah Date: Thu, 14 Jun 2018 16:27:37 +0530 Subject: [PATCH 2/2] nvs_partition_generator: Add support for base64 representation of Binary data --- components/nvs_flash/nvs_partition_generator/README.rst | 4 ++-- .../nvs_partition_generator/nvs_partition_gen.py | 9 ++++++--- components/nvs_flash/nvs_partition_generator/sample.csv | 2 ++ .../nvs_partition_generator/testdata/sample.base64 | 1 + components/nvs_flash/test_nvs_host/test_nvs.cpp | 3 +++ 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 components/nvs_flash/nvs_partition_generator/testdata/sample.base64 diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst index 5b62ebc39..f324f016f 100644 --- a/components/nvs_flash/nvs_partition_generator/README.rst +++ b/components/nvs_flash/nvs_partition_generator/README.rst @@ -19,9 +19,9 @@ Type Supported values are ``file``, ``data`` and ``namespace``. Encoding - Supported values are: ``u8``, ``i8``, ``u16``, ``u32``, ``i32``, ``string``, ``hex2bin`` and ``binary``. This specifies how actual data values are encoded in the resultant binary file. Difference between ``string`` and ``binary`` encoding is that ``string`` data is terminated with a NULL character, whereas ``binary`` data is not. + Supported values are: ``u8``, ``i8``, ``u16``, ``u32``, ``i32``, ``string``, ``hex2bin``, ``base64`` and ``binary``. This specifies how actual data values are encoded in the resultant binary file. Difference between ``string`` and ``binary`` encoding is that ``string`` data is terminated with a NULL character, whereas ``binary`` data is not. - .. note:: For ``file`` type, only ``hex2bin``, ``string`` and ``binary`` is supported as of now. + .. note:: For ``file`` type, only ``hex2bin``, ``base64``, ``string`` and ``binary`` is supported as of now. Value Data value. diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index b895fc1ca..6224d3b94 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -132,7 +132,7 @@ class Page(object): # set Type if encoding == "string": entry_struct[1] = Page.SZ - elif encoding == "hex2bin" or encoding == "binary": + elif encoding in ["hex2bin", "binary", "base64"]: entry_struct[1] = Page.BLOB # compute CRC of data @@ -248,11 +248,14 @@ class NVS(object): raise InputError("%s: Invalid data length. Should be multiple of 2." % key) value = binascii.a2b_hex(value) + if encoding == "base64": + value = binascii.a2b_base64(value) + if encoding == "string": value += '\0' encoding = encoding.lower() - varlen_encodings = ["string", "binary", "hex2bin"] + varlen_encodings = ["string", "binary", "hex2bin", "base64"] primitive_encodings = ["u8", "i8", "u16", "u32", "i32"] if encoding in varlen_encodings: try: @@ -308,7 +311,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value): :param nvs_instance: Instance of an NVS class returned by nvs_open() :param key: Key of the data :param datatype: Data type. Valid values are "file", "data" and "namespace" - :param encoding: Data encoding. Valid values are "u8", "i8", "u16", "u32", "i32", "string", "binary" and "hex2bin" + :param encoding: Data encoding. Valid values are "u8", "i8", "u16", "u32", "i32", "string", "binary", "hex2bin" and "base64" :param value: Data value in ascii encoded string format for "data" datatype and filepath for "file" datatype :return: None """ diff --git a/components/nvs_flash/nvs_partition_generator/sample.csv b/components/nvs_flash/nvs_partition_generator/sample.csv index ccade4dbb..bc70aa14c 100644 --- a/components/nvs_flash/nvs_partition_generator/sample.csv +++ b/components/nvs_flash/nvs_partition_generator/sample.csv @@ -7,6 +7,8 @@ dummyU32Key,data,u32,4294967295 dummyI32Key,data,i32,-2147483648 dummyStringKey,data,string,0A:0B:0C:0D:0E:0F dummyHex2BinKey,data,hex2bin,010203abcdef +dummyBase64Key,data,base64,MTIzYWJj hexFileKey,file,hex2bin,testdata/sample.hex +base64FileKey,file,base64,testdata/sample.base64 stringFileKey,file,string,testdata/sample.txt binFileKey,file,binary,testdata/sample.bin diff --git a/components/nvs_flash/nvs_partition_generator/testdata/sample.base64 b/components/nvs_flash/nvs_partition_generator/testdata/sample.base64 new file mode 100644 index 000000000..0c16c99a2 --- /dev/null +++ b/components/nvs_flash/nvs_partition_generator/testdata/sample.base64 @@ -0,0 +1 @@ +AQIDBAUGBwgJq83v diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp index 658fdb53f..22d30d055 100644 --- a/components/nvs_flash/test_nvs_host/test_nvs.cpp +++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp @@ -1548,6 +1548,9 @@ TEST_CASE("read data from partition generated via partition generation utility", uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); CHECK(memcmp(buf, hexdata, buflen) == 0); + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); } TEST_CASE("dump all performance data", "[nvs]")