Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'update/mfg_gen' into 'master'

Browse source 
Update mfg utility and nvs partition utility to make Python2 and Python3 compatible

See merge request idf/esp-idf!3243
This commit is contained in:
Angus Gratton 2018-10-17 15:23:27 +08:00
parent 95e4bdf030 75a4007675
commit dad3531f96
5 changed files with 153 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
components/nvs_flash/nvs_partition_generator/README.rst
View file

@ -116,6 +116,7 @@ A sample CSV file is provided with the utility::
python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1 python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| Arguments | Description | | Arguments | Description |
+========================+==============================================================================================+ +========================+==============================================================================================+
@ -123,7 +124,7 @@ A sample CSV file is provided with the utility::
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| output | Path to output converted binary file. Will use stdout if omitted | | output | Path to output converted binary file. Will use stdout if omitted |
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| size | Size of NVS Partition in KB. Eg. 12KB | | size | Size of NVS Partition in bytes (must be multiple of 4096) |
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| --version {v1,v2} | Set version. Default: v2 | | --version {v1,v2} | Set version. Default: v2 |
+-------------------------------+---------------------------------------------------------------------------------------+ +-------------------------------+---------------------------------------------------------------------------------------+

138
components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
View file

@ -17,8 +17,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
from __future__ import division
from __future__ import print_function from __future__ import division, print_function
from builtins import int, range
from io import open
import sys import sys
import argparse import argparse
import binascii import binascii
@ -28,6 +30,7 @@ import os
import array import array
import csv import csv
import zlib import zlib
import codecs
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
@ -79,20 +82,20 @@ class Page(object):
global page_header global page_header
# set page state to active # set page state to active
page_header= bytearray(b'\xff')*32 page_header= bytearray(b'\xff') *32
page_state_active_seq = Page.ACTIVE page_state_active_seq = Page.ACTIVE
page_header[0:4] = struct.pack('<I', page_state_active_seq) struct.pack_into('<I', page_header, 0, page_state_active_seq)
# set page sequence number # set page sequence number
page_header[4:8] = struct.pack('<I', page_num) struct.pack_into('<I', page_header, 4, page_num)
# set version # set version
if version == Page.VERSION2: if version == Page.VERSION2:
page_header[8] = Page.VERSION2 page_header[8] = Page.VERSION2
elif version == Page.VERSION1: elif version == Page.VERSION1:
page_header[8] = Page.VERSION1 page_header[8] = Page.VERSION1
# set header's CRC # set header's CRC
crc_data = page_header[4:28] crc_data = bytes(page_header[4:28])
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF) crc = zlib.crc32(crc_data, 0xFFFFFFFF)
page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF) struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
self.page_buf[0:len(page_header)] = page_header self.page_buf[0:len(page_header)] = page_header
@ -118,8 +121,9 @@ class Page(object):
def encrypt_entry(self, data_arr, tweak_arr, encr_key): def encrypt_entry(self, data_arr, tweak_arr, encr_key):
# Encrypt 32 bytes of data using AES-XTS encryption # Encrypt 32 bytes of data using AES-XTS encryption
backend = default_backend() backend = default_backend()
plain_text = data_arr.decode('hex') plain_text = codecs.decode(data_arr, 'hex')
tweak = tweak_arr.decode('hex') tweak = codecs.decode(tweak_arr, 'hex')
cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend) cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
encryptor = cipher.encryptor() encryptor = cipher.encryptor()
encrypted_data = encryptor.update(plain_text) encrypted_data = encryptor.update(plain_text)
@ -139,7 +143,7 @@ class Page(object):
def encrypt_data(self, data_input, no_of_entries, nvs_obj): def encrypt_data(self, data_input, no_of_entries, nvs_obj):
# Set values needed for encryption and encrypt data byte wise # Set values needed for encryption and encrypt data byte wise
encr_data_to_write = '' encr_data_to_write = bytearray()
data_len_needed = 64 #in hex data_len_needed = 64 #in hex
tweak_len_needed = 32 #in hex tweak_len_needed = 32 #in hex
init_tweak_val = '0' init_tweak_val = '0'
@ -147,17 +151,18 @@ class Page(object):
tweak_tmp = '' tweak_tmp = ''
encr_key_input = None encr_key_input = None
# Extract encryption key and tweak key from given key input # Extract encryption key and tweak key from given key input
encr_key_input = self.encr_key.decode('hex') encr_key_input = codecs.decode(self.encr_key, 'hex')
rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
if type(data_input) != bytearray: if not isinstance(data_input, bytearray):
byte_arr = bytearray('\xff') * 32 byte_arr = bytearray(b'\xff') * 32
byte_arr[0:len(data_input)] = data_input byte_arr[0:len(data_input)] = data_input
data_input = byte_arr data_input = byte_arr
data_input = binascii.hexlify(bytearray(data_input)) data_input = binascii.hexlify(data_input)
entry_no = self.entry_num entry_no = self.entry_num
start_idx = 0 start_idx = 0
@ -182,6 +187,9 @@ class Page(object):
# Encrypt data # Encrypt data
data_bytes = data_input[start_idx:end_idx] data_bytes = data_input[start_idx:end_idx]
if type(data_bytes) == bytes:
data_bytes = data_bytes.decode()
data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes))) data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input) encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
encr_data_to_write = encr_data_to_write + encr_data_ret encr_data_to_write = encr_data_to_write + encr_data_ret
@ -190,12 +198,12 @@ class Page(object):
end_idx = start_idx + 64 end_idx = start_idx + 64
entry_no += 1 entry_no += 1
return encr_data_to_write return encr_data_to_write
def write_entry_to_buf(self, data, entrycount,nvs_obj): def write_entry_to_buf(self, data, entrycount,nvs_obj):
encr_data = bytearray() encr_data = bytearray()
if self.is_encrypt: if self.is_encrypt:
encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj) encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
encr_data[0:len(encr_data_ret)] = encr_data_ret encr_data[0:len(encr_data_ret)] = encr_data_ret
@ -213,13 +221,15 @@ class Page(object):
def set_crc_header(self, entry_struct): def set_crc_header(self, entry_struct):
crc_data = bytearray(28) crc_data = bytearray(b'28')
crc_data[0:4] = entry_struct[0:4] crc_data[0:4] = entry_struct[0:4]
crc_data[4:28] = entry_struct[8:32] crc_data[4:28] = entry_struct[8:32]
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF) crc_data = bytes(crc_data)
entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF) crc = zlib.crc32(crc_data, 0xFFFFFFFF)
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
return entry_struct return entry_struct
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj): def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
chunk_start = 0 chunk_start = 0
chunk_count = 0 chunk_count = 0
@ -262,9 +272,10 @@ class Page(object):
data_chunk = data[offset:offset + chunk_size] data_chunk = data[offset:offset + chunk_size]
# Compute CRC of data chunk # Compute CRC of data chunk
entry_struct[24:26] = struct.pack('<H', chunk_size) struct.pack_into('<H', entry_struct, 24, chunk_size)
data_chunk = bytes(data_chunk)
crc = zlib.crc32(data_chunk, 0xFFFFFFFF) crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF) struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
# compute crc of entry header # compute crc of entry header
entry_struct = self.set_crc_header(entry_struct) entry_struct = self.set_crc_header(entry_struct)
@ -279,7 +290,7 @@ class Page(object):
if remaining_size or (tailroom - chunk_size) < Page.SINGLE_ENTRY_SIZE: if remaining_size or (tailroom - chunk_size) < Page.SINGLE_ENTRY_SIZE:
if page_header[0:4] != Page.FULL: if page_header[0:4] != Page.FULL:
page_state_full_seq = Page.FULL page_state_full_seq = Page.FULL
page_header[0:4] = struct.pack('<I', page_state_full_seq) struct.pack_into('<I', page_header, 0, page_state_full_seq)
nvs_obj.create_new_page() nvs_obj.create_new_page()
self = nvs_obj.cur_page self = nvs_obj.cur_page
@ -289,7 +300,7 @@ class Page(object):
# All chunks are stored, now store the index # All chunks are stored, now store the index
if not remaining_size: if not remaining_size:
# Initialise data field to 0xff # Initialise data field to 0xff
data_array = bytearray('\xff')*8 data_array = bytearray(b'\xff')*8
entry_struct[24:32] = data_array entry_struct[24:32] = data_array
# change type of data to BLOB_IDX # change type of data to BLOB_IDX
@ -302,7 +313,7 @@ class Page(object):
chunk_index = Page.CHUNK_ANY chunk_index = Page.CHUNK_ANY
entry_struct[3] = chunk_index entry_struct[3] = chunk_index
entry_struct[24:28] = struct.pack('<I', data_size) struct.pack_into('<I', entry_struct, 24, data_size)
entry_struct[28] = chunk_count entry_struct[28] = chunk_count
entry_struct[29] = chunk_start entry_struct[29] = chunk_start
@ -318,9 +329,11 @@ class Page(object):
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj): def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
# compute CRC of data # compute CRC of data
entry_struct[24:26] = struct.pack('<H', datalen) struct.pack_into('<H', entry_struct, 24, datalen)
if not type(data) == bytes:
data = data.encode()
crc = zlib.crc32(data, 0xFFFFFFFF) crc = zlib.crc32(data, 0xFFFFFFFF)
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF) struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
# compute crc of entry header # compute crc of entry header
entry_struct = self.set_crc_header(entry_struct) entry_struct = self.set_crc_header(entry_struct)
@ -359,7 +372,7 @@ class Page(object):
raise PageFullError() raise PageFullError()
# Entry header # Entry header
entry_struct = bytearray('\xff')*32 entry_struct = bytearray(b'\xff')*32
# Set Namespace Index # Set Namespace Index
entry_struct[0] = ns_index entry_struct[0] = ns_index
# Set Span # Set Span
@ -373,9 +386,9 @@ class Page(object):
entry_struct[2] = data_entry_count + 1 entry_struct[2] = data_entry_count + 1
# set key # set key
key_array = bytearray('\x00')*16 key_array = b'\x00' * 16
entry_struct[8:24] = key_array entry_struct[8:24] = key_array
entry_struct[8:8 + len(key)] = key entry_struct[8:8 + len(key)] = key.encode()
# set Type # set Type
if encoding == "string": if encoding == "string":
@ -397,39 +410,40 @@ class Page(object):
if self.entry_num >= Page.PAGE_PARAMS["max_entries"]: if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
raise PageFullError() raise PageFullError()
entry_struct = bytearray('\xff')*32 entry_struct = bytearray(b'\xff')*32
entry_struct[0] = ns_index # namespace index entry_struct[0] = ns_index # namespace index
entry_struct[2] = 0x01 # Span entry_struct[2] = 0x01 # Span
chunk_index = Page.CHUNK_ANY chunk_index = Page.CHUNK_ANY
entry_struct[3] = chunk_index entry_struct[3] = chunk_index
# write key # write key
key_array = bytearray('\x00')*16 key_array = b'\x00' *16
entry_struct[8:24] = key_array entry_struct[8:24] = key_array
entry_struct[8:8 + len(key)] = key entry_struct[8:8 + len(key)] = key.encode()
if encoding == "u8": if encoding == "u8":
entry_struct[1] = Page.U8 entry_struct[1] = Page.U8
entry_struct[24] = struct.pack('<B', data) struct.pack_into('<B', entry_struct, 24, data)
elif encoding == "i8": elif encoding == "i8":
entry_struct[1] = Page.I8 entry_struct[1] = Page.I8
entry_struct[24] = struct.pack('<b', data) struct.pack_into('<b', entry_struct, 24, data)
elif encoding == "u16": elif encoding == "u16":
entry_struct[1] = Page.U16 entry_struct[1] = Page.U16
entry_struct[24:26] = struct.pack('<H', data) struct.pack_into('<H', entry_struct, 24, data)
elif encoding == "u32": elif encoding == "u32":
entry_struct[1] = Page.U32 entry_struct[1] = Page.U32
entry_struct[24:28] = struct.pack('<I', data) struct.pack_into('<I', entry_struct, 24, data)
elif encoding == "i32": elif encoding == "i32":
entry_struct[1] = Page.I32 entry_struct[1] = Page.I32
entry_struct[24:28] = struct.pack('<i', data) struct.pack_into('<i', entry_struct, 24, data)
# Compute CRC # Compute CRC
crc_data = bytearray(28) crc_data = bytearray(b'28')
crc_data[0:4] = entry_struct[0:4] crc_data[0:4] = entry_struct[0:4]
crc_data[4:28] = entry_struct[8:32] crc_data[4:28] = entry_struct[8:32]
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF) crc_data = bytes(crc_data)
entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF) crc = zlib.crc32(crc_data, 0xFFFFFFFF)
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
# write to file # write to file
self.write_entry_to_buf(entry_struct, 1,nvs_obj) self.write_entry_to_buf(entry_struct, 1,nvs_obj)
@ -478,7 +492,8 @@ class NVS(object):
new_page = Page(self.page_num, is_rsrv_page) new_page = Page(self.page_num, is_rsrv_page)
new_page.version = version new_page.version = version
new_page.is_encrypt = is_encrypt_data new_page.is_encrypt = is_encrypt_data
new_page.encr_key = key_input if new_page.is_encrypt:
new_page.encr_key = key_input
self.pages.append(new_page) self.pages.append(new_page)
self.cur_page = new_page self.cur_page = new_page
return new_page return new_page
@ -494,7 +509,6 @@ class NVS(object):
except PageFullError: except PageFullError:
new_page = self.create_new_page() new_page = self.create_new_page()
new_page.write_primitive_data(key, self.namespace_idx, "u8", 0,self) new_page.write_primitive_data(key, self.namespace_idx, "u8", 0,self)
pass
""" """
Write key-value pair. Function accepts value in the form of ascii character and converts Write key-value pair. Function accepts value in the form of ascii character and converts
@ -512,6 +526,8 @@ class NVS(object):
value = binascii.a2b_base64(value) value = binascii.a2b_base64(value)
if encoding == "string": if encoding == "string":
if type(value) == bytes:
value = value.decode()
value += '\0' value += '\0'
encoding = encoding.lower() encoding = encoding.lower()
@ -523,15 +539,12 @@ class NVS(object):
except PageFullError: except PageFullError:
new_page = self.create_new_page() new_page = self.create_new_page()
new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self) new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
pass
elif encoding in primitive_encodings: elif encoding in primitive_encodings:
try: try:
self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self) self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
except PageFullError: except PageFullError:
new_page = self.create_new_page() new_page = self.create_new_page()
new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self) new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
sys.exc_clear()
pass
else: else:
raise InputError("%s: Unsupported encoding" % encoding) raise InputError("%s: Unsupported encoding" % encoding)
@ -590,6 +603,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
if os.path.isabs(value) == False: if os.path.isabs(value) == False:
script_dir = os.path.dirname(__file__) script_dir = os.path.dirname(__file__)
abs_file_path = os.path.join(script_dir, value) abs_file_path = os.path.join(script_dir, value)
with open(abs_file_path, 'rb') as f: with open(abs_file_path, 'rb') as f:
value = f.read() value = f.read()
@ -611,9 +625,9 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
:param input_filename: Name of input file containing data :param input_filename: Name of input file containing data
:param output_filename: Name of output file to store generated binary :param output_filename: Name of output file to store generated binary
:param input_size: Size of partition :param input_size: Size of partition in bytes (must be multiple of 4096)
:param key_gen: Enable encryption key generation in encryption mode :param key_gen: Enable encryption key generation in encryption mode
:param encryption_mode: Enable/Disable encryption mode :param encrypt_mode: Enable/Disable encryption mode
:param key_file: Input file having encryption keys in encryption mode :param key_file: Input file having encryption keys in encryption mode
:return: None :return: None
""" """
@ -623,16 +637,16 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
is_encrypt_data = encrypt_mode is_encrypt_data = encrypt_mode
# Set size # Set size
input_size = int(input_size.split('KB')[0]) * 1024 input_size = int(input_size, 0)
if input_size % 4096 !=0: if input_size % 4096 !=0:
sys.exit("Size parameter should be a multiple of 4KB.") sys.exit("Size of partition (must be multiple of 4096)")
if version == 'v1': if version == 'v1':
version = Page.VERSION1 version = Page.VERSION1
elif version == 'v2': elif version == 'v2':
version = Page.VERSION2 version = Page.VERSION2
# Update size as a page needs to be reserved of size 4KB # Update size as a page needs to be reserved of size 4KB
input_size = input_size - Page.PAGE_PARAMS["max_size"] input_size = input_size - Page.PAGE_PARAMS["max_size"]
@ -662,13 +676,13 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.") sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
if key_gen: if key_gen:
key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128)) key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
elif key_file: elif key_file:
with open(key_file) as key_f: with open(key_file, 'rt', encoding='utf8') as key_f:
key_input = key_f.readline() key_input = key_f.readline()
key_input = key_input.strip() key_input = key_input.strip()
input_file = open(input_filename, 'rb') input_file = open(input_filename, 'rt', encoding='utf8')
output_file = open(output_filename, 'wb') output_file = open(output_filename, 'wb')
with nvs_open(output_file, input_size) as nvs_obj: with nvs_open(output_file, input_size) as nvs_obj:
@ -686,15 +700,21 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
output_file.close() output_file.close()
if is_encrypt_data: if is_encrypt_data:
output_keys_file = open("encryption_keys.bin",'wb')
keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"] keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
key_bytes = key_input.decode('hex') key_bytes = bytearray()
key_bytes = codecs.decode(key_input, 'hex')
key_len = len(key_bytes) key_len = len(key_bytes)
keys_page_buf[0:key_len] = key_bytes keys_page_buf[0:key_len] = key_bytes
crc_data = keys_page_buf[0:key_len] crc_data = keys_page_buf[0:key_len]
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF) crc_data = bytes(crc_data)
keys_page_buf[64:68] = struct.pack('<I', crc & 0xFFFFFFFF) crc = zlib.crc32(crc_data, 0xFFFFFFFF)
output_keys_file.write(keys_page_buf)
struct.pack_into('<I', keys_page_buf, key_len, crc & 0xFFFFFFFF)
with open("encryption_keys.bin",'wb') as output_keys_file:
output_keys_file.write(keys_page_buf)
@ -712,7 +732,7 @@ def main():
parser.add_argument( parser.add_argument(
"size", "size",
help='Size of NVS Partition in KB. Eg. 12KB') help='Size of NVS Partition in bytes (must be multiple of 4096)')
parser.add_argument( parser.add_argument(
"--version", "--version",

13
components/nvs_flash/test_nvs_host/test_nvs.cpp
View file

@ -2003,7 +2003,7 @@ TEST_CASE("check partition generation utility with multipage blob support disabl
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"../nvs_partition_generator/sample_singlepage_blob.csv", "../nvs_partition_generator/sample_singlepage_blob.csv",
"../nvs_partition_generator/partition_single_page.bin", "../nvs_partition_generator/partition_single_page.bin",
"12KB", "0x3000",
"--version", "--version",
"v1",NULL)); "v1",NULL));
} else { } else {
@ -2071,6 +2071,7 @@ TEST_CASE("read data from partition generated via partition generation utility w
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
file.close(); file.close();
nvs_close(handle); nvs_close(handle);
} }
@ -2082,7 +2083,7 @@ TEST_CASE("check partition generation utility with multipage blob support enable
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"../nvs_partition_generator/sample_multipage_blob.csv", "../nvs_partition_generator/sample_multipage_blob.csv",
"../nvs_partition_generator/partition_multipage_blob.bin", "../nvs_partition_generator/partition_multipage_blob.bin",
"12KB", "0x3000",
"--version", "--version",
"v2",NULL)); "v2",NULL));
} else { } else {
@ -2148,8 +2149,9 @@ TEST_CASE("read data from partition generated via partition generation utility w
file.read(binfiledata,5200); file.read(binfiledata,5200);
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
file.close(); file.close();
nvs_close(handle);
} }
@ -2300,7 +2302,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
"../nvs_partition_generator/nvs_partition_gen.py", "../nvs_partition_generator/nvs_partition_gen.py",
"../nvs_partition_generator/sample_multipage_blob.csv", "../nvs_partition_generator/sample_multipage_blob.csv",
"../nvs_partition_generator/partition_encrypted.bin", "../nvs_partition_generator/partition_encrypted.bin",
"12KB", "0x3000",
"--encrypt", "--encrypt",
"True", "True",
"--keyfile", "--keyfile",
@ -2329,6 +2331,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
uint8_t u8v; uint8_t u8v;
TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
CHECK(u8v == 127); CHECK(u8v == 127);
int8_t i8v; int8_t i8v;
TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v)); TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
CHECK(i8v == -128); CHECK(i8v == -128);
@ -2380,7 +2383,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
file.read(binfiledata,5120); file.read(binfiledata,5120);
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
nvs_close(handle); nvs_close(handle);
TEST_ESP_OK(nvs_flash_deinit()); TEST_ESP_OK(nvs_flash_deinit());

6
tools/mass_mfg/docs/README.rst
View file

@ -115,11 +115,13 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
**Usage**:: **Usage**::
$ ./mfg_gen.py [-h] --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR] $ ./mfg_gen.py [-h] --size PART_SIZE --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| Arguments | Description | | Arguments | Description |
+========================+==============================================================================================+ +========================+==============================================================================================+
| --size PART_SIZE | Size of NVS Partition in bytes (must be multiple of 4096) |
+------------------------+----------------------------------------------------------------------------------------------+
| --conf CONFIG_FILE | the input configuration csv file | | --conf CONFIG_FILE | the input configuration csv file |
+------------------------+----------------------------------------------------------------------------------------------+ +------------------------+----------------------------------------------------------------------------------------------+
| --values VALUES_FILE | the input values csv file | | --values VALUES_FILE | the input values csv file |
@ -134,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
**You can use the below command to run this utility with the sample files provided**:: **You can use the below command to run this utility with the sample files provided**::
$ ./mfg_gen.py --conf sample_config.csv --values sample_values.csv --prefix Fan $ ./mfg_gen.py --size 0x3000 --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
.. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file. .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.

107
tools/mass_mfg/mfg_gen.py Normal file → Executable file
View file

@ -15,6 +15,9 @@
# limitations under the License. # limitations under the License.
# #
from __future__ import print_function
from builtins import range
from future.moves.itertools import zip_longest
import sys import sys
import os import os
import csv import csv
@ -31,15 +34,15 @@ def verify_values_exist(input_values_file, keys_in_values_file):
line_no = 1 line_no = 1
key_count_in_values_file = len(keys_in_values_file) key_count_in_values_file = len(keys_in_values_file)
values_file = open(input_values_file,'rb') values_file = open(input_values_file, 'r')
values_file_reader = csv.reader(values_file, delimiter=',') values_file_reader = csv.reader(values_file, delimiter=',')
keys = values_file_reader.next() keys = next(values_file_reader)
for values_data in values_file_reader: for values_data in values_file_reader:
line_no +=1 line_no +=1
if len(values_data) != key_count_in_values_file: if len(values_data) != key_count_in_values_file:
raise SystemExit("\nOops...Number of values is not equal to number of keys in file: '" + \ raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"\
str(input_values_file) + "' at line No:" + str(line_no) ) % (str(input_values_file), str(line_no)))
def verify_keys_exist(values_file_keys, input_config_file): def verify_keys_exist(values_file_keys, input_config_file):
@ -59,12 +62,11 @@ def verify_keys_exist(values_file_keys, input_config_file):
else: else:
keys_missing.append([config_data[0], line_no]) keys_missing.append([config_data[0], line_no])
if keys_missing: if keys_missing:
print "Oops..."
for key, line_no in keys_missing: for key, line_no in keys_missing:
print "Key:`" + str(key) + "` at line no:" + str(line_no) + \ print("Key:`", str(key), "` at line no:", str(line_no),\
" in config file is not found in values file..." " in config file is not found in values file.")
config_file.close() config_file.close()
raise SystemExit(1) raise SystemExit(1)
@ -84,12 +86,12 @@ def verify_datatype_encoding(input_config_file):
for config_data in config_file_reader: for config_data in config_file_reader:
line_no+=1 line_no+=1
if config_data[1] not in valid_datatypes: if config_data[1] not in valid_datatypes:
raise SystemExit("Oops...config file: `" + str(input_config_file) + \ raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" \
"` has invalid datatype at line no:" + str(line_no)) % (str(input_config_file), str(line_no)))
if 'namespace' not in config_data: if 'namespace' not in config_data:
if config_data[2] not in valid_encodings: if config_data[2] not in valid_encodings:
raise SystemExit("Oops...config file: `" + str(input_config_file) + \ raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" \
"` has invalid encoding at line no:" + str(line_no)) % (str(input_config_file), str(line_no)))
@ -103,9 +105,8 @@ def verify_file_data_count(input_config_file, keys_repeat):
for line in config_file_reader: for line in config_file_reader:
line_no += 1 line_no += 1
if len(line) != 3 and line[0] not in keys_repeat: if len(line) != 3 and line[0] not in keys_repeat:
raise SystemExit("Oops...data missing in config file at line no: " + str(line_no) + \ raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" \
" <format needed:key,type,encoding>") % str(line_no) )
config_file.close() config_file.close()
@ -131,7 +132,7 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
verify_values_exist(input_values_file, keys_in_values_file) verify_values_exist(input_values_file, keys_in_values_file)
except StandardError as std_err: except StandardError as std_err:
print std_err print(std_err)
except: except:
raise raise
@ -203,6 +204,7 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
data_to_write = [] data_to_write = []
target_csv_file = open(output_csv_file, 'w') target_csv_file = open(output_csv_file, 'w')
output_file_writer = csv.writer(target_csv_file, delimiter=',') output_file_writer = csv.writer(target_csv_file, delimiter=',')
output_file_writer.writerow(header) output_file_writer.writerow(header)
@ -246,18 +248,18 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
target_filename = filename + "_created" + file_ext target_filename = filename + "_created" + file_ext
with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to: with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
csv_file_reader = csv.reader(read_from, delimiter=',') csv_file_reader = csv.reader(read_from, delimiter=',')
headers = csv_file_reader.next() headers = next(csv_file_reader)
values = csv_file_reader.next() values = next(csv_file_reader)
total_keys_values = map(None, keys, values)
csv_file_writer = csv.writer(write_to, delimiter=',') csv_file_writer = csv.writer(write_to, delimiter=',')
csv_file_writer.writerow(headers) csv_file_writer.writerow(headers)
csv_file_writer.writerow(values) csv_file_writer.writerow(values)
total_keys_values = list(zip_longest(keys, values))
# read new data, add value if key has repeat tag, write to new file # read new data, add value if key has repeat tag, write to new file
for row in csv_file_reader: for row in csv_file_reader:
index = -1 index = -1
key_val_new = map(None, keys, row) key_val_new = list(zip_longest(keys, row))
key_val_pair = total_keys_values[:] key_val_pair = total_keys_values[:]
key_repeated = total_keys_repeat[:] key_repeated = total_keys_repeat[:]
while key_val_new and key_repeated: while key_val_new and key_repeated:
@ -270,7 +272,7 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
del key_repeated[0] del key_repeated[0]
del key_val_new[0] del key_val_new[0]
del key_val_pair[0] del key_val_pair[0]
return target_filename return target_filename
@ -284,6 +286,11 @@ file_identifier=None,output_dir_path=None):
description="Create binary files from input config and values file", description="Create binary files from input config and values file",
formatter_class=argparse.RawDescriptionHelpFormatter) formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--size",
dest='part_size',
required=True,
help='Size of NVS Partition in bytes (must be multiple of 4096)')
parser.add_argument('--conf', parser.add_argument('--conf',
dest='config_file', dest='config_file',
required=True, required=True,
@ -320,6 +327,7 @@ file_identifier=None,output_dir_path=None):
if not args.outdir.endswith('/'): if not args.outdir.endswith('/'):
args.outdir = args.outdir + '/' args.outdir = args.outdir + '/'
input_part_size = args.part_size
input_config_file = args.config_file input_config_file = args.config_file
input_values_file = args.values_file input_values_file = args.values_file
target_file_name_prefix = args.prefix target_file_name_prefix = args.prefix
@ -345,11 +353,11 @@ file_identifier=None,output_dir_path=None):
# Verify config file is not empty # Verify config file is not empty
if os.stat(input_config_file).st_size == 0: if os.stat(input_config_file).st_size == 0:
raise SystemExit("Oops...config file: " + input_config_file + " is empty...") raise SystemExit("Oops...config file: %s is empty." % input_config_file)
# Verify values file is not empty # Verify values file is not empty
if os.stat(input_values_file).st_size == 0: if os.stat(input_values_file).st_size == 0:
raise SystemExit("Oops...values file: " + input_values_file + " is empty...") raise SystemExit("Oops...values file: %s is empty." % input_values_file )
# Verify config file does not have empty lines # Verify config file does not have empty lines
csv_config_file = open(input_config_file,'r') csv_config_file = open(input_config_file,'r')
@ -364,9 +372,9 @@ file_identifier=None,output_dir_path=None):
is_empty_line = False is_empty_line = False
break break
if is_empty_line: if is_empty_line:
raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...") raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file )
if not config_data: if not config_data:
raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...") raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file )
csv_config_file.seek(0) csv_config_file.seek(0)
@ -381,7 +389,7 @@ file_identifier=None,output_dir_path=None):
csv_config_file.close() csv_config_file.close()
except Exception as e: except Exception as e:
print e print(e)
finally: finally:
csv_config_file.close() csv_config_file.close()
@ -389,7 +397,7 @@ file_identifier=None,output_dir_path=None):
# Verify values file does not have empty lines # Verify values file does not have empty lines
csv_values_file = open(input_values_file,'rb') csv_values_file = open(input_values_file, 'r')
try: try:
values_file_reader = csv.reader(csv_values_file, delimiter=',') values_file_reader = csv.reader(csv_values_file, delimiter=',')
for values_data in values_file_reader: for values_data in values_file_reader:
@ -401,18 +409,18 @@ file_identifier=None,output_dir_path=None):
is_empty_line = False is_empty_line = False
break break
if is_empty_line: if is_empty_line:
raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...") raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
if not values_data: if not values_data:
raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...") raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
csv_values_file.seek(0) csv_values_file.seek(0)
# Extract keys from values file # Extract keys from values file
keys_in_values_file = values_file_reader.next() keys_in_values_file = next(values_file_reader)
csv_values_file.close() csv_values_file.close()
except Exception as e: except Exception as e:
print e print(e)
exit(1) exit(1)
finally: finally:
csv_values_file.close() csv_values_file.close()
@ -420,8 +428,8 @@ file_identifier=None,output_dir_path=None):
# Verify file identifier exists in values file # Verify file identifier exists in values file
if file_identifier: if file_identifier:
if file_identifier not in keys_in_values_file: if file_identifier not in keys_in_values_file:
raise SystemExit('Oops...target_file_identifier: ' + file_identifier + \ raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier )
' does not exist in values file...\n')
# Verify data in the input_config_file and input_values_file # Verify data in the input_config_file and input_values_file
verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\ verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\
@ -431,16 +439,20 @@ file_identifier=None,output_dir_path=None):
config_data_to_write = add_config_data_per_namespace(input_config_file) config_data_to_write = add_config_data_per_namespace(input_config_file)
try: try:
with open(input_values_file,'rb') as csv_values_file: with open(input_values_file, 'r') as csv_values_file:
values_file_reader = csv.reader(csv_values_file, delimiter=',') values_file_reader = csv.reader(csv_values_file, delimiter=',')
keys = values_file_reader.next() keys = next(values_file_reader)
target_values_file = set_repeat_value(keys_repeat, keys, input_values_file) target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
csv_values_file = open(target_values_file, 'rb')
csv_values_file = open(target_values_file, 'r')
values_file_reader = csv.reader(csv_values_file, delimiter=',') values_file_reader = csv.reader(csv_values_file, delimiter=',')
values_file_reader.next() next(values_file_reader)
for values_data_line in values_file_reader: for values_data_line in values_file_reader:
key_value_data = map(None,keys_in_values_file,values_data_line) key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
# Get file identifier value from values file # Get file identifier value from values file
file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \ file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
@ -456,7 +468,7 @@ file_identifier=None,output_dir_path=None):
csv_file_list.append(csv_filename) csv_file_list.append(csv_filename)
output_csv_file = output_target_dir + csv_filename output_csv_file = output_target_dir + csv_filename
if os.path.isfile(output_csv_file): if os.path.isfile(output_csv_file):
raise SystemExit("Target csv file: `" + output_csv_file + "` already exists...") raise SystemExit("Target csv file: %s already exists.`" % output_csv_file )
# Add values corresponding to each key to csv target file # Add values corresponding to each key to csv target file
add_data_to_file(config_data_to_write, key_value_pair, output_csv_file) add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
@ -468,18 +480,19 @@ file_identifier=None,output_dir_path=None):
output_bin_file = output_target_dir + target_file_name_prefix + "-" +\ output_bin_file = output_target_dir + target_file_name_prefix + "-" +\
file_identifier_value + ".bin" file_identifier_value + ".bin"
if os.path.isfile(output_bin_file): if os.path.isfile(output_bin_file):
raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...") raise SystemExit("Target csv file: %s already exists.`" % output_bin_file )
# Create output csv and bin file # Create output csv and bin file
print "CSV Generated: " + str(output_csv_file) print("CSV Generated: ", str(output_csv_file))
nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file) nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
print "NVS Flash Binary Generated: " + str(output_bin_file) input_size=input_part_size)
print("NVS Flash Binary Generated: ", str(output_bin_file))
files_created = True files_created = True
csv_values_file.close() csv_values_file.close()
except Exception as e: except Exception as e:
print e print(e)
exit(1) exit(1)
finally: finally:
csv_values_file.close() csv_values_file.close()
@ -487,8 +500,8 @@ file_identifier=None,output_dir_path=None):
return csv_file_list, files_created return csv_file_list, files_created
except StandardError as std_err: except ValueError as err:
print std_err print(err)
except: except:
raise raise