nvs_util: Fix to support write of multiple singlepage big blob data
Closes https://github.com/espressif/esp-idf/issues/3011
(cherry picked from commit 60b5cdde20
)
This commit is contained in:
parent
286202caa3
commit
86052eec78
|
@ -39,6 +39,8 @@ VERSION2_PRINT = "v2 - Multipage Blob Support Enabled"
|
||||||
|
|
||||||
|
|
||||||
""" Class for standard NVS page structure """
|
""" Class for standard NVS page structure """
|
||||||
|
|
||||||
|
|
||||||
class Page(object):
|
class Page(object):
|
||||||
PAGE_PARAMS = {
|
PAGE_PARAMS = {
|
||||||
"max_size": 4096,
|
"max_size": 4096,
|
||||||
|
@ -102,7 +104,6 @@ class Page(object):
|
||||||
struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
|
struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
|
||||||
self.page_buf[0:len(page_header)] = page_header
|
self.page_buf[0:len(page_header)] = page_header
|
||||||
|
|
||||||
|
|
||||||
def create_bitmap_array(self):
|
def create_bitmap_array(self):
|
||||||
bitarray = array.array('B')
|
bitarray = array.array('B')
|
||||||
charsize = 32 # bitmaparray has 256 bits, hence 32 bytes
|
charsize = 32 # bitmaparray has 256 bits, hence 32 bytes
|
||||||
|
@ -110,7 +111,6 @@ class Page(object):
|
||||||
bitarray.extend((fill,) * charsize)
|
bitarray.extend((fill,) * charsize)
|
||||||
return bitarray
|
return bitarray
|
||||||
|
|
||||||
|
|
||||||
def write_bitmaparray(self):
|
def write_bitmaparray(self):
|
||||||
bitnum = self.entry_num * 2
|
bitnum = self.entry_num * 2
|
||||||
byte_idx = bitnum // 8 # Find byte index in the array
|
byte_idx = bitnum // 8 # Find byte index in the array
|
||||||
|
@ -121,7 +121,6 @@ class Page(object):
|
||||||
end_idx = Page.BITMAPARRAY_OFFSET + Page.BITMAPARRAY_SIZE_IN_BYTES
|
end_idx = Page.BITMAPARRAY_OFFSET + Page.BITMAPARRAY_SIZE_IN_BYTES
|
||||||
self.page_buf[start_idx:end_idx] = self.bitmap_array
|
self.page_buf[start_idx:end_idx] = self.bitmap_array
|
||||||
|
|
||||||
|
|
||||||
def encrypt_entry(self, data_arr, tweak_arr, encr_key):
|
def encrypt_entry(self, data_arr, tweak_arr, encr_key):
|
||||||
# Encrypt 32 bytes of data using AES-XTS encryption
|
# Encrypt 32 bytes of data using AES-XTS encryption
|
||||||
backend = default_backend()
|
backend = default_backend()
|
||||||
|
@ -134,7 +133,6 @@ class Page(object):
|
||||||
|
|
||||||
return encrypted_data
|
return encrypted_data
|
||||||
|
|
||||||
|
|
||||||
def reverse_hexbytes(self, addr_tmp):
|
def reverse_hexbytes(self, addr_tmp):
|
||||||
addr = []
|
addr = []
|
||||||
reversed_bytes = ""
|
reversed_bytes = ""
|
||||||
|
@ -144,7 +142,6 @@ class Page(object):
|
||||||
|
|
||||||
return reversed_bytes
|
return reversed_bytes
|
||||||
|
|
||||||
|
|
||||||
def encrypt_data(self, data_input, no_of_entries, nvs_obj):
|
def encrypt_data(self, data_input, no_of_entries, nvs_obj):
|
||||||
# Set values needed for encryption and encrypt data byte wise
|
# Set values needed for encryption and encrypt data byte wise
|
||||||
encr_data_to_write = bytearray()
|
encr_data_to_write = bytearray()
|
||||||
|
@ -155,7 +152,6 @@ class Page(object):
|
||||||
tweak_tmp = ''
|
tweak_tmp = ''
|
||||||
encr_key_input = None
|
encr_key_input = None
|
||||||
|
|
||||||
|
|
||||||
# Extract encryption key and tweak key from given key input
|
# Extract encryption key and tweak key from given key input
|
||||||
if len(self.encr_key) == key_len_needed:
|
if len(self.encr_key) == key_len_needed:
|
||||||
encr_key_input = self.encr_key
|
encr_key_input = self.encr_key
|
||||||
|
@ -207,7 +203,6 @@ class Page(object):
|
||||||
|
|
||||||
return encr_data_to_write
|
return encr_data_to_write
|
||||||
|
|
||||||
|
|
||||||
def write_entry_to_buf(self, data, entrycount,nvs_obj):
|
def write_entry_to_buf(self, data, entrycount,nvs_obj):
|
||||||
encr_data = bytearray()
|
encr_data = bytearray()
|
||||||
|
|
||||||
|
@ -226,7 +221,6 @@ class Page(object):
|
||||||
self.write_bitmaparray()
|
self.write_bitmaparray()
|
||||||
self.entry_num += 1
|
self.entry_num += 1
|
||||||
|
|
||||||
|
|
||||||
def set_crc_header(self, entry_struct):
|
def set_crc_header(self, entry_struct):
|
||||||
crc_data = bytearray(b'28')
|
crc_data = bytearray(b'28')
|
||||||
crc_data[0:4] = entry_struct[0:4]
|
crc_data[0:4] = entry_struct[0:4]
|
||||||
|
@ -236,7 +230,6 @@ class Page(object):
|
||||||
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
|
struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
|
||||||
return entry_struct
|
return entry_struct
|
||||||
|
|
||||||
|
|
||||||
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, encoding, nvs_obj):
|
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, encoding, nvs_obj):
|
||||||
chunk_start = 0
|
chunk_start = 0
|
||||||
chunk_count = 0
|
chunk_count = 0
|
||||||
|
@ -306,7 +299,6 @@ class Page(object):
|
||||||
|
|
||||||
offset = offset + chunk_size
|
offset = offset + chunk_size
|
||||||
|
|
||||||
|
|
||||||
# All chunks are stored, now store the index
|
# All chunks are stored, now store the index
|
||||||
if not remaining_size:
|
if not remaining_size:
|
||||||
# Initialise data field to 0xff
|
# Initialise data field to 0xff
|
||||||
|
@ -336,7 +328,6 @@ class Page(object):
|
||||||
|
|
||||||
return entry_struct
|
return entry_struct
|
||||||
|
|
||||||
|
|
||||||
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
|
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
|
||||||
# compute CRC of data
|
# compute CRC of data
|
||||||
struct.pack_into('<H', entry_struct, 24, datalen)
|
struct.pack_into('<H', entry_struct, 24, datalen)
|
||||||
|
@ -355,7 +346,6 @@ class Page(object):
|
||||||
# write actual data
|
# write actual data
|
||||||
self.write_entry_to_buf(data, data_entry_count, nvs_obj)
|
self.write_entry_to_buf(data, data_entry_count, nvs_obj)
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Low-level function to write variable length data into page buffer. Data should be formatted
|
Low-level function to write variable length data into page buffer. Data should be formatted
|
||||||
according to encoding specified.
|
according to encoding specified.
|
||||||
|
@ -364,13 +354,11 @@ class Page(object):
|
||||||
# Set size of data
|
# Set size of data
|
||||||
datalen = len(data)
|
datalen = len(data)
|
||||||
|
|
||||||
if version == Page.VERSION1:
|
|
||||||
if datalen > Page.PAGE_PARAMS["max_old_blob_size"]:
|
if datalen > Page.PAGE_PARAMS["max_old_blob_size"]:
|
||||||
|
if version == Page.VERSION1:
|
||||||
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key))
|
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION1_PRINT,key))
|
||||||
|
else:
|
||||||
if version == Page.VERSION2:
|
|
||||||
if encoding == "string":
|
if encoding == "string":
|
||||||
if datalen > Page.PAGE_PARAMS["max_new_blob_size"]:
|
|
||||||
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key))
|
raise InputError("Version %s\n%s: Size exceeds max allowed length." % (VERSION2_PRINT,key))
|
||||||
|
|
||||||
# Calculate no. of entries data will require
|
# Calculate no. of entries data will require
|
||||||
|
@ -379,13 +367,10 @@ class Page(object):
|
||||||
total_entry_count = data_entry_count + 1 # +1 for the entry header
|
total_entry_count = data_entry_count + 1 # +1 for the entry header
|
||||||
|
|
||||||
# Check if page is already full and new page is needed to be created right away
|
# Check if page is already full and new page is needed to be created right away
|
||||||
if version == Page.VERSION1:
|
if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
|
||||||
if encoding in ["string", "hex2bin", "binary", "base64"]:
|
|
||||||
if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]:
|
|
||||||
raise PageFullError()
|
raise PageFullError()
|
||||||
else:
|
elif (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]:
|
||||||
if encoding == "string":
|
if not (version == Page.VERSION2 and encoding in ["hex2bin", "binary", "base64"]):
|
||||||
if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]:
|
|
||||||
raise PageFullError()
|
raise PageFullError()
|
||||||
|
|
||||||
# Entry header
|
# Entry header
|
||||||
|
@ -414,13 +399,11 @@ class Page(object):
|
||||||
entry_struct[1] = Page.BLOB
|
entry_struct[1] = Page.BLOB
|
||||||
|
|
||||||
if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]):
|
if version == Page.VERSION2 and (encoding in ["hex2bin", "binary", "base64"]):
|
||||||
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\
|
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,
|
||||||
datalen,total_entry_count, encoding, nvs_obj)
|
datalen,total_entry_count, encoding, nvs_obj)
|
||||||
else:
|
else:
|
||||||
self.write_single_page_entry(entry_struct, data, datalen, data_entry_count, nvs_obj)
|
self.write_single_page_entry(entry_struct, data, datalen, data_entry_count, nvs_obj)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
""" Low-level function to write data of primitive type into page buffer. """
|
""" Low-level function to write data of primitive type into page buffer. """
|
||||||
def write_primitive_data(self, key, data, encoding, ns_index,nvs_obj):
|
def write_primitive_data(self, key, data, encoding, ns_index,nvs_obj):
|
||||||
# Check if entry exceeds max number of entries allowed per page
|
# Check if entry exceeds max number of entries allowed per page
|
||||||
|
@ -469,9 +452,13 @@ class Page(object):
|
||||||
def get_data(self):
|
def get_data(self):
|
||||||
return self.page_buf
|
return self.page_buf
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
NVS class encapsulates all NVS specific operations to create a binary with given key-value pairs. Binary can later be flashed onto device via a flashing utility.
|
NVS class encapsulates all NVS specific operations to create a binary with given key-value pairs.
|
||||||
|
Binary can later be flashed onto device via a flashing utility.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class NVS(object):
|
class NVS(object):
|
||||||
def __init__(self, fout, input_size):
|
def __init__(self, fout, input_size):
|
||||||
self.size = input_size
|
self.size = input_size
|
||||||
|
@ -485,11 +472,11 @@ class NVS(object):
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_value, traceback):
|
def __exit__(self, exc_type, exc_value, traceback):
|
||||||
if exc_type == None and exc_value == None:
|
if exc_type is None and exc_value is None:
|
||||||
# Create pages for remaining available size
|
# Create pages for remaining available size
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
new_page = self.create_new_page()
|
self.create_new_page()
|
||||||
except InsufficientSizeError:
|
except InsufficientSizeError:
|
||||||
self.size = None
|
self.size = None
|
||||||
# Creating the last reserved page
|
# Creating the last reserved page
|
||||||
|
@ -577,6 +564,7 @@ class NVS(object):
|
||||||
data += page.get_data()
|
data += page.get_data()
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
class PageFullError(RuntimeError):
|
class PageFullError(RuntimeError):
|
||||||
"""
|
"""
|
||||||
Represents error when current page doesn't have sufficient entries left
|
Represents error when current page doesn't have sufficient entries left
|
||||||
|
@ -585,6 +573,7 @@ class PageFullError(RuntimeError):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super(PageFullError, self).__init__()
|
super(PageFullError, self).__init__()
|
||||||
|
|
||||||
|
|
||||||
class InputError(RuntimeError):
|
class InputError(RuntimeError):
|
||||||
"""
|
"""
|
||||||
Represents error on the input
|
Represents error on the input
|
||||||
|
@ -592,6 +581,7 @@ class InputError(RuntimeError):
|
||||||
def __init__(self, e):
|
def __init__(self, e):
|
||||||
super(InputError, self).__init__(e)
|
super(InputError, self).__init__(e)
|
||||||
|
|
||||||
|
|
||||||
class InsufficientSizeError(RuntimeError):
|
class InsufficientSizeError(RuntimeError):
|
||||||
"""
|
"""
|
||||||
Represents error when NVS Partition size given is insufficient
|
Represents error when NVS Partition size given is insufficient
|
||||||
|
@ -600,6 +590,7 @@ class InsufficientSizeError(RuntimeError):
|
||||||
def __init__(self, e):
|
def __init__(self, e):
|
||||||
super(InsufficientSizeError, self).__init__(e)
|
super(InsufficientSizeError, self).__init__(e)
|
||||||
|
|
||||||
|
|
||||||
def nvs_open(result_obj, input_size):
|
def nvs_open(result_obj, input_size):
|
||||||
""" Wrapper to create and NVS class object. This object can later be used to set key-value pairs
|
""" Wrapper to create and NVS class object. This object can later be used to set key-value pairs
|
||||||
|
|
||||||
|
@ -609,6 +600,7 @@ def nvs_open(result_obj, input_size):
|
||||||
"""
|
"""
|
||||||
return NVS(result_obj, input_size)
|
return NVS(result_obj, input_size)
|
||||||
|
|
||||||
|
|
||||||
def write_entry(nvs_instance, key, datatype, encoding, value):
|
def write_entry(nvs_instance, key, datatype, encoding, value):
|
||||||
""" Wrapper to set key-value pair in NVS format
|
""" Wrapper to set key-value pair in NVS format
|
||||||
|
|
||||||
|
@ -622,7 +614,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
|
||||||
|
|
||||||
if datatype == "file":
|
if datatype == "file":
|
||||||
abs_file_path = value
|
abs_file_path = value
|
||||||
if os.path.isabs(value) == False:
|
if os.path.isabs(value) is False:
|
||||||
script_dir = os.path.dirname(__file__)
|
script_dir = os.path.dirname(__file__)
|
||||||
abs_file_path = os.path.join(script_dir, value)
|
abs_file_path = os.path.join(script_dir, value)
|
||||||
|
|
||||||
|
@ -634,6 +626,7 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
|
||||||
else:
|
else:
|
||||||
nvs_instance.write_entry(key, value, encoding)
|
nvs_instance.write_entry(key, value, encoding)
|
||||||
|
|
||||||
|
|
||||||
def nvs_close(nvs_instance):
|
def nvs_close(nvs_instance):
|
||||||
""" Wrapper to finish writing to NVS and write data to file/stream object provided to nvs_open method
|
""" Wrapper to finish writing to NVS and write data to file/stream object provided to nvs_open method
|
||||||
|
|
||||||
|
@ -643,7 +636,7 @@ def nvs_close(nvs_instance):
|
||||||
nvs_instance.__exit__(None, None, None)
|
nvs_instance.__exit__(None, None, None)
|
||||||
|
|
||||||
|
|
||||||
def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None,\
|
def check_input_args(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None,
|
||||||
encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None):
|
encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_encrypt_arg_str=None):
|
||||||
|
|
||||||
global version, is_encrypt_data, input_size, key_gen
|
global version, is_encrypt_data, input_size, key_gen
|
||||||
|
@ -658,7 +651,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
|
||||||
elif is_encrypt_data.lower() == 'false':
|
elif is_encrypt_data.lower() == 'false':
|
||||||
is_encrypt_data = False
|
is_encrypt_data = False
|
||||||
|
|
||||||
|
|
||||||
if version == 'v1':
|
if version == 'v1':
|
||||||
version = Page.VERSION1
|
version = Page.VERSION1
|
||||||
elif version == 'v2':
|
elif version == 'v2':
|
||||||
|
@ -669,7 +661,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
|
||||||
elif key_gen.lower() == 'false':
|
elif key_gen.lower() == 'false':
|
||||||
key_gen = False
|
key_gen = False
|
||||||
|
|
||||||
|
|
||||||
if key_gen:
|
if key_gen:
|
||||||
if all(arg is not None for arg in [input_filename, output_filename, input_size]):
|
if all(arg is not None for arg in [input_filename, output_filename, input_size]):
|
||||||
if not is_encrypt_data:
|
if not is_encrypt_data:
|
||||||
|
@ -681,7 +672,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
|
||||||
if not all(arg is not None for arg in [input_filename, output_filename]):
|
if not all(arg is not None for arg in [input_filename, output_filename]):
|
||||||
sys.exit(print_arg_str)
|
sys.exit(print_arg_str)
|
||||||
|
|
||||||
|
|
||||||
if is_encrypt_data and not key_gen and not key_file:
|
if is_encrypt_data and not key_gen and not key_file:
|
||||||
sys.exit(print_encrypt_arg_str)
|
sys.exit(print_encrypt_arg_str)
|
||||||
|
|
||||||
|
@ -705,8 +695,6 @@ encrypt_mode=None, key_file=None, version_no=None, print_arg_str=None, print_enc
|
||||||
sys.exit("Minimum NVS partition size needed is 0x3000 bytes.")
|
sys.exit("Minimum NVS partition size needed is 0x3000 bytes.")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None):
|
def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None, is_key_gen=None, encrypt_mode=None, key_file=None, version_no=None):
|
||||||
""" Wrapper to generate nvs partition binary
|
""" Wrapper to generate nvs partition binary
|
||||||
|
|
||||||
|
@ -749,7 +737,6 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_part_size=None
|
||||||
input_file.close()
|
input_file.close()
|
||||||
output_file.close()
|
output_file.close()
|
||||||
|
|
||||||
|
|
||||||
if key_gen:
|
if key_gen:
|
||||||
keys_page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"]
|
keys_page_buf = bytearray(b'\xff') * Page.PAGE_PARAMS["max_size"]
|
||||||
key_bytes = bytearray()
|
key_bytes = bytearray()
|
||||||
|
@ -824,13 +811,13 @@ def main():
|
||||||
is_encrypt_data = args.encrypt
|
is_encrypt_data = args.encrypt
|
||||||
key_file = args.keyfile
|
key_file = args.keyfile
|
||||||
|
|
||||||
print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory.\nTo generate encryption keys --keygen argument is mandatory."
|
print_arg_str = "Invalid.\nTo generate nvs partition binary --input, --output and --size arguments are mandatory.\n \
|
||||||
|
To generate encryption keys --keygen argument is mandatory."
|
||||||
print_encrypt_arg_str = "Missing parameter. Enter --keyfile or --keygen."
|
print_encrypt_arg_str = "Missing parameter. Enter --keyfile or --keygen."
|
||||||
|
|
||||||
check_input_args(input_filename,output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no, print_arg_str, print_encrypt_arg_str)
|
check_input_args(input_filename,output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no, print_arg_str, print_encrypt_arg_str)
|
||||||
nvs_part_gen(input_filename, output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no)
|
nvs_part_gen(input_filename, output_filename, part_size, is_key_gen, is_encrypt_data, key_file, version_no)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -2,8 +2,10 @@
|
||||||
|
|
||||||
# Regexp for matching job names which are incompatible with Python 3
|
# Regexp for matching job names which are incompatible with Python 3
|
||||||
# - assign_test, nvs_compatible_test, IT - auto_test_script causes the incompatibility
|
# - assign_test, nvs_compatible_test, IT - auto_test_script causes the incompatibility
|
||||||
# - UT_009_ - RS485 multi-device test is not started properly
|
# - UT_009_ - multi-device tests are not compatible
|
||||||
py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_'
|
# - UT_014_ - multi-device tests are not compatible
|
||||||
|
# - UT_017_ - multi-device tests are not compatible
|
||||||
|
py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_|UT_013_|UT_014_|UT_017_'
|
||||||
|
|
||||||
if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
|
if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
|
||||||
# Use this version of the Python interpreter if it was not defined before or
|
# Use this version of the Python interpreter if it was not defined before or
|
||||||
|
|
Loading…
Reference in a new issue