Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'feature/nvs_util_backport' into 'master'

Browse source 
nvs_util: Add changes for utility to support old and new multipage blob

See merge request idf/esp-idf!3270
This commit is contained in:
Angus Gratton 2018-09-24 11:53:29 +08:00
parent ae5df5d6ce ca3acef82c
commit 4d50427e87
8 changed files with 220 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
components/nvs_flash/README.rst
View file

@ -147,6 +147,8 @@ Erased (2'b00)
A key-value pair in this entry has been discarded. Contents of this entry will not be parsed anymore.
.. _structure_of_entry:
Structure of entry
^^^^^^^^^^^^^^^^^^

51
components/nvs_flash/nvs_partition_generator/README.rst
View file

@ -46,21 +46,58 @@ When a new namespace entry is encountered in the CSV file, each follow-up entrie
.. note:: First entry in a CSV file should always be ``namespace`` entry.
Multipage Blob Support
----------------------
By default, binary blobs are allowed to span over multiple pages and written in the format mentioned in section :ref:`structure_of_entry`.
If older format is intended to be used, the utility provides an option to disable this feature.
Running the utility
-------------------
You can run the utility using below command::
You can run the utility in two modes using below command:
- Multipage Blob Support Enabled (v2)
- Multipage Blob Support Disabled (v1)
python nvs_partition_gen.py [-h] input output size
*Usage*::
python nvs_partition_gen.py [--version {v1,v2}] input output
Positional arguments:
| Arguments | Description
| --- | ---
| input | Path to CSV file to parse. Will use stdin if omitted (a sample.csv is provided)
| output | Path to output converted binary file. Will use stdout if omitted
| size | Size of NVS Partition in KB. E.g. 12KB
+------------------------+----------------------------------------------------------------------------------------------+
| Arguments | Description |
+========================+==============================================================================================+
| input | Path to CSV file to parse. Will use stdin if omitted (sample files are provided) |
+------------------------+----------------------------------------------------------------------------------------------+
| output | Path to output converted binary file. Will use stdout if omitted |
+------------------------+----------------------------------------------------------------------------------------------+
Optional arguments:
+-------------------------------+---------------------------------------------------------------------------------------+
| Arguments | Description |
+===============================+=======================================================================================+
| --version {v1,v2} | Set version. Default: v2 |
+-------------------------------+---------------------------------------------------------------------------------------+
*Multipage Blob Support Enabled Mode:*
You can run the utility in this mode by setting the version parameter to v2, as shown below.
A sample CSV file is provided with the utility::
python nvs_partition_gen.py sample_multipage_blob.csv partition_multipage_blob.bin --version v2
*Multipage Blob Support Disabled Mode:*
You can run the utility in this mode by setting the version parameter to v1, as shown below.
A sample CSV file is provided with the utility::
python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1
Caveats

102
components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
View file

@ -34,7 +34,8 @@ from os import path
class Page(object):
PAGE_PARAMS = {
"max_size": 4096,
"max_blob_size": 4000,
"max_old_blob_size": 1984,
"max_new_blob_size": 4000,
"max_entries": 126
}
@ -59,10 +60,13 @@ class Page(object):
CHUNK_ANY = 0xFF
ACTIVE = 0xFFFFFFFE
FULL = 0xFFFFFFFC
VERSION1=0xFF
VERSION2=0xFE
def __init__(self, page_num, is_rsrv_page=False):
self.entry_num = 0
self.bitmap_array = array.array('B')
self.version = Page.VERSION2
self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
if not is_rsrv_page:
self.bitmap_array = self.create_bitmap_array()
@ -77,6 +81,11 @@ class Page(object):
page_header[0:4] = struct.pack('<I', page_state_active_seq)
# set page sequence number
page_header[4:8] = struct.pack('<I', page_num)
# set version
if version == Page.VERSION2:
page_header[8] = Page.VERSION2
elif version == Page.VERSION1:
page_header[8] = Page.VERSION1
# set header's CRC
crc_data = page_header[4:28]
crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
@ -118,7 +127,7 @@ class Page(object):
entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
return entry_struct
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count, nvs_obj):
def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
chunk_start = 0
chunk_count = 0
chunk_index = Page.CHUNK_ANY
@ -206,11 +215,23 @@ class Page(object):
self.write_entry_to_buf(entry_struct, 1)
break
return entry_struct
def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count):
# compute CRC of data
entry_struct[24:26] = struct.pack('<H', datalen)
crc = zlib.crc32(data, 0xFFFFFFFF)
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
# compute crc of entry header
entry_struct = self.set_crc_header(entry_struct)
# write entry header
self.write_entry_to_buf(entry_struct, 1)
# write actual data
self.write_entry_to_buf(data, data_entry_count)
"""
Low-level function to write variable length data into page buffer. Data should be formatted
@ -220,10 +241,15 @@ class Page(object):
# Set size of data
datalen = len(data)
if encoding == "string":
if datalen > Page.PAGE_PARAMS["max_blob_size"]:
if version == Page.VERSION1:
if datalen > Page.PAGE_PARAMS["max_old_blob_size"]:
raise InputError("%s: Size exceeds max allowed length." % key)
if version == Page.VERSION2:
if encoding == "string":
if datalen > Page.PAGE_PARAMS["max_new_blob_size"]:
raise InputError("%s: Size exceeds max allowed length." % key)
# Calculate no. of entries data will require
rounded_size = (datalen + 31) & ~31
data_entry_count = rounded_size // 32
@ -239,11 +265,14 @@ class Page(object):
# Set Namespace Index
entry_struct[0] = ns_index
# Set Span
if encoding == "string":
if version == Page.VERSION2:
if encoding == "string":
entry_struct[2] = data_entry_count + 1
# Set Chunk Index
chunk_index = Page.CHUNK_ANY
entry_struct[3] = chunk_index
else:
entry_struct[2] = data_entry_count + 1
# Set Chunk Index
chunk_index = Page.CHUNK_ANY
entry_struct[3] = chunk_index
# set key
key_array = bytearray('\x00')*16
@ -256,22 +285,12 @@ class Page(object):
elif encoding in ["hex2bin", "binary", "base64"]:
entry_struct[1] = Page.BLOB
if encoding == "binary" or encoding == "hex2bin" or encoding == "base64":
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\
datalen,total_entry_count,nvs_obj)
if version == Page.VERSION2 and encoding == "binary" or encoding == "hex2bin" or encoding == "base64":
entry_struct = self.write_varlen_binary_data(entry_struct,ns_index,key,data,\
datalen,total_entry_count, nvs_obj)
else:
# compute CRC of data
entry_struct[24:26] = struct.pack('<H', datalen)
crc = zlib.crc32(data, 0xFFFFFFFF)
entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
self.write_single_page_entry(entry_struct, data, datalen, data_entry_count)
# compute crc of entry header
entry_struct = self.set_crc_header(entry_struct)
# write entry header
self.write_entry_to_buf(entry_struct, 1)
# write actual data
self.write_entry_to_buf(data, data_entry_count)
""" Low-level function to write data of primitive type into page buffer. """
@ -345,7 +364,7 @@ class NVS(object):
except InsufficientSizeError:
self.size = None
# Creating the last reserved page
self.create_new_page(True)
self.create_new_page(is_rsrv_page=True)
break
result = self.get_binary_data()
@ -359,6 +378,7 @@ class NVS(object):
self.size = self.size - Page.PAGE_PARAMS["max_size"]
self.page_num += 1
new_page = Page(self.page_num, is_rsrv_page)
new_page.version = version
self.pages.append(new_page)
self.cur_page = new_page
return new_page
@ -402,7 +422,7 @@ class NVS(object):
self.cur_page.write_varlen_data(key, value, encoding, self.namespace_idx, self)
except PageFullError:
new_page = self.create_new_page()
new_page.write_varlen_data(key, value, encoding, self.namespace_idx, self)
new_page.write_varlen_data(key, value, encoding, self.namespace_idx, self)
pass
elif encoding in primitive_encodings:
try:
@ -484,7 +504,7 @@ def nvs_close(nvs_instance):
"""
nvs_instance.__exit__(None, None, None)
def nvs_part_gen(input_filename=None, output_filename=None, input_size=None):
def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, version_no=None):
""" Wrapper to generate nvs partition binary
:param input_filename: Name of input file containing data
@ -492,9 +512,20 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None):
:param input_size: Size of partition
:return: None
"""
global version
version = version_no
# Set size
input_size = int(input_size.split('KB')[0]) * 1024
if input_size % 4096 !=0:
sys.exit("Size parameter should be a multiple of 4KB.")
if version == 'v1':
version = Page.VERSION1
elif version == 'v2':
version = Page.VERSION2
# Update size as a page needs to be reserved of size 4KB
input_size = input_size - Page.PAGE_PARAMS["max_size"]
@ -505,8 +536,6 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None):
output_file = open(output_filename, 'wb')
with nvs_open(output_file, input_size) as nvs_obj:
# Update size as one page is created
#nvs_obj.size = input_size - Page.PAGE_PARAMS["max_size"]
reader = csv.DictReader(input_file, delimiter=',')
for row in reader:
try:
@ -515,7 +544,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None):
print(e)
input_file.close()
output_file.close()
exit(-2)
sys.exit(-2)
input_file.close()
output_file.close()
@ -536,13 +565,20 @@ def main():
"size",
help='Size of NVS Partition in KB. Eg. 12KB')
parser.add_argument(
"--version",
help='Set version. Default: v2',
choices=['v1','v2'],
default='v2')
args = parser.parse_args()
input_filename = args.input
output_filename = args.output
input_size = args.size
version_no = args.version
# Set size
input_size = int(args.size.split('KB')[0]) * 1024
nvs_part_gen(input_filename, output_filename, input_size)
nvs_part_gen(input_filename, output_filename, input_size, version_no)

2
components/nvs_flash/nvs_partition_generator/sample.csv → components/nvs_flash/nvs_partition_generator/sample_multipage_blob.csv
View file

@ -11,4 +11,4 @@ dummyBase64Key,data,base64,MTIzYWJj
hexFileKey,file,hex2bin,testdata/sample.hex
base64FileKey,file,base64,testdata/sample.base64
stringFileKey,file,string,testdata/sample.txt
binFileKey,file,binary,testdata/sample.bin
binFileKey,file,binary,testdata/sample_multipage_blob.bin
1 key type encoding value
11 hexFileKey file hex2bin testdata/sample.hex
12 base64FileKey file base64 testdata/sample.base64
13 stringFileKey file string testdata/sample.txt
14 binFileKey file binary testdata/sample.bin testdata/sample_multipage_blob.bin

14
components/nvs_flash/nvs_partition_generator/sample_singlepage_blob.csv Normal file
View file

@ -0,0 +1,14 @@
key,type,encoding,value
dummyNamespace,namespace,,
dummyU8Key,data,u8,127
dummyI8Key,data,i8,-128
dummyU16Key,data,u16,32768
dummyU32Key,data,u32,4294967295
dummyI32Key,data,i32,-2147483648
dummyStringKey,data,string,0A:0B:0C:0D:0E:0F
dummyHex2BinKey,data,hex2bin,010203abcdef
dummyBase64Key,data,base64,MTIzYWJj
hexFileKey,file,hex2bin,testdata/sample.hex
base64FileKey,file,base64,testdata/sample.base64
stringFileKey,file,string,testdata/sample.txt
binFileKey,file,binary,testdata/sample_singlepage_blob.bin
1 key type encoding value
2 dummyNamespace namespace
3 dummyU8Key data u8 127
4 dummyI8Key data i8 -128
5 dummyU16Key data u16 32768
6 dummyU32Key data u32 4294967295
7 dummyI32Key data i32 -2147483648
8 dummyStringKey data string 0A:0B:0C:0D:0E:0F
9 dummyHex2BinKey data hex2bin 010203abcdef
10 dummyBase64Key data base64 MTIzYWJj
11 hexFileKey file hex2bin testdata/sample.hex
12 base64FileKey file base64 testdata/sample.base64
13 stringFileKey file string testdata/sample.txt
14 binFileKey file binary testdata/sample_singlepage_blob.bin

0
components/nvs_flash/nvs_partition_generator/testdata/sample.bin → components/nvs_flash/nvs_partition_generator/testdata/sample_multipage_blob.bin vendored
View file

1
components/nvs_flash/nvs_partition_generator/testdata/sample_singlepage_blob.bin vendored Normal file
View file

@ -0,0 +1 @@
start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000

96
components/nvs_flash/test_nvs_host/test_nvs.cpp
View file

@ -1991,15 +1991,17 @@ TEST_CASE("Recovery from power-off during modification of blob present in old-fo
/* Add new tests above */
/* This test has to be the final one */
TEST_CASE("check partition generation utility", "[nvs_part_gen]")
TEST_CASE("check partition generation utility with multipage blob support disabled", "[nvs_part_gen]")
{
int childpid = fork();
if (childpid == 0) {
exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py",
"../nvs_partition_generator/sample.csv",
"../nvs_partition_generator/partition.bin",
"12KB",NULL));
"../nvs_partition_generator/sample_singlepage_blob.csv",
"../nvs_partition_generator/partition_single_page.bin",
"12KB",
"--version",
"v1",NULL));
} else {
CHECK(childpid > 0);
int status;
@ -2008,9 +2010,9 @@ TEST_CASE("check partition generation utility", "[nvs_part_gen]")
}
}
TEST_CASE("read data from partition generated via partition generation utility", "[nvs_part_gen]")
TEST_CASE("read data from partition generated via partition generation utility with multipage blob support disabled", "[nvs_part_gen]")
{
SpiFlashEmulator emu("../nvs_partition_generator/partition.bin");
SpiFlashEmulator emu("../nvs_partition_generator/partition_single_page.bin");
nvs_handle handle;
TEST_ESP_OK( nvs_flash_init_custom("test", 0, 3) );
TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle));
@ -2059,7 +2061,7 @@ TEST_CASE("read data from partition generated via partition generation utility",
size_t bin_len = sizeof(bin_data);
char binfiledata[5200];
ifstream file;
file.open("../nvs_partition_generator/testdata/sample.bin");
file.open("../nvs_partition_generator/testdata/sample_singlepage_blob.bin");
file.read(binfiledata,5200);
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
@ -2068,6 +2070,86 @@ TEST_CASE("read data from partition generated via partition generation utility",
}
TEST_CASE("check partition generation utility with multipage blob support enabled", "[nvs_part_gen]")
{
int childpid = fork();
if (childpid == 0) {
exit(execlp("python", "python",
"../nvs_partition_generator/nvs_partition_gen.py",
"../nvs_partition_generator/sample_multipage_blob.csv",
"../nvs_partition_generator/partition_multipage_blob.bin",
"12KB",
"--version",
"v2",NULL));
} else {
CHECK(childpid > 0);
int status;
waitpid(childpid, &status, 0);
CHECK(WEXITSTATUS(status) != -1);
}
}
TEST_CASE("read data from partition generated via partition generation utility with multipage blob support enabled", "[nvs_part_gen]")
{
SpiFlashEmulator emu("../nvs_partition_generator/partition_multipage_blob.bin");
nvs_handle handle;
TEST_ESP_OK( nvs_flash_init_custom("test", 0, 3) );
TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle));
uint8_t u8v;
TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
CHECK(u8v == 127);
int8_t i8v;
TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
CHECK(i8v == -128);
uint16_t u16v;
TEST_ESP_OK( nvs_get_u16(handle, "dummyU16Key", &u16v));
CHECK(u16v == 32768);
uint32_t u32v;
TEST_ESP_OK( nvs_get_u32(handle, "dummyU32Key", &u32v));
CHECK(u32v == 4294967295);
int32_t i32v;
TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v));
CHECK(i32v == -2147483648);
char buf[64] = {0};
size_t buflen = 64;
TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen));
CHECK(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0);
uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef};
buflen = 64;
int j;
TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen));
CHECK(memcmp(buf, hexdata, buflen) == 0);
uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'};
TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen));
CHECK(memcmp(buf, base64data, buflen) == 0);
buflen = 64;
uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef};
TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen));
CHECK(memcmp(buf, hexfiledata, buflen) == 0);
buflen = 64;
uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0";
TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen));
CHECK(memcmp(buf, strfiledata, buflen) == 0);
char bin_data[5200];
size_t bin_len = sizeof(bin_data);
char binfiledata[5200];
ifstream file;
file.open("../nvs_partition_generator/testdata/sample_multipage_blob.bin");
file.read(binfiledata,5200);
TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
file.close();
}
TEST_CASE("dump all performance data", "[nvs]")
{
std::cout << "====================" << std::endl << "Dumping benchmarks" << std::endl;