Merge branch 'feature/nvs_part_gen_util_file_size' into 'master'

Add support for input custom partition size

See merge request idf/esp-idf!2828

components/nvs_flash/nvs_partition_generator/README.rst

@@ -5,7 +5,7 @@ Introduction
 ------------
 
 :component_file:`nvs_flash/nvs_partition_generator/nvs_partition_gen.py` utility is designed to help create a binary file, compatible with NVS architecture defined in :doc:`Non-Volatile Storage </api-reference/storage/nvs_flash>`, based on user provided key-value pairs in a CSV file.
-Utility is ideally suited for generating a binary blob, containing data specific to ODM/OEM, which can be flashed externally at the time of device manufacturing. This helps manufacturers set unique value for various parameters for each device, e.g. serial number, while using same application firmaware for all devices.
+Utility is ideally suited for generating a binary blob, containing data specific to ODM/OEM, which can be flashed externally at the time of device manufacturing. This helps manufacturers set unique value for various parameters for each device, e.g. serial number, while using same application firmware for all devices.
 
 CSV file format
 ---------------
@@ -49,9 +49,19 @@ When a new namespace entry is encountered in the CSV file, each follow-up entrie
 Running the utility
 -------------------
 
-A sample CSV file provided with the utility. You can run the utility using below command::
+You can run the utility using below command::
+
+	python nvs_partition_gen.py [-h] input output size
+
+
+Positional arguments:
+
+| Arguments 					| Description
+|	---		  					| 	---
+|  input          			| Path to CSV file to parse. Will use stdin if omitted (a sample.csv is provided)
+|  output         			| Path to output converted binary file. Will use stdout if omitted
+|  size    	    		    | Size of NVS Partition in KB. E.g. 12KB
 
-    python nvs_partition_generator.py sample.csv sample.bin
 
 Caveats
 -------

components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py

@@ -58,12 +58,13 @@ class Page(object):
     ACTIVE = 0xFFFFFFFE
     FULL = 0xFFFFFFFC
 
-    def __init__(self, page_num):
+    def __init__(self, page_num, is_rsrv_page=False):
         self.entry_num = 0
         self.bitmap_array = array.array('B')
         self.page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
-        self.bitmap_array = self.create_bitmap_array()
+        if not is_rsrv_page:
-        self.set_header(page_num)
+            self.bitmap_array = self.create_bitmap_array()
+            self.set_header(page_num)
 
     def set_header(self, page_num):
         global page_header
@@ -322,7 +323,8 @@ class Page(object):
 NVS class encapsulates all NVS specific operations to create a binary with given key-value pairs. Binary can later be flashed onto device via a flashing utility.
 """
 class NVS(object):
-    def __init__(self, fout):
+    def __init__(self, fout, input_size):
+        self.size = input_size
         self.namespace_idx = 0
         self.page_num = -1
         self.pages = []
@@ -334,12 +336,27 @@ class NVS(object):
 
     def __exit__(self, exc_type, exc_value, traceback):
         if exc_type == None and exc_value == None:
+            # Create pages for remaining available size
+            while True:
+                try:
+                    new_page = self.create_new_page()
+                except InsufficientSizeError:
+                    self.size = None
+                    # Creating the last reserved page
+                    self.create_new_page(True)
+                    break
+
             result = self.get_binary_data()
             self.fout.write(result)
 
-    def create_new_page(self):
+    def create_new_page(self, is_rsrv_page=False):
+        # Update available size as each page is created
+        if self.size == 0:
+            raise InsufficientSizeError("Size parameter is is less than the size of data in csv.Please increase size.")
+        if not is_rsrv_page:
+            self.size = self.size - Page.PAGE_PARAMS["max_size"]
         self.page_num += 1
-        new_page = Page(self.page_num)
+        new_page = Page(self.page_num, is_rsrv_page)
         self.pages.append(new_page)
         self.cur_page = new_page
         return new_page
@@ -418,13 +435,21 @@ class InputError(RuntimeError):
     def __init__(self, e):
         super(InputError, self).__init__(e)
 
-def nvs_open(result_obj):
+class InsufficientSizeError(RuntimeError):
+    """
+    Represents error when NVS Partition size given is insufficient
+    to accomodate the data in the given csv file
+    """
+    def __init__(self, e):
+       super(InsufficientSizeError, self).__init__(e)
+
+def nvs_open(result_obj, input_size):
     """ Wrapper to create and NVS class object. This object can later be used to set key-value pairs
 
     :param result_obj: File/Stream object to dump resultant binary. If data is to be dumped into memory, one way is to use BytesIO object
     :return: NVS class instance
     """
-    return NVS(result_obj)
+    return NVS(result_obj, input_size)
 
 def write_entry(nvs_instance, key, datatype, encoding, value):
     """ Wrapper to set key-value pair in NVS format
@@ -457,16 +482,34 @@ def nvs_close(nvs_instance):
     """
     nvs_instance.__exit__(None, None, None)
 
-def nvs_part_gen(input_filename=None, output_filename=None):
+def nvs_part_gen(input_filename=None, output_filename=None, input_size=None):
+    """ Wrapper to generate nvs partition binary
+
+    :param input_filename: Name of input file containing data
+    :param output_filename: Name of output file to store generated binary
+    :param input_size: Size of partition
+    :return: None
+    """
+    if input_size % 4096 !=0:
+        sys.exit("Size parameter should be a multiple of 4KB.")
+
+    # Update size as a page needs to be reserved of size 4KB
+    input_size = input_size - Page.PAGE_PARAMS["max_size"]
+
+    if input_size == 0:
+        sys.exit("Size parameter is insufficient.")
+
     input_file = open(input_filename, 'rb')
     output_file = open(output_filename, 'wb')
 
-    with nvs_open(output_file) as nvs_obj:
+    with nvs_open(output_file, input_size) as nvs_obj:
+        # Update size as one page is created
+        #nvs_obj.size = input_size - Page.PAGE_PARAMS["max_size"]
         reader = csv.DictReader(input_file, delimiter=',')
         for row in reader:
             try:
                 write_entry(nvs_obj, row["key"], row["type"], row["encoding"], row["value"])
-            except InputError as e:
+            except (InputError, InsufficientSizeError) as e:
                 print(e)
                 input_file.close()
                 output_file.close()
@@ -487,10 +530,17 @@ def main():
             help='Path to output converted binary file. Will use stdout if omitted',
             default=sys.stdout)
 
+    parser.add_argument(
+            "size",
+            help='Size of NVS Partition in KB. Eg. 12KB')
+
     args = parser.parse_args()
     input_filename = args.input
     output_filename = args.output
-    nvs_part_gen(input_filename, output_filename)
+
+    # Set size
+    input_size = int(args.size.split('KB')[0]) * 1024
+    nvs_part_gen(input_filename, output_filename, input_size)
 
 
 

components/nvs_flash/test_nvs_host/test_nvs.cpp

@@ -1977,7 +1977,8 @@ TEST_CASE("check partition generation utility", "[nvs_part_gen]")
         exit(execlp("python", "python",
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample.csv",
-                "../nvs_partition_generator/partition.bin", NULL));
+                "../nvs_partition_generator/partition.bin", 
+                "12KB",NULL));
     } else {
         CHECK(childpid > 0);
         int status;