From bc39e9ac75540e58d4b71a362e0c3316c4e12f2b Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Tue, 11 Sep 2018 13:43:05 +0530
Subject: [PATCH 01/24] Update mfg utility as per updated nvs partition util

---
 tools/mass_mfg/docs/README.rst | 6 ++++--
 tools/mass_mfg/mfg_gen.py      | 9 ++++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 tools/mass_mfg/mfg_gen.py

diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index 4914806d0..f71ec7eac 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -115,11 +115,13 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **Usage**::
 
-    $ ./mfg_gen.py [-h] --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
+    $ ./mfg_gen.py [-h] --size PART_SIZE --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
 
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
+| --size PART_SIZE       |  the size of NVS Partition in KB. Eg. 12KB                                                   |
++------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --values VALUES_FILE   |  the input values csv file                                                                   |
@@ -134,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **You can use the below command to run this utility with the sample files provided**::
    
-   $ ./mfg_gen.py --conf sample_config.csv --values sample_values.csv --prefix Fan
+   $ ./mfg_gen.py --size 12KB --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
 
 
 .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
old mode 100644
new mode 100755
index 9060dde22..56aa61cac
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -284,6 +284,11 @@ file_identifier=None,output_dir_path=None):
                                              description="Create binary files from input config and values file",
                                              formatter_class=argparse.RawDescriptionHelpFormatter)
 
+            parser.add_argument("--size",
+                                dest='part_size',
+                                required=True,
+                                help='Size of NVS Partition in KB. Eg. 12KB')
+            
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,
@@ -320,6 +325,7 @@ file_identifier=None,output_dir_path=None):
             if not args.outdir.endswith('/'):
                 args.outdir = args.outdir + '/'
 
+            input_part_size = args.part_size
             input_config_file = args.config_file
             input_values_file = args.values_file
             target_file_name_prefix = args.prefix
@@ -472,7 +478,8 @@ file_identifier=None,output_dir_path=None):
 
                 # Create output csv and bin file
                 print "CSV Generated: " + str(output_csv_file)
-                nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file)
+                nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
+                                               input_size=input_part_size)
                 print "NVS Flash Binary Generated: " + str(output_bin_file)
 
                 files_created = True

From 98433d48f1387284d1008dd9e6922e3718fa6170 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Wed, 3 Oct 2018 17:01:00 +0530
Subject: [PATCH 02/24] mfg_gen: Add changes to make Python2 and Python3
 compatible

---
 tools/mass_mfg/mfg_gen.py | 102 +++++++++++++++++++++++++++-----------
 1 file changed, 73 insertions(+), 29 deletions(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 56aa61cac..a9813ae22 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -15,25 +15,32 @@
 # limitations under the License.
 #
 
+from __future__ import print_function
 import sys
 import os
 import csv
 import argparse
 import shutil
 import distutils.dir_util
+from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
 
-
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
     """
     line_no = 1
     key_count_in_values_file = len(keys_in_values_file)
 
-    values_file = open(input_values_file,'rb')
+    if sys.version_info[0] < 3:
+        values_file = open(input_values_file, 'rb')
+    else:
+        values_file = open(input_values_file, 'r', newline='')
     values_file_reader = csv.reader(values_file, delimiter=',')
-    keys = values_file_reader.next()
+    if sys.version_info[0] < 3:
+        keys = values_file_reader.next()
+    else:
+        keys = next(values_file_reader)
 
     for values_data in values_file_reader:
         line_no +=1
@@ -59,12 +66,11 @@ def verify_keys_exist(values_file_keys, input_config_file):
             else:
                 keys_missing.append([config_data[0], line_no])
 
-    
+
     if keys_missing:
-        print "Oops..."
         for key, line_no in keys_missing:
-            print "Key:`" + str(key) + "` at line no:" + str(line_no) + \
-            " in config file is not found in values file..."
+            print("Key:`", str(key), "` at line no:", str(line_no),\
+            " in config file is not found in values file.")
         config_file.close()
         raise SystemExit(1)
 
@@ -131,7 +137,7 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
         verify_values_exist(input_values_file, keys_in_values_file)
 
     except StandardError as std_err:
-        print  std_err
+        print(std_err)
     except:
         raise
 
@@ -203,6 +209,7 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
     data_to_write = []
 
     target_csv_file = open(output_csv_file, 'w')
+
     output_file_writer = csv.writer(target_csv_file, delimiter=',')
     output_file_writer.writerow(header)
 
@@ -246,9 +253,17 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
     target_filename = filename + "_created" + file_ext
     with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
         csv_file_reader = csv.reader(read_from, delimiter=',')
-        headers = csv_file_reader.next()
-        values = csv_file_reader.next()
-        total_keys_values = map(None, keys, values)
+        if sys.version_info[0] < 3:
+            headers = csv_file_reader.next()
+            values = csv_file_reader.next()
+        else:
+            headers = next(csv_file_reader)
+            values = next(csv_file_reader)
+
+        if sys.version_info[0] < 3:
+            total_keys_values = map(None, keys, values)
+        else:
+            total_keys_values = list(zip_longest(keys, values))
 
         csv_file_writer = csv.writer(write_to, delimiter=',')
         csv_file_writer.writerow(headers)
@@ -257,7 +272,11 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
         # read new data, add value if key has repeat tag, write to new file
         for row in csv_file_reader:
             index = -1
-            key_val_new = map(None, keys, row)
+            if sys.version_info[0] < 3:
+                key_val_new = map(None, keys, row)
+            else:
+                key_val_new = list(zip_longest(keys, row))
+
             key_val_pair = total_keys_values[:]
             key_repeated = total_keys_repeat[:]
             while key_val_new and key_repeated:
@@ -270,7 +289,7 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
                     del key_repeated[0]
                 del key_val_new[0]
                 del key_val_pair[0]
-           
+
 
     return target_filename
 
@@ -288,7 +307,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in KB. Eg. 12KB')
-            
+
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,
@@ -387,7 +406,7 @@ file_identifier=None,output_dir_path=None):
 
             csv_config_file.close()
         except Exception as e:
-            print e
+            print(e)
         finally:
             csv_config_file.close()
 
@@ -395,7 +414,10 @@ file_identifier=None,output_dir_path=None):
 
 
         # Verify values file does not have empty lines
-        csv_values_file = open(input_values_file,'rb')
+        if sys.version_info[0] < 3:
+            csv_values_file = open(input_values_file,'rb')
+        else:
+            csv_values_file = open(input_values_file,'r', newline='')
         try:
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
             for values_data in values_file_reader:
@@ -414,11 +436,14 @@ file_identifier=None,output_dir_path=None):
             csv_values_file.seek(0)
 
             # Extract keys from values file
-            keys_in_values_file = values_file_reader.next()
+            if sys.version_info[0] < 3:
+                keys_in_values_file = values_file_reader.next()
+            else:
+                keys_in_values_file = next(values_file_reader)
 
             csv_values_file.close()
         except Exception as e:
-            print e
+            print(e)
             exit(1)
         finally:
             csv_values_file.close()
@@ -437,16 +462,35 @@ file_identifier=None,output_dir_path=None):
         config_data_to_write = add_config_data_per_namespace(input_config_file)
 
         try:
-            with open(input_values_file,'rb') as csv_values_file:
-                values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                keys = values_file_reader.next()
+            if sys.version_info[0] < 3:
+                with open(input_values_file,'rb') as csv_values_file:
+                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                    keys = values_file_reader.next()
+            else:
+                with open(input_values_file,'r', newline='') as csv_values_file:
+                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                    keys = next(values_file_reader)
+
             target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
-            csv_values_file = open(target_values_file, 'rb')
+
+            if sys.version_info[0] < 3:
+                csv_values_file = open(target_values_file, 'rb')
+            else:
+                csv_values_file = open(target_values_file, 'r', newline='')
+
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
-            values_file_reader.next()
+            if sys.version_info[0] < 3:
+                values_file_reader.next()
+            else:
+                next(values_file_reader)
+
 
             for values_data_line in values_file_reader:
-                key_value_data = map(None,keys_in_values_file,values_data_line)
+                if sys.version_info[0] < 3:
+                    key_value_data = map(None,keys_in_values_file,values_data_line)
+                else:
+                    key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
+
 
                 # Get file identifier value from values file
                 file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
@@ -477,16 +521,16 @@ file_identifier=None,output_dir_path=None):
                     raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...")
 
                 # Create output csv and bin file
-                print "CSV Generated: " + str(output_csv_file)
+                print("CSV Generated: ", str(output_csv_file))
                 nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
                                                input_size=input_part_size)
-                print "NVS Flash Binary Generated: " + str(output_bin_file)
+                print("NVS Flash Binary Generated: ", str(output_bin_file))
 
                 files_created = True
 
             csv_values_file.close()
         except Exception as e:
-            print e
+            print(e)
             exit(1)
         finally:
             csv_values_file.close()
@@ -494,8 +538,8 @@ file_identifier=None,output_dir_path=None):
 
         return csv_file_list, files_created
 
-    except StandardError as std_err:
-        print std_err
+    except ValueError as err:
+        print(err)
     except:
         raise
 

From ff30a1fbd35a01dfbbbb4bb6420db956f78e002a Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 10:32:09 +0530
Subject: [PATCH 03/24] nvs_part_gen: Update to make python version compatible

(cherry picked from commit 3506b2d60792e5d4ed4f744b28f2da6733c6aae7)
---
 .../nvs_partition_gen.py                      | 199 +++++++++++++-----
 .../nvs_flash/test_nvs_host/test_nvs.cpp      |  13 +-
 2 files changed, 154 insertions(+), 58 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 0f641d612..a1cc81a65 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -28,6 +28,7 @@ import os
 import array
 import csv
 import zlib
+import codecs
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends import default_backend
 
@@ -81,9 +82,9 @@ class Page(object):
         # set page state to active
         page_header= bytearray(b'\xff')*32
         page_state_active_seq = Page.ACTIVE
-        page_header[0:4] = struct.pack('<I', page_state_active_seq)
+        struct.pack_into('<I', page_header, 0,  page_state_active_seq)
         # set page sequence number
-        page_header[4:8] = struct.pack('<I', page_num)
+        struct.pack_into('<I', page_header, 4, page_num)
         # set version
         if version == Page.VERSION2:
             page_header[8] = Page.VERSION2
@@ -91,8 +92,11 @@ class Page(object):
             page_header[8] = Page.VERSION1
         # set header's CRC
         crc_data = page_header[4:28]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
         self.page_buf[0:len(page_header)] = page_header
 
 
@@ -118,8 +122,13 @@ class Page(object):
     def encrypt_entry(self, data_arr, tweak_arr, encr_key):
         # Encrypt 32 bytes of data using AES-XTS encryption
         backend = default_backend()
-        plain_text = data_arr.decode('hex')
-        tweak = tweak_arr.decode('hex')
+        if sys.version_info[0] < 3:
+            plain_text = data_arr.decode('hex')
+            tweak = tweak_arr.decode('hex')
+        else:
+            plain_text = codecs.decode(data_arr, 'hex')
+            tweak = codecs.decode(tweak_arr, 'hex')
+
         cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
         encryptor = cipher.encryptor()
         encrypted_data = encryptor.update(plain_text)
@@ -139,7 +148,7 @@ class Page(object):
 
     def encrypt_data(self, data_input, no_of_entries, nvs_obj):
         # Set values needed for encryption and encrypt data byte wise
-        encr_data_to_write = ''
+        encr_data_to_write = bytearray()
         data_len_needed = 64 #in hex
         tweak_len_needed = 32 #in hex
         init_tweak_val = '0'
@@ -147,17 +156,34 @@ class Page(object):
         tweak_tmp = ''
         encr_key_input = None
 
+
         # Extract encryption key and tweak key from given key input
-        encr_key_input = self.encr_key.decode('hex')
+        if sys.version_info[0] < 3:
+            encr_key_input = self.encr_key.decode('hex')
+        else:
+            encr_key_input = codecs.decode(self.encr_key, 'hex')
+
 
         rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
 
+
         if type(data_input) != bytearray:
-            byte_arr = bytearray('\xff') * 32
-            byte_arr[0:len(data_input)] = data_input
+            byte_arr = bytearray(b'\xff') * 32
+            if sys.version_info[0] < 3:
+                byte_arr[0:len(data_input)] = data_input
+            else:
+                if type(data_input) == str:
+                    byte_arr[0:len(data_input)] = data_input
+                else:
+                    byte_arr[0:len(data_input)] = data_input
+
             data_input = byte_arr
 
-        data_input = binascii.hexlify(bytearray(data_input))
+        
+        if sys.version_info[0] < 3:
+            data_input = binascii.hexlify(bytearray(data_input))
+        else:
+            data_input = data_input.hex()
 
         entry_no = self.entry_num
         start_idx = 0
@@ -182,8 +208,13 @@ class Page(object):
 
             # Encrypt data
             data_bytes = data_input[start_idx:end_idx]
-            data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
+            if sys.version_info[0] < 3:
+                data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
+            else:
+                data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
+
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
+            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -198,13 +229,24 @@ class Page(object):
         encr_data = bytearray()
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            encr_data[0:len(encr_data_ret)] = encr_data_ret
+            if sys.version_info[0] < 3:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            else:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        self.page_buf[start_idx:end_idx]  = data
+        if not sys.version_info[0] < 3:
+            if type(data) == str:
+                self.page_buf[start_idx:end_idx]  = data
+            else:
+                self.page_buf[start_idx:end_idx]  = data
+        else:
+            self.page_buf[start_idx:end_idx]  = data
+
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):
@@ -213,11 +255,14 @@ class Page(object):
 
 
     def set_crc_header(self, entry_struct):
-        crc_data = bytearray(28)
+        crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(crc_data, 0xFFFFFFFF)
+        struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
         return entry_struct
 
     def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
@@ -262,9 +307,13 @@ class Page(object):
             data_chunk =  data[offset:offset + chunk_size]
 
             # Compute CRC of data chunk
-            entry_struct[24:26] = struct.pack('<H', chunk_size)
+            struct.pack_into('<H', entry_struct, 24, chunk_size)
+            if not sys.version_info[0] < 3:
+                if type(data_chunk) == str:
+                    data_chunk = codecs.encode(data_chunk, 'utf8')
+            
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
-            entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+            struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
             # compute crc of entry header
             entry_struct = self.set_crc_header(entry_struct)
@@ -279,7 +328,7 @@ class Page(object):
             if remaining_size or (tailroom - chunk_size) < Page.SINGLE_ENTRY_SIZE:
                 if page_header[0:4] != Page.FULL:
                     page_state_full_seq = Page.FULL
-                    page_header[0:4] = struct.pack('<I', page_state_full_seq)
+                    struct.pack_into('<I', page_header, 0, page_state_full_seq)
                 nvs_obj.create_new_page()
                 self = nvs_obj.cur_page
 
@@ -289,7 +338,7 @@ class Page(object):
             # All chunks are stored, now store the index
             if not remaining_size:
                 # Initialise data field to 0xff
-                data_array = bytearray('\xff')*8
+                data_array = bytearray(b'\xff')*8
                 entry_struct[24:32] = data_array
 
                 # change type of data to BLOB_IDX
@@ -302,7 +351,7 @@ class Page(object):
                 chunk_index = Page.CHUNK_ANY
                 entry_struct[3] = chunk_index
 
-                entry_struct[24:28] = struct.pack('<I', data_size)
+                struct.pack_into('<I', entry_struct, 24, data_size)
                 entry_struct[28] = chunk_count
                 entry_struct[29] = chunk_start
 
@@ -318,9 +367,17 @@ class Page(object):
 
     def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
         # compute CRC of data
-        entry_struct[24:26] = struct.pack('<H', datalen)
-        crc = zlib.crc32(data, 0xFFFFFFFF)
-        entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+        struct.pack_into('<H', entry_struct, 24, datalen)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(data, 0xFFFFFFFF)
+        else:
+            if (type(data)) == str:
+                data = data.encode('utf8')
+                crc = zlib.crc32(data, 0xFFFFFFFF)
+            else:
+                crc = zlib.crc32(data, 0xFFFFFFFF)
+
+        struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
         # compute crc of entry header
         entry_struct = self.set_crc_header(entry_struct)
@@ -359,7 +416,7 @@ class Page(object):
                 raise PageFullError()
 
         # Entry header
-        entry_struct = bytearray('\xff')*32
+        entry_struct = bytearray(b'\xff')*32
         # Set Namespace Index
         entry_struct[0] = ns_index
         # Set Span
@@ -373,9 +430,12 @@ class Page(object):
             entry_struct[2] = data_entry_count + 1
 
         # set key
-        key_array = bytearray('\x00')*16
+        key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        entry_struct[8:8 + len(key)] = key
+        if sys.version_info[0] < 3:
+            entry_struct[8:8 + len(key)] = key
+        else:
+            entry_struct[8:8 + len(key)] = key.encode('utf8')
 
         # set Type
         if encoding == "string":
@@ -397,39 +457,47 @@ class Page(object):
         if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
             raise PageFullError()
 
-        entry_struct = bytearray('\xff')*32
+        entry_struct = bytearray(b'\xff')*32
         entry_struct[0] = ns_index # namespace index
         entry_struct[2] = 0x01 # Span
         chunk_index = Page.CHUNK_ANY
         entry_struct[3] = chunk_index
 
         # write key
-        key_array = bytearray('\x00')*16
+        key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        entry_struct[8:8 + len(key)] = key
+        
+        if sys.version_info[0] < 3:
+            entry_struct[8:8 + len(key)] = key
+        else:
+            entry_struct[8:8 + len(key)] = key.encode('utf8')
+
 
         if encoding == "u8":
             entry_struct[1] = Page.U8
-            entry_struct[24] = struct.pack('<B', data)
+            struct.pack_into('<B', entry_struct, 24, data)
         elif encoding == "i8":
             entry_struct[1] = Page.I8
-            entry_struct[24] = struct.pack('<b', data)
+            struct.pack_into('<b', entry_struct, 24, data)
         elif encoding == "u16":
             entry_struct[1] = Page.U16
-            entry_struct[24:26] = struct.pack('<H', data)
+            struct.pack_into('<H', entry_struct, 24, data)
         elif encoding == "u32":
             entry_struct[1] = Page.U32
-            entry_struct[24:28] = struct.pack('<I', data)
+            struct.pack_into('<I', entry_struct, 24, data)
         elif encoding == "i32":
             entry_struct[1] = Page.I32
-            entry_struct[24:28] = struct.pack('<i', data)
+            struct.pack_into('<i', entry_struct, 24, data)
 
         # Compute CRC
-        crc_data = bytearray(28)
+        crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
 
         # write to file
         self.write_entry_to_buf(entry_struct, 1,nvs_obj)
@@ -478,7 +546,8 @@ class NVS(object):
         new_page = Page(self.page_num, is_rsrv_page)
         new_page.version = version
         new_page.is_encrypt = is_encrypt_data
-        new_page.encr_key = key_input
+        if new_page.is_encrypt:
+            new_page.encr_key = key_input
         self.pages.append(new_page)
         self.cur_page = new_page
         return new_page
@@ -590,8 +659,12 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
         if os.path.isabs(value) == False:
             script_dir = os.path.dirname(__file__)
             abs_file_path = os.path.join(script_dir, value)
-        with open(abs_file_path, 'rb') as f:
-            value = f.read()
+        if sys.version_info[0] < 3:
+            with open(abs_file_path, 'rb') as f:
+                value = f.read()
+        else:
+            with open(abs_file_path, 'r', newline='') as f:
+                value = f.read()
 
     if datatype == "namespace":
         nvs_instance.write_namespace(key)
@@ -611,9 +684,9 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
 
     :param input_filename: Name of input file containing data
     :param output_filename: Name of output file to store generated binary
-    :param input_size: Size of partition
+    :param input_size: Size of partition (must be multiple of 4096)
     :param key_gen: Enable encryption key generation in encryption mode
-    :param encryption_mode: Enable/Disable encryption mode
+    :param encrypt_mode: Enable/Disable encryption mode
     :param key_file: Input file having encryption keys in encryption mode
     :return: None
     """
@@ -623,16 +696,16 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     is_encrypt_data = encrypt_mode
 
     # Set size
-    input_size = int(input_size.split('KB')[0]) * 1024
+    input_size = int(input_size, 16)
 
     if input_size % 4096 !=0:
-        sys.exit("Size parameter should be a multiple of 4KB.")
+        sys.exit("Size of partition (must be multiple of 4096)")
 
     if version == 'v1':
         version = Page.VERSION1
     elif version == 'v2':
         version = Page.VERSION2
-    
+
     # Update size as a page needs to be reserved of size 4KB
     input_size = input_size - Page.PAGE_PARAMS["max_size"]
 
@@ -662,13 +735,20 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
 
     if key_gen:
-        key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
+        if sys.version_info[0] < 3:
+            key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
+        else:
+            key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
     elif key_file:
         with open(key_file) as key_f:
             key_input = key_f.readline()
             key_input = key_input.strip()
 
-    input_file = open(input_filename, 'rb')
+    if sys.version_info[0] < 3:
+        input_file = open(input_filename, 'rb')
+    else:
+        input_file = open(input_filename, 'r', newline='')
+    
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -688,12 +768,25 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     if is_encrypt_data:
         output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
-        key_bytes = key_input.decode('hex')
+        
+        if sys.version_info[0] < 3:
+            key_bytes = key_input.decode('hex')
+        else:
+            key_bytes = bytearray()
+            key_bytes = codecs.decode(key_input, 'hex') 
+
+        
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
+        
         crc_data = keys_page_buf[0:key_len]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        keys_page_buf[64:68] = struct.pack('<I', crc & 0xFFFFFFFF)
+        
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        
+        struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
         output_keys_file.write(keys_page_buf)
 
 
@@ -712,7 +805,7 @@ def main():
 
     parser.add_argument(
             "size",
-            help='Size of NVS Partition in KB. Eg. 12KB')
+            help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
 
     parser.add_argument(
             "--version",
diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp
index da06e328f..aeeb3188c 100644
--- a/components/nvs_flash/test_nvs_host/test_nvs.cpp
+++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp
@@ -2003,7 +2003,7 @@ TEST_CASE("check partition generation utility with multipage blob support disabl
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_singlepage_blob.csv",
                 "../nvs_partition_generator/partition_single_page.bin", 
-                "12KB",
+                "0x3000",
                 "--version",
                 "v1",NULL));
     } else {
@@ -2071,6 +2071,7 @@ TEST_CASE("read data from partition generated via partition generation utility w
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
 
     file.close();
+    
     nvs_close(handle);
 }
 
@@ -2082,7 +2083,7 @@ TEST_CASE("check partition generation utility with multipage blob support enable
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_multipage_blob.csv",
                 "../nvs_partition_generator/partition_multipage_blob.bin", 
-                "12KB",
+                "0x3000",
                 "--version",
                 "v2",NULL));
     } else {
@@ -2148,8 +2149,9 @@ TEST_CASE("read data from partition generated via partition generation utility w
     file.read(binfiledata,5200);
     TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
-
     file.close();
+    
+    nvs_close(handle);
 
 }
 
@@ -2300,7 +2302,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_multipage_blob.csv",
                 "../nvs_partition_generator/partition_encrypted.bin",
-                "12KB",
+                "0x3000",
                 "--encrypt",
                 "True",
                 "--keyfile",
@@ -2329,6 +2331,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
     uint8_t u8v;
     TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
     CHECK(u8v == 127);
+    
     int8_t i8v;
     TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
     CHECK(i8v == -128);
@@ -2380,7 +2383,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
     file.read(binfiledata,5120);
     TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
-
+    
     nvs_close(handle);
     TEST_ESP_OK(nvs_flash_deinit());
 

From cd7dcd0c859df3e90e3428d59f88e869b299d6cd Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 11:11:56 +0530
Subject: [PATCH 04/24] nvs_part_gen: Add changes to partition size input

---
 .../nvs_partition_generator/README.rst        |  2 +-
 .../nvs_partition_gen.py                      | 23 +++++++++----------
 tools/mass_mfg/docs/README.rst                |  4 ++--
 tools/mass_mfg/mfg_gen.py                     |  5 ++--
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst
index aa7a945c2..f0c6d6623 100644
--- a/components/nvs_flash/nvs_partition_generator/README.rst
+++ b/components/nvs_flash/nvs_partition_generator/README.rst
@@ -123,7 +123,7 @@ A sample CSV file is provided with the utility::
 +------------------------+----------------------------------------------------------------------------------------------+
 | output                 | Path to output converted binary file. Will use stdout if omitted                             |
 +------------------------+----------------------------------------------------------------------------------------------+
-| size                   | Size of NVS Partition in KB. Eg. 12KB                                                        |
+| size                   | Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                          |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --version {v1,v2}      |  Set version. Default: v2                                                                    |
 +-------------------------------+---------------------------------------------------------------------------------------+
diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index a1cc81a65..2b11877ca 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -179,7 +179,7 @@ class Page(object):
 
             data_input = byte_arr
 
-        
+
         if sys.version_info[0] < 3:
             data_input = binascii.hexlify(bytearray(data_input))
         else:
@@ -214,7 +214,6 @@ class Page(object):
                 data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
 
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
-            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -233,7 +232,7 @@ class Page(object):
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
             else:
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
-            
+
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
@@ -311,7 +310,7 @@ class Page(object):
             if not sys.version_info[0] < 3:
                 if type(data_chunk) == str:
                     data_chunk = codecs.encode(data_chunk, 'utf8')
-            
+
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
             struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
@@ -466,7 +465,7 @@ class Page(object):
         # write key
         key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        
+
         if sys.version_info[0] < 3:
             entry_struct[8:8 + len(key)] = key
         else:
@@ -748,7 +747,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         input_file = open(input_filename, 'rb')
     else:
         input_file = open(input_filename, 'r', newline='')
-    
+
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -768,24 +767,24 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     if is_encrypt_data:
         output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
-        
+
         if sys.version_info[0] < 3:
             key_bytes = key_input.decode('hex')
         else:
             key_bytes = bytearray()
-            key_bytes = codecs.decode(key_input, 'hex') 
+            key_bytes = codecs.decode(key_input, 'hex')
+
 
-        
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
-        
+
         crc_data = keys_page_buf[0:key_len]
-        
+
         if sys.version_info[0] < 3:
             crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
         else:
             crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
-        
+
         struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
         output_keys_file.write(keys_page_buf)
 
diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index f71ec7eac..8d50aedf8 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -120,7 +120,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
-| --size PART_SIZE       |  the size of NVS Partition in KB. Eg. 12KB                                                   |
+| --size PART_SIZE       |  Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                         |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
@@ -136,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **You can use the below command to run this utility with the sample files provided**::
    
-   $ ./mfg_gen.py --size 12KB --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
+   $ ./mfg_gen.py --size 0x3000 --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
 
 
 .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index a9813ae22..ca9c94cb0 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,9 +22,10 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
-from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
+if not sys.version_info[0] < 3:
+    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -306,7 +307,7 @@ file_identifier=None,output_dir_path=None):
             parser.add_argument("--size",
                                 dest='part_size',
                                 required=True,
-                                help='Size of NVS Partition in KB. Eg. 12KB')
+                                help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
 
             parser.add_argument('--conf',
                                 dest='config_file',

From dca5ac78f06d2513be4c594bd074f2696603b600 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 11:49:32 +0530
Subject: [PATCH 05/24] nvs_util: Add Python2 and Python3 compatible

---
 .../nvs_partition_gen.py                      | 171 +++++-------------
 1 file changed, 50 insertions(+), 121 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2b11877ca..2330c6102 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -17,8 +17,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from __future__ import division
-from __future__ import print_function
+
+from __future__ import division, print_function #, unicode_literals
+from future.utils import raise_
+from builtins import int, range
+from io import open
 import sys
 import argparse
 import binascii
@@ -80,7 +83,7 @@ class Page(object):
         global page_header
 
         # set page state to active
-        page_header= bytearray(b'\xff')*32
+        page_header= bytearray(b'\xff') *32
         page_state_active_seq = Page.ACTIVE
         struct.pack_into('<I', page_header, 0,  page_state_active_seq)
         # set page sequence number
@@ -91,11 +94,8 @@ class Page(object):
         elif version == Page.VERSION1:
             page_header[8] = Page.VERSION1
         # set header's CRC
-        crc_data = page_header[4:28]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(page_header[4:28])
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
         self.page_buf[0:len(page_header)] = page_header
 
@@ -122,12 +122,8 @@ class Page(object):
     def encrypt_entry(self, data_arr, tweak_arr, encr_key):
         # Encrypt 32 bytes of data using AES-XTS encryption
         backend = default_backend()
-        if sys.version_info[0] < 3:
-            plain_text = data_arr.decode('hex')
-            tweak = tweak_arr.decode('hex')
-        else:
-            plain_text = codecs.decode(data_arr, 'hex')
-            tweak = codecs.decode(tweak_arr, 'hex')
+        plain_text = codecs.decode(data_arr, 'hex')
+        tweak = codecs.decode(tweak_arr, 'hex')
 
         cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
         encryptor = cipher.encryptor()
@@ -158,32 +154,16 @@ class Page(object):
 
 
         # Extract encryption key and tweak key from given key input
-        if sys.version_info[0] < 3:
-            encr_key_input = self.encr_key.decode('hex')
-        else:
-            encr_key_input = codecs.decode(self.encr_key, 'hex')
-
+        encr_key_input = codecs.decode(self.encr_key, 'hex')
 
         rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
 
-
-        if type(data_input) != bytearray:
+        if not isinstance(data_input, bytearray):
             byte_arr = bytearray(b'\xff') * 32
-            if sys.version_info[0] < 3:
-                byte_arr[0:len(data_input)] = data_input
-            else:
-                if type(data_input) == str:
-                    byte_arr[0:len(data_input)] = data_input
-                else:
-                    byte_arr[0:len(data_input)] = data_input
-
+            byte_arr[0:len(data_input)] = data_input
             data_input = byte_arr
 
-
-        if sys.version_info[0] < 3:
-            data_input = binascii.hexlify(bytearray(data_input))
-        else:
-            data_input = data_input.hex()
+        data_input = binascii.hexlify(data_input)
 
         entry_no = self.entry_num
         start_idx = 0
@@ -208,11 +188,10 @@ class Page(object):
 
             # Encrypt data
             data_bytes = data_input[start_idx:end_idx]
-            if sys.version_info[0] < 3:
-                data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
-            else:
-                data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
+            if type(data_bytes) == bytes:
+                data_bytes = data_bytes.decode()
 
+            data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
@@ -220,32 +199,21 @@ class Page(object):
             end_idx = start_idx + 64
             entry_no += 1
 
-
         return encr_data_to_write
 
 
     def write_entry_to_buf(self, data, entrycount,nvs_obj):
         encr_data = bytearray()
+
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            if sys.version_info[0] < 3:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-            else:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-
+            encr_data[0:len(encr_data_ret)] = encr_data_ret
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        if not sys.version_info[0] < 3:
-            if type(data) == str:
-                self.page_buf[start_idx:end_idx]  = data
-            else:
-                self.page_buf[start_idx:end_idx]  = data
-        else:
-            self.page_buf[start_idx:end_idx]  = data
-
+        self.page_buf[start_idx:end_idx]  = data
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):
@@ -257,13 +225,12 @@ class Page(object):
         crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(crc_data, 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
         return entry_struct
 
+
     def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
         chunk_start = 0
         chunk_count = 0
@@ -307,10 +274,7 @@ class Page(object):
 
             # Compute CRC of data chunk
             struct.pack_into('<H', entry_struct, 24, chunk_size)
-            if not sys.version_info[0] < 3:
-                if type(data_chunk) == str:
-                    data_chunk = codecs.encode(data_chunk, 'utf8')
-
+            data_chunk = bytes(data_chunk)
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
             struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
@@ -367,15 +331,9 @@ class Page(object):
     def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
         # compute CRC of data
         struct.pack_into('<H', entry_struct, 24, datalen)
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(data, 0xFFFFFFFF)
-        else:
-            if (type(data)) == str:
-                data = data.encode('utf8')
-                crc = zlib.crc32(data, 0xFFFFFFFF)
-            else:
-                crc = zlib.crc32(data, 0xFFFFFFFF)
-
+        if not type(data) == bytes:
+            data = data.encode()
+        crc = zlib.crc32(data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
         # compute crc of entry header
@@ -429,12 +387,9 @@ class Page(object):
             entry_struct[2] = data_entry_count + 1
 
         # set key
-        key_array = bytearray(b'\x00')*16
+        key_array = b'\x00' * 16
         entry_struct[8:24] = key_array
-        if sys.version_info[0] < 3:
-            entry_struct[8:8 + len(key)] = key
-        else:
-            entry_struct[8:8 + len(key)] = key.encode('utf8')
+        entry_struct[8:8 + len(key)] = key.encode()
 
         # set Type
         if encoding == "string":
@@ -463,14 +418,9 @@ class Page(object):
         entry_struct[3] = chunk_index
 
         # write key
-        key_array = bytearray(b'\x00')*16
+        key_array = b'\x00' *16
         entry_struct[8:24] = key_array
-
-        if sys.version_info[0] < 3:
-            entry_struct[8:8 + len(key)] = key
-        else:
-            entry_struct[8:8 + len(key)] = key.encode('utf8')
-
+        entry_struct[8:8 + len(key)] = key.encode()
 
         if encoding == "u8":
             entry_struct[1] = Page.U8
@@ -492,10 +442,8 @@ class Page(object):
         crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
 
         # write to file
@@ -562,7 +510,6 @@ class NVS(object):
         except PageFullError:
             new_page = self.create_new_page()
             new_page.write_primitive_data(key, self.namespace_idx, "u8", 0,self)
-            pass
 
     """
     Write key-value pair. Function accepts value in the form of ascii character and converts
@@ -580,6 +527,8 @@ class NVS(object):
             value = binascii.a2b_base64(value)
 
         if encoding == "string":
+            if type(value) == bytes:
+                value = value.decode()
             value += '\0'
 
         encoding = encoding.lower()
@@ -591,15 +540,12 @@ class NVS(object):
             except PageFullError:
                 new_page = self.create_new_page()
                 new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
-                pass
         elif encoding in primitive_encodings:
             try:
                 self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
             except PageFullError:
                 new_page = self.create_new_page()
                 new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
-                sys.exc_clear()
-                pass
         else:
             raise InputError("%s: Unsupported encoding" % encoding)
 
@@ -658,12 +604,9 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
         if os.path.isabs(value) == False:
             script_dir = os.path.dirname(__file__)
             abs_file_path = os.path.join(script_dir, value)
-        if sys.version_info[0] < 3:
-            with open(abs_file_path, 'rb') as f:
-                value = f.read()
-        else:
-            with open(abs_file_path, 'r', newline='') as f:
-                value = f.read()
+
+        with open(abs_file_path, 'rb') as f:
+            value = f.read()
 
     if datatype == "namespace":
         nvs_instance.write_namespace(key)
@@ -683,7 +626,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
 
     :param input_filename: Name of input file containing data
     :param output_filename: Name of output file to store generated binary
-    :param input_size: Size of partition (must be multiple of 4096)
+    :param input_size: Size of partition in bytes (must be multiple of 4096)
     :param key_gen: Enable encryption key generation in encryption mode
     :param encrypt_mode: Enable/Disable encryption mode
     :param key_file: Input file having encryption keys in encryption mode
@@ -695,7 +638,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     is_encrypt_data = encrypt_mode
 
     # Set size
-    input_size = int(input_size, 16)
+    input_size = int(input_size, 0)
 
     if input_size % 4096 !=0:
         sys.exit("Size of partition (must be multiple of 4096)")
@@ -734,20 +677,13 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
 
     if key_gen:
-        if sys.version_info[0] < 3:
-            key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
-        else:
-            key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
+        key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
     elif key_file:
-        with open(key_file) as key_f:
+        with open(key_file, 'rt', encoding='utf8') as key_f:
             key_input = key_f.readline()
             key_input = key_input.strip()
 
-    if sys.version_info[0] < 3:
-        input_file = open(input_filename, 'rb')
-    else:
-        input_file = open(input_filename, 'r', newline='')
-
+    input_file = open(input_filename, 'rt', encoding='utf8')
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -765,28 +701,21 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     output_file.close()
 
     if is_encrypt_data:
-        output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
+        key_bytes = bytearray()
 
-        if sys.version_info[0] < 3:
-            key_bytes = key_input.decode('hex')
-        else:
-            key_bytes = bytearray()
-            key_bytes = codecs.decode(key_input, 'hex')
-
-
+        key_bytes = codecs.decode(key_input, 'hex')
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
 
         crc_data = keys_page_buf[0:key_len]
-
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
 
         struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
-        output_keys_file.write(keys_page_buf)
+
+        with open("encryption_keys.bin",'wb') as output_keys_file:
+            output_keys_file.write(keys_page_buf)
 
 
 

From dd2bc7256d49bc0dd80b514a6b91781567715b76 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Tue, 11 Sep 2018 13:43:05 +0530
Subject: [PATCH 06/24] Update mfg utility as per updated nvs partition util

---
 tools/mass_mfg/mfg_gen.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index ca9c94cb0..c02c0e156 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -308,7 +308,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
-
+            
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,

From 24fa2342e8434317d5bd693141e7f37989bd4c3a Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Wed, 3 Oct 2018 17:01:00 +0530
Subject: [PATCH 07/24] mfg_gen: Add changes to make Python2 and Python3
 compatible

---
 tools/mass_mfg/mfg_gen.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index c02c0e156..255aa50b1 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,10 +22,9 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
+from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
-if not sys.version_info[0] < 3:
-    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file

From 226b6929e526055ac570ad5cfd468441136b8f7f Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 10:32:09 +0530
Subject: [PATCH 08/24] nvs_part_gen: Update to make python version compatible

(cherry picked from commit 3506b2d60792e5d4ed4f744b28f2da6733c6aae7)
---
 .../nvs_partition_generator/nvs_partition_gen.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2330c6102..d0c529791 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -193,6 +193,7 @@ class Page(object):
 
             data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
+            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -207,13 +208,24 @@ class Page(object):
 
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            encr_data[0:len(encr_data_ret)] = encr_data_ret
+            if sys.version_info[0] < 3:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            else:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        self.page_buf[start_idx:end_idx]  = data
+        if not sys.version_info[0] < 3:
+            if type(data) == str:
+                self.page_buf[start_idx:end_idx]  = data
+            else:
+                self.page_buf[start_idx:end_idx]  = data
+        else:
+            self.page_buf[start_idx:end_idx]  = data
+
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):

From 0064357a986a0a15dc1d79d3693c8d89a1ba5667 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 11:11:56 +0530
Subject: [PATCH 09/24] nvs_part_gen: Add changes to partition size input

---
 .../nvs_flash/nvs_partition_generator/nvs_partition_gen.py   | 3 +--
 tools/mass_mfg/mfg_gen.py                                    | 5 +++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index d0c529791..003d1bab9 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -193,7 +193,6 @@ class Page(object):
 
             data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
-            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -212,7 +211,7 @@ class Page(object):
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
             else:
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
-            
+
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 255aa50b1..ca9c94cb0 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,9 +22,10 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
-from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
+if not sys.version_info[0] < 3:
+    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -307,7 +308,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
-            
+
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,

From b521f1b8ba209ec3bd2fa726c66d5ce2be16c4e6 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 11:49:32 +0530
Subject: [PATCH 10/24] nvs_util: Add Python2 and Python3 compatible

---
 .../nvs_partition_generator/nvs_partition_gen.py  | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 003d1bab9..2330c6102 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -207,24 +207,13 @@ class Page(object):
 
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            if sys.version_info[0] < 3:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-            else:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-
+            encr_data[0:len(encr_data_ret)] = encr_data_ret
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        if not sys.version_info[0] < 3:
-            if type(data) == str:
-                self.page_buf[start_idx:end_idx]  = data
-            else:
-                self.page_buf[start_idx:end_idx]  = data
-        else:
-            self.page_buf[start_idx:end_idx]  = data
-
+        self.page_buf[start_idx:end_idx]  = data
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):

From 64728a74f970b6dd3cb501da54544584023cee83 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 17:50:12 +0530
Subject: [PATCH 11/24] tools: Add Python2 and Python3 compatibility support to
 nvs util tool and mfg tool

---
 .../nvs_partition_gen.py                      |   1 -
 tools/mass_mfg/mfg_gen.py                     | 108 ++++++------------
 2 files changed, 34 insertions(+), 75 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2330c6102..2a01d0744 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -19,7 +19,6 @@
 #
 
 from __future__ import division, print_function #, unicode_literals
-from future.utils import raise_
 from builtins import int, range
 from io import open
 import sys
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index ca9c94cb0..07b2b28e6 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -16,6 +16,8 @@
 #
 
 from __future__ import print_function
+from builtins import map, range
+from future.moves.itertools import zip_longest
 import sys
 import os
 import csv
@@ -24,8 +26,6 @@ import shutil
 import distutils.dir_util
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
-if not sys.version_info[0] < 3:
-    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -33,21 +33,15 @@ def verify_values_exist(input_values_file, keys_in_values_file):
     line_no = 1
     key_count_in_values_file = len(keys_in_values_file)
 
-    if sys.version_info[0] < 3:
-        values_file = open(input_values_file, 'rb')
-    else:
-        values_file = open(input_values_file, 'r', newline='')
+    values_file = open(input_values_file, 'r')
     values_file_reader = csv.reader(values_file, delimiter=',')
-    if sys.version_info[0] < 3:
-        keys = values_file_reader.next()
-    else:
-        keys = next(values_file_reader)
+    keys = next(values_file_reader)
 
     for values_data in values_file_reader:
         line_no +=1
         if len(values_data) != key_count_in_values_file:
-            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: '" + \
-            str(input_values_file) + "' at line No:" + str(line_no) )
+            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"\
+            % (str(input_values_file), str(line_no)))
 
 
 def verify_keys_exist(values_file_keys, input_config_file):
@@ -91,12 +85,12 @@ def verify_datatype_encoding(input_config_file):
     for config_data in config_file_reader:
         line_no+=1
         if config_data[1] not in valid_datatypes:
-            raise SystemExit("Oops...config file: `" + str(input_config_file) + \
-            "` has invalid datatype at line no:" + str(line_no))
+            raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" \
+            % (str(input_config_file), str(line_no)))
         if 'namespace' not in config_data:
             if config_data[2] not in valid_encodings:
-                raise SystemExit("Oops...config file: `" + str(input_config_file) + \
-                "` has invalid encoding at line no:" + str(line_no))
+                raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" \
+                % (str(input_config_file), str(line_no)))
 
 
 
@@ -110,9 +104,8 @@ def verify_file_data_count(input_config_file, keys_repeat):
     for line in config_file_reader:
         line_no += 1
         if len(line) != 3 and line[0] not in keys_repeat:
-            raise SystemExit("Oops...data missing in config file at line no: " + str(line_no) + \
-            " <format needed:key,type,encoding>")
-
+            raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" \
+            % str(line_no) )
     config_file.close()
 
 
@@ -254,29 +247,17 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
     target_filename = filename + "_created" + file_ext
     with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
         csv_file_reader = csv.reader(read_from, delimiter=',')
-        if sys.version_info[0] < 3:
-            headers = csv_file_reader.next()
-            values = csv_file_reader.next()
-        else:
-            headers = next(csv_file_reader)
-            values = next(csv_file_reader)
-
-        if sys.version_info[0] < 3:
-            total_keys_values = map(None, keys, values)
-        else:
-            total_keys_values = list(zip_longest(keys, values))
-
+        headers = next(csv_file_reader)
+        values = next(csv_file_reader)
         csv_file_writer = csv.writer(write_to, delimiter=',')
         csv_file_writer.writerow(headers)
         csv_file_writer.writerow(values)
+        total_keys_values = list(zip_longest(keys, values))
 
         # read new data, add value if key has repeat tag, write to new file
         for row in csv_file_reader:
             index = -1
-            if sys.version_info[0] < 3:
-                key_val_new = map(None, keys, row)
-            else:
-                key_val_new = list(zip_longest(keys, row))
+            key_val_new = list(zip_longest(keys, row))
 
             key_val_pair = total_keys_values[:]
             key_repeated = total_keys_repeat[:]
@@ -371,11 +352,11 @@ file_identifier=None,output_dir_path=None):
 
         # Verify config file is not empty
         if os.stat(input_config_file).st_size == 0:
-            raise SystemExit("Oops...config file: " + input_config_file + " is empty...")
+            raise SystemExit("Oops...config file: %s is empty." % input_config_file)
 
         # Verify values file is not empty
         if os.stat(input_values_file).st_size == 0:
-            raise SystemExit("Oops...values file: " + input_values_file + " is empty...")
+            raise SystemExit("Oops...values file: %s is empty." % input_values_file )
 
         # Verify config file does not have empty lines
         csv_config_file = open(input_config_file,'r')
@@ -390,9 +371,9 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file )
                 if not config_data:
-                    raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file )
 
             csv_config_file.seek(0)
 
@@ -415,10 +396,7 @@ file_identifier=None,output_dir_path=None):
 
 
         # Verify values file does not have empty lines
-        if sys.version_info[0] < 3:
-            csv_values_file = open(input_values_file,'rb')
-        else:
-            csv_values_file = open(input_values_file,'r', newline='')
+        csv_values_file = open(input_values_file, 'r')
         try:
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
             for values_data in values_file_reader:
@@ -430,17 +408,14 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
                 if not values_data:
-                    raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
 
             csv_values_file.seek(0)
 
             # Extract keys from values file
-            if sys.version_info[0] < 3:
-                keys_in_values_file = values_file_reader.next()
-            else:
-                keys_in_values_file = next(values_file_reader)
+            keys_in_values_file = next(values_file_reader)
 
             csv_values_file.close()
         except Exception as e:
@@ -452,8 +427,8 @@ file_identifier=None,output_dir_path=None):
         # Verify file identifier exists in values file
         if file_identifier:
             if file_identifier not in keys_in_values_file:
-                raise SystemExit('Oops...target_file_identifier: ' + file_identifier + \
-                ' does not exist in values file...\n')
+                raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier )
+
 
         # Verify data in the input_config_file and input_values_file
         verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\
@@ -463,35 +438,20 @@ file_identifier=None,output_dir_path=None):
         config_data_to_write = add_config_data_per_namespace(input_config_file)
 
         try:
-            if sys.version_info[0] < 3:
-                with open(input_values_file,'rb') as csv_values_file:
-                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                    keys = values_file_reader.next()
-            else:
-                with open(input_values_file,'r', newline='') as csv_values_file:
-                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                    keys = next(values_file_reader)
+            with open(input_values_file, 'r') as csv_values_file:
+                values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                keys = next(values_file_reader)
 
             target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
 
-            if sys.version_info[0] < 3:
-                csv_values_file = open(target_values_file, 'rb')
-            else:
-                csv_values_file = open(target_values_file, 'r', newline='')
+            csv_values_file = open(target_values_file, 'r')
 
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
-            if sys.version_info[0] < 3:
-                values_file_reader.next()
-            else:
-                next(values_file_reader)
+            next(values_file_reader)
 
 
             for values_data_line in values_file_reader:
-                if sys.version_info[0] < 3:
-                    key_value_data = map(None,keys_in_values_file,values_data_line)
-                else:
-                    key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
-
+                key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
 
                 # Get file identifier value from values file
                 file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
@@ -507,7 +467,7 @@ file_identifier=None,output_dir_path=None):
                 csv_file_list.append(csv_filename)
                 output_csv_file = output_target_dir + csv_filename
                 if os.path.isfile(output_csv_file):
-                    raise SystemExit("Target csv file: `" + output_csv_file + "` already exists...")
+                    raise SystemExit("Target csv file: %s already exists.`" % output_csv_file )
 
                 # Add values corresponding to each key to csv target file
                 add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
@@ -519,7 +479,7 @@ file_identifier=None,output_dir_path=None):
                 output_bin_file = output_target_dir + target_file_name_prefix + "-" +\
                 file_identifier_value + ".bin"
                 if os.path.isfile(output_bin_file):
-                    raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...")
+                    raise SystemExit("Target csv file: %s already exists.`" % output_bin_file )
 
                 # Create output csv and bin file
                 print("CSV Generated: ", str(output_csv_file))

From 94f6b1a9ee7290366d94fdd571c8a4325cad943a Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Fri, 12 Oct 2018 10:16:39 +0530
Subject: [PATCH 12/24] util: Update docs as per review comments

---
 components/nvs_flash/nvs_partition_generator/README.rst      | 3 ++-
 .../nvs_flash/nvs_partition_generator/nvs_partition_gen.py   | 4 ++--
 tools/mass_mfg/docs/README.rst                               | 2 +-
 tools/mass_mfg/mfg_gen.py                                    | 5 +++--
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst
index f0c6d6623..bea753864 100644
--- a/components/nvs_flash/nvs_partition_generator/README.rst
+++ b/components/nvs_flash/nvs_partition_generator/README.rst
@@ -116,6 +116,7 @@ A sample CSV file is provided with the utility::
 
     python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1
 
+
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |
 +========================+==============================================================================================+
@@ -123,7 +124,7 @@ A sample CSV file is provided with the utility::
 +------------------------+----------------------------------------------------------------------------------------------+
 | output                 | Path to output converted binary file. Will use stdout if omitted                             |
 +------------------------+----------------------------------------------------------------------------------------------+
-| size                   | Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                          |
+| size                   | Size of NVS Partition in bytes (must be multiple of 4096)                                    |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --version {v1,v2}      |  Set version. Default: v2                                                                    |
 +-------------------------------+---------------------------------------------------------------------------------------+
diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2a01d0744..416c146fc 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -18,7 +18,7 @@
 # limitations under the License.
 #
 
-from __future__ import division, print_function #, unicode_literals
+from __future__ import division, print_function
 from builtins import int, range
 from io import open
 import sys
@@ -732,7 +732,7 @@ def main():
 
     parser.add_argument(
             "size",
-            help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
+            help='Size of NVS Partition in bytes (must be multiple of 4096)')
 
     parser.add_argument(
             "--version",
diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index 8d50aedf8..881fed696 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -120,7 +120,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
-| --size PART_SIZE       |  Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                         |
+| --size PART_SIZE       |  Size of NVS Partition in bytes (must be multiple of 4096)                                   |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 07b2b28e6..c5502161c 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -16,7 +16,7 @@
 #
 
 from __future__ import print_function
-from builtins import map, range
+from builtins import range
 from future.moves.itertools import zip_longest
 import sys
 import os
@@ -27,6 +27,7 @@ import distutils.dir_util
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
 
+
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
     """
@@ -288,7 +289,7 @@ file_identifier=None,output_dir_path=None):
             parser.add_argument("--size",
                                 dest='part_size',
                                 required=True,
-                                help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
+                                help='Size of NVS Partition in bytes (must be multiple of 4096)')
 
             parser.add_argument('--conf',
                                 dest='config_file',

From 17c907b35e913c8608c6104d627c4d08880c3ade Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Tue, 11 Sep 2018 13:43:05 +0530
Subject: [PATCH 13/24] Update mfg utility as per updated nvs partition util

---
 tools/mass_mfg/docs/README.rst | 6 ++++--
 tools/mass_mfg/mfg_gen.py      | 9 ++++++++-
 2 files changed, 12 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 tools/mass_mfg/mfg_gen.py

diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index 4914806d0..f71ec7eac 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -115,11 +115,13 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **Usage**::
 
-    $ ./mfg_gen.py [-h] --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
+    $ ./mfg_gen.py [-h] --size PART_SIZE --conf CONFIG_FILE --values VALUES_FILE --prefix PREFIX [--fileid FILEID] [--outdir OUTDIR]
 
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
+| --size PART_SIZE       |  the size of NVS Partition in KB. Eg. 12KB                                                   |
++------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --values VALUES_FILE   |  the input values csv file                                                                   |
@@ -134,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **You can use the below command to run this utility with the sample files provided**::
    
-   $ ./mfg_gen.py --conf sample_config.csv --values sample_values.csv --prefix Fan
+   $ ./mfg_gen.py --size 12KB --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
 
 
 .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
old mode 100644
new mode 100755
index 9060dde22..56aa61cac
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -284,6 +284,11 @@ file_identifier=None,output_dir_path=None):
                                              description="Create binary files from input config and values file",
                                              formatter_class=argparse.RawDescriptionHelpFormatter)
 
+            parser.add_argument("--size",
+                                dest='part_size',
+                                required=True,
+                                help='Size of NVS Partition in KB. Eg. 12KB')
+            
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,
@@ -320,6 +325,7 @@ file_identifier=None,output_dir_path=None):
             if not args.outdir.endswith('/'):
                 args.outdir = args.outdir + '/'
 
+            input_part_size = args.part_size
             input_config_file = args.config_file
             input_values_file = args.values_file
             target_file_name_prefix = args.prefix
@@ -472,7 +478,8 @@ file_identifier=None,output_dir_path=None):
 
                 # Create output csv and bin file
                 print "CSV Generated: " + str(output_csv_file)
-                nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file)
+                nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
+                                               input_size=input_part_size)
                 print "NVS Flash Binary Generated: " + str(output_bin_file)
 
                 files_created = True

From 807e72adce0d5afd267a954ed6a706864b795c84 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Wed, 3 Oct 2018 17:01:00 +0530
Subject: [PATCH 14/24] mfg_gen: Add changes to make Python2 and Python3
 compatible

---
 tools/mass_mfg/mfg_gen.py | 102 +++++++++++++++++++++++++++-----------
 1 file changed, 73 insertions(+), 29 deletions(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 56aa61cac..a9813ae22 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -15,25 +15,32 @@
 # limitations under the License.
 #
 
+from __future__ import print_function
 import sys
 import os
 import csv
 import argparse
 import shutil
 import distutils.dir_util
+from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
 
-
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
     """
     line_no = 1
     key_count_in_values_file = len(keys_in_values_file)
 
-    values_file = open(input_values_file,'rb')
+    if sys.version_info[0] < 3:
+        values_file = open(input_values_file, 'rb')
+    else:
+        values_file = open(input_values_file, 'r', newline='')
     values_file_reader = csv.reader(values_file, delimiter=',')
-    keys = values_file_reader.next()
+    if sys.version_info[0] < 3:
+        keys = values_file_reader.next()
+    else:
+        keys = next(values_file_reader)
 
     for values_data in values_file_reader:
         line_no +=1
@@ -59,12 +66,11 @@ def verify_keys_exist(values_file_keys, input_config_file):
             else:
                 keys_missing.append([config_data[0], line_no])
 
-    
+
     if keys_missing:
-        print "Oops..."
         for key, line_no in keys_missing:
-            print "Key:`" + str(key) + "` at line no:" + str(line_no) + \
-            " in config file is not found in values file..."
+            print("Key:`", str(key), "` at line no:", str(line_no),\
+            " in config file is not found in values file.")
         config_file.close()
         raise SystemExit(1)
 
@@ -131,7 +137,7 @@ def verify_data_in_file(input_config_file, input_values_file, config_file_keys,
         verify_values_exist(input_values_file, keys_in_values_file)
 
     except StandardError as std_err:
-        print  std_err
+        print(std_err)
     except:
         raise
 
@@ -203,6 +209,7 @@ def add_data_to_file(config_data_to_write, key_value_pair, output_csv_file):
     data_to_write = []
 
     target_csv_file = open(output_csv_file, 'w')
+
     output_file_writer = csv.writer(target_csv_file, delimiter=',')
     output_file_writer.writerow(header)
 
@@ -246,9 +253,17 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
     target_filename = filename + "_created" + file_ext
     with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
         csv_file_reader = csv.reader(read_from, delimiter=',')
-        headers = csv_file_reader.next()
-        values = csv_file_reader.next()
-        total_keys_values = map(None, keys, values)
+        if sys.version_info[0] < 3:
+            headers = csv_file_reader.next()
+            values = csv_file_reader.next()
+        else:
+            headers = next(csv_file_reader)
+            values = next(csv_file_reader)
+
+        if sys.version_info[0] < 3:
+            total_keys_values = map(None, keys, values)
+        else:
+            total_keys_values = list(zip_longest(keys, values))
 
         csv_file_writer = csv.writer(write_to, delimiter=',')
         csv_file_writer.writerow(headers)
@@ -257,7 +272,11 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
         # read new data, add value if key has repeat tag, write to new file
         for row in csv_file_reader:
             index = -1
-            key_val_new = map(None, keys, row)
+            if sys.version_info[0] < 3:
+                key_val_new = map(None, keys, row)
+            else:
+                key_val_new = list(zip_longest(keys, row))
+
             key_val_pair = total_keys_values[:]
             key_repeated = total_keys_repeat[:]
             while key_val_new and key_repeated:
@@ -270,7 +289,7 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
                     del key_repeated[0]
                 del key_val_new[0]
                 del key_val_pair[0]
-           
+
 
     return target_filename
 
@@ -288,7 +307,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in KB. Eg. 12KB')
-            
+
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,
@@ -387,7 +406,7 @@ file_identifier=None,output_dir_path=None):
 
             csv_config_file.close()
         except Exception as e:
-            print e
+            print(e)
         finally:
             csv_config_file.close()
 
@@ -395,7 +414,10 @@ file_identifier=None,output_dir_path=None):
 
 
         # Verify values file does not have empty lines
-        csv_values_file = open(input_values_file,'rb')
+        if sys.version_info[0] < 3:
+            csv_values_file = open(input_values_file,'rb')
+        else:
+            csv_values_file = open(input_values_file,'r', newline='')
         try:
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
             for values_data in values_file_reader:
@@ -414,11 +436,14 @@ file_identifier=None,output_dir_path=None):
             csv_values_file.seek(0)
 
             # Extract keys from values file
-            keys_in_values_file = values_file_reader.next()
+            if sys.version_info[0] < 3:
+                keys_in_values_file = values_file_reader.next()
+            else:
+                keys_in_values_file = next(values_file_reader)
 
             csv_values_file.close()
         except Exception as e:
-            print e
+            print(e)
             exit(1)
         finally:
             csv_values_file.close()
@@ -437,16 +462,35 @@ file_identifier=None,output_dir_path=None):
         config_data_to_write = add_config_data_per_namespace(input_config_file)
 
         try:
-            with open(input_values_file,'rb') as csv_values_file:
-                values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                keys = values_file_reader.next()
+            if sys.version_info[0] < 3:
+                with open(input_values_file,'rb') as csv_values_file:
+                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                    keys = values_file_reader.next()
+            else:
+                with open(input_values_file,'r', newline='') as csv_values_file:
+                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                    keys = next(values_file_reader)
+
             target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
-            csv_values_file = open(target_values_file, 'rb')
+
+            if sys.version_info[0] < 3:
+                csv_values_file = open(target_values_file, 'rb')
+            else:
+                csv_values_file = open(target_values_file, 'r', newline='')
+
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
-            values_file_reader.next()
+            if sys.version_info[0] < 3:
+                values_file_reader.next()
+            else:
+                next(values_file_reader)
+
 
             for values_data_line in values_file_reader:
-                key_value_data = map(None,keys_in_values_file,values_data_line)
+                if sys.version_info[0] < 3:
+                    key_value_data = map(None,keys_in_values_file,values_data_line)
+                else:
+                    key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
+
 
                 # Get file identifier value from values file
                 file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
@@ -477,16 +521,16 @@ file_identifier=None,output_dir_path=None):
                     raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...")
 
                 # Create output csv and bin file
-                print "CSV Generated: " + str(output_csv_file)
+                print("CSV Generated: ", str(output_csv_file))
                 nvs_partition_gen.nvs_part_gen(input_filename = output_csv_file, output_filename = output_bin_file,\
                                                input_size=input_part_size)
-                print "NVS Flash Binary Generated: " + str(output_bin_file)
+                print("NVS Flash Binary Generated: ", str(output_bin_file))
 
                 files_created = True
 
             csv_values_file.close()
         except Exception as e:
-            print e
+            print(e)
             exit(1)
         finally:
             csv_values_file.close()
@@ -494,8 +538,8 @@ file_identifier=None,output_dir_path=None):
 
         return csv_file_list, files_created
 
-    except StandardError as std_err:
-        print std_err
+    except ValueError as err:
+        print(err)
     except:
         raise
 

From 4bc87a414f471c1b23ddd9e93a2295af80374351 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 10:32:09 +0530
Subject: [PATCH 15/24] nvs_part_gen: Update to make python version compatible

(cherry picked from commit 3506b2d60792e5d4ed4f744b28f2da6733c6aae7)
---
 .../nvs_partition_gen.py                      | 199 +++++++++++++-----
 .../nvs_flash/test_nvs_host/test_nvs.cpp      |  13 +-
 2 files changed, 154 insertions(+), 58 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 0f641d612..a1cc81a65 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -28,6 +28,7 @@ import os
 import array
 import csv
 import zlib
+import codecs
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends import default_backend
 
@@ -81,9 +82,9 @@ class Page(object):
         # set page state to active
         page_header= bytearray(b'\xff')*32
         page_state_active_seq = Page.ACTIVE
-        page_header[0:4] = struct.pack('<I', page_state_active_seq)
+        struct.pack_into('<I', page_header, 0,  page_state_active_seq)
         # set page sequence number
-        page_header[4:8] = struct.pack('<I', page_num)
+        struct.pack_into('<I', page_header, 4, page_num)
         # set version
         if version == Page.VERSION2:
             page_header[8] = Page.VERSION2
@@ -91,8 +92,11 @@ class Page(object):
             page_header[8] = Page.VERSION1
         # set header's CRC
         crc_data = page_header[4:28]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        page_header[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
         self.page_buf[0:len(page_header)] = page_header
 
 
@@ -118,8 +122,13 @@ class Page(object):
     def encrypt_entry(self, data_arr, tweak_arr, encr_key):
         # Encrypt 32 bytes of data using AES-XTS encryption
         backend = default_backend()
-        plain_text = data_arr.decode('hex')
-        tweak = tweak_arr.decode('hex')
+        if sys.version_info[0] < 3:
+            plain_text = data_arr.decode('hex')
+            tweak = tweak_arr.decode('hex')
+        else:
+            plain_text = codecs.decode(data_arr, 'hex')
+            tweak = codecs.decode(tweak_arr, 'hex')
+
         cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
         encryptor = cipher.encryptor()
         encrypted_data = encryptor.update(plain_text)
@@ -139,7 +148,7 @@ class Page(object):
 
     def encrypt_data(self, data_input, no_of_entries, nvs_obj):
         # Set values needed for encryption and encrypt data byte wise
-        encr_data_to_write = ''
+        encr_data_to_write = bytearray()
         data_len_needed = 64 #in hex
         tweak_len_needed = 32 #in hex
         init_tweak_val = '0'
@@ -147,17 +156,34 @@ class Page(object):
         tweak_tmp = ''
         encr_key_input = None
 
+
         # Extract encryption key and tweak key from given key input
-        encr_key_input = self.encr_key.decode('hex')
+        if sys.version_info[0] < 3:
+            encr_key_input = self.encr_key.decode('hex')
+        else:
+            encr_key_input = codecs.decode(self.encr_key, 'hex')
+
 
         rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
 
+
         if type(data_input) != bytearray:
-            byte_arr = bytearray('\xff') * 32
-            byte_arr[0:len(data_input)] = data_input
+            byte_arr = bytearray(b'\xff') * 32
+            if sys.version_info[0] < 3:
+                byte_arr[0:len(data_input)] = data_input
+            else:
+                if type(data_input) == str:
+                    byte_arr[0:len(data_input)] = data_input
+                else:
+                    byte_arr[0:len(data_input)] = data_input
+
             data_input = byte_arr
 
-        data_input = binascii.hexlify(bytearray(data_input))
+        
+        if sys.version_info[0] < 3:
+            data_input = binascii.hexlify(bytearray(data_input))
+        else:
+            data_input = data_input.hex()
 
         entry_no = self.entry_num
         start_idx = 0
@@ -182,8 +208,13 @@ class Page(object):
 
             # Encrypt data
             data_bytes = data_input[start_idx:end_idx]
-            data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
+            if sys.version_info[0] < 3:
+                data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
+            else:
+                data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
+
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
+            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -198,13 +229,24 @@ class Page(object):
         encr_data = bytearray()
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            encr_data[0:len(encr_data_ret)] = encr_data_ret
+            if sys.version_info[0] < 3:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            else:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        self.page_buf[start_idx:end_idx]  = data
+        if not sys.version_info[0] < 3:
+            if type(data) == str:
+                self.page_buf[start_idx:end_idx]  = data
+            else:
+                self.page_buf[start_idx:end_idx]  = data
+        else:
+            self.page_buf[start_idx:end_idx]  = data
+
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):
@@ -213,11 +255,14 @@ class Page(object):
 
 
     def set_crc_header(self, entry_struct):
-        crc_data = bytearray(28)
+        crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(crc_data, 0xFFFFFFFF)
+        struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
         return entry_struct
 
     def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
@@ -262,9 +307,13 @@ class Page(object):
             data_chunk =  data[offset:offset + chunk_size]
 
             # Compute CRC of data chunk
-            entry_struct[24:26] = struct.pack('<H', chunk_size)
+            struct.pack_into('<H', entry_struct, 24, chunk_size)
+            if not sys.version_info[0] < 3:
+                if type(data_chunk) == str:
+                    data_chunk = codecs.encode(data_chunk, 'utf8')
+            
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
-            entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+            struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
             # compute crc of entry header
             entry_struct = self.set_crc_header(entry_struct)
@@ -279,7 +328,7 @@ class Page(object):
             if remaining_size or (tailroom - chunk_size) < Page.SINGLE_ENTRY_SIZE:
                 if page_header[0:4] != Page.FULL:
                     page_state_full_seq = Page.FULL
-                    page_header[0:4] = struct.pack('<I', page_state_full_seq)
+                    struct.pack_into('<I', page_header, 0, page_state_full_seq)
                 nvs_obj.create_new_page()
                 self = nvs_obj.cur_page
 
@@ -289,7 +338,7 @@ class Page(object):
             # All chunks are stored, now store the index
             if not remaining_size:
                 # Initialise data field to 0xff
-                data_array = bytearray('\xff')*8
+                data_array = bytearray(b'\xff')*8
                 entry_struct[24:32] = data_array
 
                 # change type of data to BLOB_IDX
@@ -302,7 +351,7 @@ class Page(object):
                 chunk_index = Page.CHUNK_ANY
                 entry_struct[3] = chunk_index
 
-                entry_struct[24:28] = struct.pack('<I', data_size)
+                struct.pack_into('<I', entry_struct, 24, data_size)
                 entry_struct[28] = chunk_count
                 entry_struct[29] = chunk_start
 
@@ -318,9 +367,17 @@ class Page(object):
 
     def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
         # compute CRC of data
-        entry_struct[24:26] = struct.pack('<H', datalen)
-        crc = zlib.crc32(data, 0xFFFFFFFF)
-        entry_struct[28:32] = struct.pack('<I', crc & 0xFFFFFFFF)
+        struct.pack_into('<H', entry_struct, 24, datalen)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(data, 0xFFFFFFFF)
+        else:
+            if (type(data)) == str:
+                data = data.encode('utf8')
+                crc = zlib.crc32(data, 0xFFFFFFFF)
+            else:
+                crc = zlib.crc32(data, 0xFFFFFFFF)
+
+        struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
         # compute crc of entry header
         entry_struct = self.set_crc_header(entry_struct)
@@ -359,7 +416,7 @@ class Page(object):
                 raise PageFullError()
 
         # Entry header
-        entry_struct = bytearray('\xff')*32
+        entry_struct = bytearray(b'\xff')*32
         # Set Namespace Index
         entry_struct[0] = ns_index
         # Set Span
@@ -373,9 +430,12 @@ class Page(object):
             entry_struct[2] = data_entry_count + 1
 
         # set key
-        key_array = bytearray('\x00')*16
+        key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        entry_struct[8:8 + len(key)] = key
+        if sys.version_info[0] < 3:
+            entry_struct[8:8 + len(key)] = key
+        else:
+            entry_struct[8:8 + len(key)] = key.encode('utf8')
 
         # set Type
         if encoding == "string":
@@ -397,39 +457,47 @@ class Page(object):
         if self.entry_num >= Page.PAGE_PARAMS["max_entries"]:
             raise PageFullError()
 
-        entry_struct = bytearray('\xff')*32
+        entry_struct = bytearray(b'\xff')*32
         entry_struct[0] = ns_index # namespace index
         entry_struct[2] = 0x01 # Span
         chunk_index = Page.CHUNK_ANY
         entry_struct[3] = chunk_index
 
         # write key
-        key_array = bytearray('\x00')*16
+        key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        entry_struct[8:8 + len(key)] = key
+        
+        if sys.version_info[0] < 3:
+            entry_struct[8:8 + len(key)] = key
+        else:
+            entry_struct[8:8 + len(key)] = key.encode('utf8')
+
 
         if encoding == "u8":
             entry_struct[1] = Page.U8
-            entry_struct[24] = struct.pack('<B', data)
+            struct.pack_into('<B', entry_struct, 24, data)
         elif encoding == "i8":
             entry_struct[1] = Page.I8
-            entry_struct[24] = struct.pack('<b', data)
+            struct.pack_into('<b', entry_struct, 24, data)
         elif encoding == "u16":
             entry_struct[1] = Page.U16
-            entry_struct[24:26] = struct.pack('<H', data)
+            struct.pack_into('<H', entry_struct, 24, data)
         elif encoding == "u32":
             entry_struct[1] = Page.U32
-            entry_struct[24:28] = struct.pack('<I', data)
+            struct.pack_into('<I', entry_struct, 24, data)
         elif encoding == "i32":
             entry_struct[1] = Page.I32
-            entry_struct[24:28] = struct.pack('<i', data)
+            struct.pack_into('<i', entry_struct, 24, data)
 
         # Compute CRC
-        crc_data = bytearray(28)
+        crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        entry_struct[4:8] = struct.pack('<I', crc & 0xFFFFFFFF)
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
 
         # write to file
         self.write_entry_to_buf(entry_struct, 1,nvs_obj)
@@ -478,7 +546,8 @@ class NVS(object):
         new_page = Page(self.page_num, is_rsrv_page)
         new_page.version = version
         new_page.is_encrypt = is_encrypt_data
-        new_page.encr_key = key_input
+        if new_page.is_encrypt:
+            new_page.encr_key = key_input
         self.pages.append(new_page)
         self.cur_page = new_page
         return new_page
@@ -590,8 +659,12 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
         if os.path.isabs(value) == False:
             script_dir = os.path.dirname(__file__)
             abs_file_path = os.path.join(script_dir, value)
-        with open(abs_file_path, 'rb') as f:
-            value = f.read()
+        if sys.version_info[0] < 3:
+            with open(abs_file_path, 'rb') as f:
+                value = f.read()
+        else:
+            with open(abs_file_path, 'r', newline='') as f:
+                value = f.read()
 
     if datatype == "namespace":
         nvs_instance.write_namespace(key)
@@ -611,9 +684,9 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
 
     :param input_filename: Name of input file containing data
     :param output_filename: Name of output file to store generated binary
-    :param input_size: Size of partition
+    :param input_size: Size of partition (must be multiple of 4096)
     :param key_gen: Enable encryption key generation in encryption mode
-    :param encryption_mode: Enable/Disable encryption mode
+    :param encrypt_mode: Enable/Disable encryption mode
     :param key_file: Input file having encryption keys in encryption mode
     :return: None
     """
@@ -623,16 +696,16 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     is_encrypt_data = encrypt_mode
 
     # Set size
-    input_size = int(input_size.split('KB')[0]) * 1024
+    input_size = int(input_size, 16)
 
     if input_size % 4096 !=0:
-        sys.exit("Size parameter should be a multiple of 4KB.")
+        sys.exit("Size of partition (must be multiple of 4096)")
 
     if version == 'v1':
         version = Page.VERSION1
     elif version == 'v2':
         version = Page.VERSION2
-    
+
     # Update size as a page needs to be reserved of size 4KB
     input_size = input_size - Page.PAGE_PARAMS["max_size"]
 
@@ -662,13 +735,20 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
 
     if key_gen:
-        key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
+        if sys.version_info[0] < 3:
+            key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
+        else:
+            key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
     elif key_file:
         with open(key_file) as key_f:
             key_input = key_f.readline()
             key_input = key_input.strip()
 
-    input_file = open(input_filename, 'rb')
+    if sys.version_info[0] < 3:
+        input_file = open(input_filename, 'rb')
+    else:
+        input_file = open(input_filename, 'r', newline='')
+    
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -688,12 +768,25 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     if is_encrypt_data:
         output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
-        key_bytes = key_input.decode('hex')
+        
+        if sys.version_info[0] < 3:
+            key_bytes = key_input.decode('hex')
+        else:
+            key_bytes = bytearray()
+            key_bytes = codecs.decode(key_input, 'hex') 
+
+        
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
+        
         crc_data = keys_page_buf[0:key_len]
-        crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        keys_page_buf[64:68] = struct.pack('<I', crc & 0xFFFFFFFF)
+        
+        if sys.version_info[0] < 3:
+            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
+        else:
+            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        
+        struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
         output_keys_file.write(keys_page_buf)
 
 
@@ -712,7 +805,7 @@ def main():
 
     parser.add_argument(
             "size",
-            help='Size of NVS Partition in KB. Eg. 12KB')
+            help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
 
     parser.add_argument(
             "--version",
diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp
index da06e328f..aeeb3188c 100644
--- a/components/nvs_flash/test_nvs_host/test_nvs.cpp
+++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp
@@ -2003,7 +2003,7 @@ TEST_CASE("check partition generation utility with multipage blob support disabl
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_singlepage_blob.csv",
                 "../nvs_partition_generator/partition_single_page.bin", 
-                "12KB",
+                "0x3000",
                 "--version",
                 "v1",NULL));
     } else {
@@ -2071,6 +2071,7 @@ TEST_CASE("read data from partition generated via partition generation utility w
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
 
     file.close();
+    
     nvs_close(handle);
 }
 
@@ -2082,7 +2083,7 @@ TEST_CASE("check partition generation utility with multipage blob support enable
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_multipage_blob.csv",
                 "../nvs_partition_generator/partition_multipage_blob.bin", 
-                "12KB",
+                "0x3000",
                 "--version",
                 "v2",NULL));
     } else {
@@ -2148,8 +2149,9 @@ TEST_CASE("read data from partition generated via partition generation utility w
     file.read(binfiledata,5200);
     TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
-
     file.close();
+    
+    nvs_close(handle);
 
 }
 
@@ -2300,7 +2302,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
                 "../nvs_partition_generator/nvs_partition_gen.py",
                 "../nvs_partition_generator/sample_multipage_blob.csv",
                 "../nvs_partition_generator/partition_encrypted.bin",
-                "12KB",
+                "0x3000",
                 "--encrypt",
                 "True",
                 "--keyfile",
@@ -2329,6 +2331,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
     uint8_t u8v;
     TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v));
     CHECK(u8v == 127);
+    
     int8_t i8v;
     TEST_ESP_OK( nvs_get_i8(handle, "dummyI8Key", &i8v));
     CHECK(i8v == -128);
@@ -2380,7 +2383,7 @@ TEST_CASE("test nvs apis for nvs partition generator utility with encryption ena
     file.read(binfiledata,5120);
     TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len));
     CHECK(memcmp(bin_data, binfiledata, bin_len) == 0);
-
+    
     nvs_close(handle);
     TEST_ESP_OK(nvs_flash_deinit());
 

From fb31788504f035b9577ce230eee4e49ddc7b3baf Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 11:11:56 +0530
Subject: [PATCH 16/24] nvs_part_gen: Add changes to partition size input

---
 .../nvs_partition_generator/README.rst        |  2 +-
 .../nvs_partition_gen.py                      | 23 +++++++++----------
 tools/mass_mfg/docs/README.rst                |  4 ++--
 tools/mass_mfg/mfg_gen.py                     |  5 ++--
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst
index aa7a945c2..f0c6d6623 100644
--- a/components/nvs_flash/nvs_partition_generator/README.rst
+++ b/components/nvs_flash/nvs_partition_generator/README.rst
@@ -123,7 +123,7 @@ A sample CSV file is provided with the utility::
 +------------------------+----------------------------------------------------------------------------------------------+
 | output                 | Path to output converted binary file. Will use stdout if omitted                             |
 +------------------------+----------------------------------------------------------------------------------------------+
-| size                   | Size of NVS Partition in KB. Eg. 12KB                                                        |
+| size                   | Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                          |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --version {v1,v2}      |  Set version. Default: v2                                                                    |
 +-------------------------------+---------------------------------------------------------------------------------------+
diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index a1cc81a65..2b11877ca 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -179,7 +179,7 @@ class Page(object):
 
             data_input = byte_arr
 
-        
+
         if sys.version_info[0] < 3:
             data_input = binascii.hexlify(bytearray(data_input))
         else:
@@ -214,7 +214,6 @@ class Page(object):
                 data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
 
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
-            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -233,7 +232,7 @@ class Page(object):
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
             else:
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
-            
+
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
@@ -311,7 +310,7 @@ class Page(object):
             if not sys.version_info[0] < 3:
                 if type(data_chunk) == str:
                     data_chunk = codecs.encode(data_chunk, 'utf8')
-            
+
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
             struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
@@ -466,7 +465,7 @@ class Page(object):
         # write key
         key_array = bytearray(b'\x00')*16
         entry_struct[8:24] = key_array
-        
+
         if sys.version_info[0] < 3:
             entry_struct[8:8 + len(key)] = key
         else:
@@ -748,7 +747,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         input_file = open(input_filename, 'rb')
     else:
         input_file = open(input_filename, 'r', newline='')
-    
+
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -768,24 +767,24 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     if is_encrypt_data:
         output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
-        
+
         if sys.version_info[0] < 3:
             key_bytes = key_input.decode('hex')
         else:
             key_bytes = bytearray()
-            key_bytes = codecs.decode(key_input, 'hex') 
+            key_bytes = codecs.decode(key_input, 'hex')
+
 
-        
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
-        
+
         crc_data = keys_page_buf[0:key_len]
-        
+
         if sys.version_info[0] < 3:
             crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
         else:
             crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
-        
+
         struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
         output_keys_file.write(keys_page_buf)
 
diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index f71ec7eac..8d50aedf8 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -120,7 +120,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
-| --size PART_SIZE       |  the size of NVS Partition in KB. Eg. 12KB                                                   |
+| --size PART_SIZE       |  Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                         |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
@@ -136,7 +136,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 
 **You can use the below command to run this utility with the sample files provided**::
    
-   $ ./mfg_gen.py --size 12KB --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
+   $ ./mfg_gen.py --size 0x3000 --conf samples/sample_config.csv --values samples/sample_values.csv --prefix Fan
 
 
 .. note:: The default numeric value: 1,2,3... of ``fileid`` argument, corresponds to each row having device instance values in master csv values file.
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index a9813ae22..ca9c94cb0 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,9 +22,10 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
-from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
+if not sys.version_info[0] < 3:
+    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -306,7 +307,7 @@ file_identifier=None,output_dir_path=None):
             parser.add_argument("--size",
                                 dest='part_size',
                                 required=True,
-                                help='Size of NVS Partition in KB. Eg. 12KB')
+                                help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
 
             parser.add_argument('--conf',
                                 dest='config_file',

From 1460f03065e1496fdb3661d9c4bfd386afd1bea6 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 11:49:32 +0530
Subject: [PATCH 17/24] nvs_util: Add Python2 and Python3 compatible

---
 .../nvs_partition_gen.py                      | 171 +++++-------------
 1 file changed, 50 insertions(+), 121 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2b11877ca..2330c6102 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -17,8 +17,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from __future__ import division
-from __future__ import print_function
+
+from __future__ import division, print_function #, unicode_literals
+from future.utils import raise_
+from builtins import int, range
+from io import open
 import sys
 import argparse
 import binascii
@@ -80,7 +83,7 @@ class Page(object):
         global page_header
 
         # set page state to active
-        page_header= bytearray(b'\xff')*32
+        page_header= bytearray(b'\xff') *32
         page_state_active_seq = Page.ACTIVE
         struct.pack_into('<I', page_header, 0,  page_state_active_seq)
         # set page sequence number
@@ -91,11 +94,8 @@ class Page(object):
         elif version == Page.VERSION1:
             page_header[8] = Page.VERSION1
         # set header's CRC
-        crc_data = page_header[4:28]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(page_header[4:28])
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', page_header, 28, crc & 0xFFFFFFFF)
         self.page_buf[0:len(page_header)] = page_header
 
@@ -122,12 +122,8 @@ class Page(object):
     def encrypt_entry(self, data_arr, tweak_arr, encr_key):
         # Encrypt 32 bytes of data using AES-XTS encryption
         backend = default_backend()
-        if sys.version_info[0] < 3:
-            plain_text = data_arr.decode('hex')
-            tweak = tweak_arr.decode('hex')
-        else:
-            plain_text = codecs.decode(data_arr, 'hex')
-            tweak = codecs.decode(tweak_arr, 'hex')
+        plain_text = codecs.decode(data_arr, 'hex')
+        tweak = codecs.decode(tweak_arr, 'hex')
 
         cipher = Cipher(algorithms.AES(encr_key), modes.XTS(tweak), backend=backend)
         encryptor = cipher.encryptor()
@@ -158,32 +154,16 @@ class Page(object):
 
 
         # Extract encryption key and tweak key from given key input
-        if sys.version_info[0] < 3:
-            encr_key_input = self.encr_key.decode('hex')
-        else:
-            encr_key_input = codecs.decode(self.encr_key, 'hex')
-
+        encr_key_input = codecs.decode(self.encr_key, 'hex')
 
         rel_addr = nvs_obj.page_num * Page.PAGE_PARAMS["max_size"] + Page.FIRST_ENTRY_OFFSET
 
-
-        if type(data_input) != bytearray:
+        if not isinstance(data_input, bytearray):
             byte_arr = bytearray(b'\xff') * 32
-            if sys.version_info[0] < 3:
-                byte_arr[0:len(data_input)] = data_input
-            else:
-                if type(data_input) == str:
-                    byte_arr[0:len(data_input)] = data_input
-                else:
-                    byte_arr[0:len(data_input)] = data_input
-
+            byte_arr[0:len(data_input)] = data_input
             data_input = byte_arr
 
-
-        if sys.version_info[0] < 3:
-            data_input = binascii.hexlify(bytearray(data_input))
-        else:
-            data_input = data_input.hex()
+        data_input = binascii.hexlify(data_input)
 
         entry_no = self.entry_num
         start_idx = 0
@@ -208,11 +188,10 @@ class Page(object):
 
             # Encrypt data
             data_bytes = data_input[start_idx:end_idx]
-            if sys.version_info[0] < 3:
-                data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
-            else:
-                data_val = str(data_bytes) + (init_data_val * (data_len_needed - len(data_bytes)))
+            if type(data_bytes) == bytes:
+                data_bytes = data_bytes.decode()
 
+            data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
@@ -220,32 +199,21 @@ class Page(object):
             end_idx = start_idx + 64
             entry_no += 1
 
-
         return encr_data_to_write
 
 
     def write_entry_to_buf(self, data, entrycount,nvs_obj):
         encr_data = bytearray()
+
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            if sys.version_info[0] < 3:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-            else:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-
+            encr_data[0:len(encr_data_ret)] = encr_data_ret
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        if not sys.version_info[0] < 3:
-            if type(data) == str:
-                self.page_buf[start_idx:end_idx]  = data
-            else:
-                self.page_buf[start_idx:end_idx]  = data
-        else:
-            self.page_buf[start_idx:end_idx]  = data
-
+        self.page_buf[start_idx:end_idx]  = data
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):
@@ -257,13 +225,12 @@ class Page(object):
         crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(crc_data, 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
         return entry_struct
 
+
     def write_varlen_binary_data(self, entry_struct, ns_index, key, data, data_size, total_entry_count,nvs_obj):
         chunk_start = 0
         chunk_count = 0
@@ -307,10 +274,7 @@ class Page(object):
 
             # Compute CRC of data chunk
             struct.pack_into('<H', entry_struct, 24, chunk_size)
-            if not sys.version_info[0] < 3:
-                if type(data_chunk) == str:
-                    data_chunk = codecs.encode(data_chunk, 'utf8')
-
+            data_chunk = bytes(data_chunk)
             crc = zlib.crc32(data_chunk, 0xFFFFFFFF)
             struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
@@ -367,15 +331,9 @@ class Page(object):
     def write_single_page_entry(self, entry_struct, data, datalen, data_entry_count, nvs_obj):
         # compute CRC of data
         struct.pack_into('<H', entry_struct, 24, datalen)
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(data, 0xFFFFFFFF)
-        else:
-            if (type(data)) == str:
-                data = data.encode('utf8')
-                crc = zlib.crc32(data, 0xFFFFFFFF)
-            else:
-                crc = zlib.crc32(data, 0xFFFFFFFF)
-
+        if not type(data) == bytes:
+            data = data.encode()
+        crc = zlib.crc32(data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 28, crc & 0xFFFFFFFF)
 
         # compute crc of entry header
@@ -429,12 +387,9 @@ class Page(object):
             entry_struct[2] = data_entry_count + 1
 
         # set key
-        key_array = bytearray(b'\x00')*16
+        key_array = b'\x00' * 16
         entry_struct[8:24] = key_array
-        if sys.version_info[0] < 3:
-            entry_struct[8:8 + len(key)] = key
-        else:
-            entry_struct[8:8 + len(key)] = key.encode('utf8')
+        entry_struct[8:8 + len(key)] = key.encode()
 
         # set Type
         if encoding == "string":
@@ -463,14 +418,9 @@ class Page(object):
         entry_struct[3] = chunk_index
 
         # write key
-        key_array = bytearray(b'\x00')*16
+        key_array = b'\x00' *16
         entry_struct[8:24] = key_array
-
-        if sys.version_info[0] < 3:
-            entry_struct[8:8 + len(key)] = key
-        else:
-            entry_struct[8:8 + len(key)] = key.encode('utf8')
-
+        entry_struct[8:8 + len(key)] = key.encode()
 
         if encoding == "u8":
             entry_struct[1] = Page.U8
@@ -492,10 +442,8 @@ class Page(object):
         crc_data = bytearray(b'28')
         crc_data[0:4] = entry_struct[0:4]
         crc_data[4:28] = entry_struct[8:32]
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
         struct.pack_into('<I', entry_struct, 4, crc & 0xFFFFFFFF)
 
         # write to file
@@ -562,7 +510,6 @@ class NVS(object):
         except PageFullError:
             new_page = self.create_new_page()
             new_page.write_primitive_data(key, self.namespace_idx, "u8", 0,self)
-            pass
 
     """
     Write key-value pair. Function accepts value in the form of ascii character and converts
@@ -580,6 +527,8 @@ class NVS(object):
             value = binascii.a2b_base64(value)
 
         if encoding == "string":
+            if type(value) == bytes:
+                value = value.decode()
             value += '\0'
 
         encoding = encoding.lower()
@@ -591,15 +540,12 @@ class NVS(object):
             except PageFullError:
                 new_page = self.create_new_page()
                 new_page.write_varlen_data(key, value, encoding, self.namespace_idx,self)
-                pass
         elif encoding in primitive_encodings:
             try:
                 self.cur_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
             except PageFullError:
                 new_page = self.create_new_page()
                 new_page.write_primitive_data(key, int(value), encoding, self.namespace_idx,self)
-                sys.exc_clear()
-                pass
         else:
             raise InputError("%s: Unsupported encoding" % encoding)
 
@@ -658,12 +604,9 @@ def write_entry(nvs_instance, key, datatype, encoding, value):
         if os.path.isabs(value) == False:
             script_dir = os.path.dirname(__file__)
             abs_file_path = os.path.join(script_dir, value)
-        if sys.version_info[0] < 3:
-            with open(abs_file_path, 'rb') as f:
-                value = f.read()
-        else:
-            with open(abs_file_path, 'r', newline='') as f:
-                value = f.read()
+
+        with open(abs_file_path, 'rb') as f:
+            value = f.read()
 
     if datatype == "namespace":
         nvs_instance.write_namespace(key)
@@ -683,7 +626,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
 
     :param input_filename: Name of input file containing data
     :param output_filename: Name of output file to store generated binary
-    :param input_size: Size of partition (must be multiple of 4096)
+    :param input_size: Size of partition in bytes (must be multiple of 4096)
     :param key_gen: Enable encryption key generation in encryption mode
     :param encrypt_mode: Enable/Disable encryption mode
     :param key_file: Input file having encryption keys in encryption mode
@@ -695,7 +638,7 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     is_encrypt_data = encrypt_mode
 
     # Set size
-    input_size = int(input_size, 16)
+    input_size = int(input_size, 0)
 
     if input_size % 4096 !=0:
         sys.exit("Size of partition (must be multiple of 4096)")
@@ -734,20 +677,13 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
         sys.exit("Invalid. Cannot give --key_file as --encrypt is set to False.")
 
     if key_gen:
-        if sys.version_info[0] < 3:
-            key_input = ''.join(random.choice('0123456789abcdef') for _ in xrange(128))
-        else:
-            key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
+        key_input = ''.join(random.choice('0123456789abcdef') for _ in range(128)).strip()
     elif key_file:
-        with open(key_file) as key_f:
+        with open(key_file, 'rt', encoding='utf8') as key_f:
             key_input = key_f.readline()
             key_input = key_input.strip()
 
-    if sys.version_info[0] < 3:
-        input_file = open(input_filename, 'rb')
-    else:
-        input_file = open(input_filename, 'r', newline='')
-
+    input_file = open(input_filename, 'rt', encoding='utf8')
     output_file = open(output_filename, 'wb')
 
     with nvs_open(output_file, input_size) as nvs_obj:
@@ -765,28 +701,21 @@ def nvs_part_gen(input_filename=None, output_filename=None, input_size=None, key
     output_file.close()
 
     if is_encrypt_data:
-        output_keys_file = open("encryption_keys.bin",'wb')
         keys_page_buf = bytearray(b'\xff')*Page.PAGE_PARAMS["max_size"]
+        key_bytes = bytearray()
 
-        if sys.version_info[0] < 3:
-            key_bytes = key_input.decode('hex')
-        else:
-            key_bytes = bytearray()
-            key_bytes = codecs.decode(key_input, 'hex')
-
-
+        key_bytes = codecs.decode(key_input, 'hex')
         key_len = len(key_bytes)
         keys_page_buf[0:key_len] = key_bytes
 
         crc_data = keys_page_buf[0:key_len]
-
-        if sys.version_info[0] < 3:
-            crc = zlib.crc32(buffer(crc_data), 0xFFFFFFFF)
-        else:
-            crc = zlib.crc32(memoryview(crc_data), 0xFFFFFFFF)
+        crc_data = bytes(crc_data)
+        crc = zlib.crc32(crc_data, 0xFFFFFFFF)
 
         struct.pack_into('<I', keys_page_buf, key_len,  crc & 0xFFFFFFFF)
-        output_keys_file.write(keys_page_buf)
+
+        with open("encryption_keys.bin",'wb') as output_keys_file:
+            output_keys_file.write(keys_page_buf)
 
 
 

From 1f49e64730bef7ee6aa37bdb61c85f043116af45 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Tue, 11 Sep 2018 13:43:05 +0530
Subject: [PATCH 18/24] Update mfg utility as per updated nvs partition util

---
 tools/mass_mfg/mfg_gen.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index ca9c94cb0..c02c0e156 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -308,7 +308,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
-
+            
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,

From 302d339640f06a382ea23b715456af7d08687b3b Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Wed, 3 Oct 2018 17:01:00 +0530
Subject: [PATCH 19/24] mfg_gen: Add changes to make Python2 and Python3
 compatible

---
 tools/mass_mfg/mfg_gen.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index c02c0e156..255aa50b1 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,10 +22,9 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
+from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
-if not sys.version_info[0] < 3:
-    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file

From 60df21115ed59f2cfebe4a6463db8bdceb9b901c Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 10:32:09 +0530
Subject: [PATCH 20/24] nvs_part_gen: Update to make python version compatible

(cherry picked from commit 3506b2d60792e5d4ed4f744b28f2da6733c6aae7)
---
 .../nvs_partition_generator/nvs_partition_gen.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2330c6102..d0c529791 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -193,6 +193,7 @@ class Page(object):
 
             data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
+            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -207,13 +208,24 @@ class Page(object):
 
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            encr_data[0:len(encr_data_ret)] = encr_data_ret
+            if sys.version_info[0] < 3:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            else:
+                encr_data[0:len(encr_data_ret)] = encr_data_ret
+            
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        self.page_buf[start_idx:end_idx]  = data
+        if not sys.version_info[0] < 3:
+            if type(data) == str:
+                self.page_buf[start_idx:end_idx]  = data
+            else:
+                self.page_buf[start_idx:end_idx]  = data
+        else:
+            self.page_buf[start_idx:end_idx]  = data
+
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):

From 0f6bcec2bc442159b1a0335fc04cb2a526870f64 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Mon, 8 Oct 2018 11:11:56 +0530
Subject: [PATCH 21/24] nvs_part_gen: Add changes to partition size input

---
 .../nvs_flash/nvs_partition_generator/nvs_partition_gen.py   | 3 +--
 tools/mass_mfg/mfg_gen.py                                    | 5 +++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index d0c529791..003d1bab9 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -193,7 +193,6 @@ class Page(object):
 
             data_val = data_bytes + (init_data_val * (data_len_needed - len(data_bytes)))
             encr_data_ret = self.encrypt_entry(data_val, tweak_val, encr_key_input)
-            #print("\n<<<\n")
             encr_data_to_write = encr_data_to_write + encr_data_ret
             # Update values for encrypting next set of data bytes
             start_idx = end_idx
@@ -212,7 +211,7 @@ class Page(object):
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
             else:
                 encr_data[0:len(encr_data_ret)] = encr_data_ret
-            
+
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 255aa50b1..ca9c94cb0 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -22,9 +22,10 @@ import csv
 import argparse
 import shutil
 import distutils.dir_util
-from itertools import zip_longest
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
+if not sys.version_info[0] < 3:
+    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -307,7 +308,7 @@ file_identifier=None,output_dir_path=None):
                                 dest='part_size',
                                 required=True,
                                 help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
-            
+
             parser.add_argument('--conf',
                                 dest='config_file',
                                 required=True,

From 1d38d8a17fed8f4ea5d200d5bb8fe0a509aba67b Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 11:49:32 +0530
Subject: [PATCH 22/24] nvs_util: Add Python2 and Python3 compatible

---
 .../nvs_partition_generator/nvs_partition_gen.py  | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 003d1bab9..2330c6102 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -207,24 +207,13 @@ class Page(object):
 
         if self.is_encrypt:
             encr_data_ret = self.encrypt_data(data, entrycount,nvs_obj)
-            if sys.version_info[0] < 3:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-            else:
-                encr_data[0:len(encr_data_ret)] = encr_data_ret
-
+            encr_data[0:len(encr_data_ret)] = encr_data_ret
             data = encr_data
 
         data_offset = Page.FIRST_ENTRY_OFFSET + (Page.SINGLE_ENTRY_SIZE * self.entry_num)
         start_idx = data_offset
         end_idx = data_offset + len(data)
-        if not sys.version_info[0] < 3:
-            if type(data) == str:
-                self.page_buf[start_idx:end_idx]  = data
-            else:
-                self.page_buf[start_idx:end_idx]  = data
-        else:
-            self.page_buf[start_idx:end_idx]  = data
-
+        self.page_buf[start_idx:end_idx]  = data
 
         # Set bitmap array for entries in current page
         for i in range(0, entrycount):

From 5bee9faf49ce1f4aabb194507cf3f13ce3b15409 Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Thu, 11 Oct 2018 17:50:12 +0530
Subject: [PATCH 23/24] tools: Add Python2 and Python3 compatibility support to
 nvs util tool and mfg tool

---
 .../nvs_partition_gen.py                      |   1 -
 tools/mass_mfg/mfg_gen.py                     | 108 ++++++------------
 2 files changed, 34 insertions(+), 75 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2330c6102..2a01d0744 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -19,7 +19,6 @@
 #
 
 from __future__ import division, print_function #, unicode_literals
-from future.utils import raise_
 from builtins import int, range
 from io import open
 import sys
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index ca9c94cb0..07b2b28e6 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -16,6 +16,8 @@
 #
 
 from __future__ import print_function
+from builtins import map, range
+from future.moves.itertools import zip_longest
 import sys
 import os
 import csv
@@ -24,8 +26,6 @@ import shutil
 import distutils.dir_util
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
-if not sys.version_info[0] < 3:
-    from itertools import zip_longest
 
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
@@ -33,21 +33,15 @@ def verify_values_exist(input_values_file, keys_in_values_file):
     line_no = 1
     key_count_in_values_file = len(keys_in_values_file)
 
-    if sys.version_info[0] < 3:
-        values_file = open(input_values_file, 'rb')
-    else:
-        values_file = open(input_values_file, 'r', newline='')
+    values_file = open(input_values_file, 'r')
     values_file_reader = csv.reader(values_file, delimiter=',')
-    if sys.version_info[0] < 3:
-        keys = values_file_reader.next()
-    else:
-        keys = next(values_file_reader)
+    keys = next(values_file_reader)
 
     for values_data in values_file_reader:
         line_no +=1
         if len(values_data) != key_count_in_values_file:
-            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: '" + \
-            str(input_values_file) + "' at line No:" + str(line_no) )
+            raise SystemExit("\nOops...Number of values is not equal to number of keys in file: %s at line No:%s\n"\
+            % (str(input_values_file), str(line_no)))
 
 
 def verify_keys_exist(values_file_keys, input_config_file):
@@ -91,12 +85,12 @@ def verify_datatype_encoding(input_config_file):
     for config_data in config_file_reader:
         line_no+=1
         if config_data[1] not in valid_datatypes:
-            raise SystemExit("Oops...config file: `" + str(input_config_file) + \
-            "` has invalid datatype at line no:" + str(line_no))
+            raise SystemExit("Oops...config file: %s has invalid datatype at line no:%s\n`" \
+            % (str(input_config_file), str(line_no)))
         if 'namespace' not in config_data:
             if config_data[2] not in valid_encodings:
-                raise SystemExit("Oops...config file: `" + str(input_config_file) + \
-                "` has invalid encoding at line no:" + str(line_no))
+                raise SystemExit("Oops...config file: %s has invalid encoding at line no:%s\n`" \
+                % (str(input_config_file), str(line_no)))
 
 
 
@@ -110,9 +104,8 @@ def verify_file_data_count(input_config_file, keys_repeat):
     for line in config_file_reader:
         line_no += 1
         if len(line) != 3 and line[0] not in keys_repeat:
-            raise SystemExit("Oops...data missing in config file at line no: " + str(line_no) + \
-            " <format needed:key,type,encoding>")
-
+            raise SystemExit("Oops...data missing in config file at line no:%s <format needed:key,type,encoding>\n" \
+            % str(line_no) )
     config_file.close()
 
 
@@ -254,29 +247,17 @@ def set_repeat_value(total_keys_repeat, keys, csv_file):
     target_filename = filename + "_created" + file_ext
     with open(csv_file, 'r') as read_from, open(target_filename,'w') as write_to:
         csv_file_reader = csv.reader(read_from, delimiter=',')
-        if sys.version_info[0] < 3:
-            headers = csv_file_reader.next()
-            values = csv_file_reader.next()
-        else:
-            headers = next(csv_file_reader)
-            values = next(csv_file_reader)
-
-        if sys.version_info[0] < 3:
-            total_keys_values = map(None, keys, values)
-        else:
-            total_keys_values = list(zip_longest(keys, values))
-
+        headers = next(csv_file_reader)
+        values = next(csv_file_reader)
         csv_file_writer = csv.writer(write_to, delimiter=',')
         csv_file_writer.writerow(headers)
         csv_file_writer.writerow(values)
+        total_keys_values = list(zip_longest(keys, values))
 
         # read new data, add value if key has repeat tag, write to new file
         for row in csv_file_reader:
             index = -1
-            if sys.version_info[0] < 3:
-                key_val_new = map(None, keys, row)
-            else:
-                key_val_new = list(zip_longest(keys, row))
+            key_val_new = list(zip_longest(keys, row))
 
             key_val_pair = total_keys_values[:]
             key_repeated = total_keys_repeat[:]
@@ -371,11 +352,11 @@ file_identifier=None,output_dir_path=None):
 
         # Verify config file is not empty
         if os.stat(input_config_file).st_size == 0:
-            raise SystemExit("Oops...config file: " + input_config_file + " is empty...")
+            raise SystemExit("Oops...config file: %s is empty." % input_config_file)
 
         # Verify values file is not empty
         if os.stat(input_values_file).st_size == 0:
-            raise SystemExit("Oops...values file: " + input_values_file + " is empty...")
+            raise SystemExit("Oops...values file: %s is empty." % input_values_file )
 
         # Verify config file does not have empty lines
         csv_config_file = open(input_config_file,'r')
@@ -390,9 +371,9 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...config file: %s cannot have empty lines. " % input_config_file )
                 if not config_data:
-                    raise SystemExit("Oops...config file: " + input_config_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...config file: %s cannot have empty lines." % input_config_file )
 
             csv_config_file.seek(0)
 
@@ -415,10 +396,7 @@ file_identifier=None,output_dir_path=None):
 
 
         # Verify values file does not have empty lines
-        if sys.version_info[0] < 3:
-            csv_values_file = open(input_values_file,'rb')
-        else:
-            csv_values_file = open(input_values_file,'r', newline='')
+        csv_values_file = open(input_values_file, 'r')
         try:
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
             for values_data in values_file_reader:
@@ -430,17 +408,14 @@ file_identifier=None,output_dir_path=None):
                         is_empty_line = False
                         break
                 if is_empty_line:
-                    raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
                 if not values_data:
-                    raise SystemExit("Oops...values file: " + input_values_file + " cannot have empty lines...")
+                    raise SystemExit("Oops...values file: %s cannot have empty lines." % input_values_file )
 
             csv_values_file.seek(0)
 
             # Extract keys from values file
-            if sys.version_info[0] < 3:
-                keys_in_values_file = values_file_reader.next()
-            else:
-                keys_in_values_file = next(values_file_reader)
+            keys_in_values_file = next(values_file_reader)
 
             csv_values_file.close()
         except Exception as e:
@@ -452,8 +427,8 @@ file_identifier=None,output_dir_path=None):
         # Verify file identifier exists in values file
         if file_identifier:
             if file_identifier not in keys_in_values_file:
-                raise SystemExit('Oops...target_file_identifier: ' + file_identifier + \
-                ' does not exist in values file...\n')
+                raise SystemExit('Oops...target_file_identifier: %s does not exist in values file.\n' % file_identifier )
+
 
         # Verify data in the input_config_file and input_values_file
         verify_data_in_file(input_config_file, input_values_file, keys_in_config_file,\
@@ -463,35 +438,20 @@ file_identifier=None,output_dir_path=None):
         config_data_to_write = add_config_data_per_namespace(input_config_file)
 
         try:
-            if sys.version_info[0] < 3:
-                with open(input_values_file,'rb') as csv_values_file:
-                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                    keys = values_file_reader.next()
-            else:
-                with open(input_values_file,'r', newline='') as csv_values_file:
-                    values_file_reader = csv.reader(csv_values_file, delimiter=',')
-                    keys = next(values_file_reader)
+            with open(input_values_file, 'r') as csv_values_file:
+                values_file_reader = csv.reader(csv_values_file, delimiter=',')
+                keys = next(values_file_reader)
 
             target_values_file = set_repeat_value(keys_repeat, keys, input_values_file)
 
-            if sys.version_info[0] < 3:
-                csv_values_file = open(target_values_file, 'rb')
-            else:
-                csv_values_file = open(target_values_file, 'r', newline='')
+            csv_values_file = open(target_values_file, 'r')
 
             values_file_reader = csv.reader(csv_values_file, delimiter=',')
-            if sys.version_info[0] < 3:
-                values_file_reader.next()
-            else:
-                next(values_file_reader)
+            next(values_file_reader)
 
 
             for values_data_line in values_file_reader:
-                if sys.version_info[0] < 3:
-                    key_value_data = map(None,keys_in_values_file,values_data_line)
-                else:
-                    key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
-
+                key_value_data = list(zip_longest(keys_in_values_file,values_data_line))
 
                 # Get file identifier value from values file
                 file_identifier_value = get_fileid_val(file_identifier, keys_in_config_file, \
@@ -507,7 +467,7 @@ file_identifier=None,output_dir_path=None):
                 csv_file_list.append(csv_filename)
                 output_csv_file = output_target_dir + csv_filename
                 if os.path.isfile(output_csv_file):
-                    raise SystemExit("Target csv file: `" + output_csv_file + "` already exists...")
+                    raise SystemExit("Target csv file: %s already exists.`" % output_csv_file )
 
                 # Add values corresponding to each key to csv target file
                 add_data_to_file(config_data_to_write, key_value_pair, output_csv_file)
@@ -519,7 +479,7 @@ file_identifier=None,output_dir_path=None):
                 output_bin_file = output_target_dir + target_file_name_prefix + "-" +\
                 file_identifier_value + ".bin"
                 if os.path.isfile(output_bin_file):
-                    raise SystemExit("Target csv file: `" + output_bin_file + "` already exists...")
+                    raise SystemExit("Target csv file: %s already exists.`" % output_bin_file )
 
                 # Create output csv and bin file
                 print("CSV Generated: ", str(output_csv_file))

From 7c08fcccafb22d8e5e1a571d1df7d80de61ea94c Mon Sep 17 00:00:00 2001
From: Shivani Tipnis <shivani@espressif.com>
Date: Fri, 12 Oct 2018 10:16:39 +0530
Subject: [PATCH 24/24] util: Update docs as per review comments

---
 components/nvs_flash/nvs_partition_generator/README.rst      | 3 ++-
 .../nvs_flash/nvs_partition_generator/nvs_partition_gen.py   | 4 ++--
 tools/mass_mfg/docs/README.rst                               | 2 +-
 tools/mass_mfg/mfg_gen.py                                    | 5 +++--
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/components/nvs_flash/nvs_partition_generator/README.rst b/components/nvs_flash/nvs_partition_generator/README.rst
index f0c6d6623..bea753864 100644
--- a/components/nvs_flash/nvs_partition_generator/README.rst
+++ b/components/nvs_flash/nvs_partition_generator/README.rst
@@ -116,6 +116,7 @@ A sample CSV file is provided with the utility::
 
     python nvs_partition_gen.py sample_singlepage_blob.csv partition_single_page.bin --version v1
 
+
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |
 +========================+==============================================================================================+
@@ -123,7 +124,7 @@ A sample CSV file is provided with the utility::
 +------------------------+----------------------------------------------------------------------------------------------+
 | output                 | Path to output converted binary file. Will use stdout if omitted                             |
 +------------------------+----------------------------------------------------------------------------------------------+
-| size                   | Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                          |
+| size                   | Size of NVS Partition in bytes (must be multiple of 4096)                                    |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --version {v1,v2}      |  Set version. Default: v2                                                                    |
 +-------------------------------+---------------------------------------------------------------------------------------+
diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
index 2a01d0744..416c146fc 100755
--- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
+++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
@@ -18,7 +18,7 @@
 # limitations under the License.
 #
 
-from __future__ import division, print_function #, unicode_literals
+from __future__ import division, print_function
 from builtins import int, range
 from io import open
 import sys
@@ -732,7 +732,7 @@ def main():
 
     parser.add_argument(
             "size",
-            help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
+            help='Size of NVS Partition in bytes (must be multiple of 4096)')
 
     parser.add_argument(
             "--version",
diff --git a/tools/mass_mfg/docs/README.rst b/tools/mass_mfg/docs/README.rst
index 8d50aedf8..881fed696 100644
--- a/tools/mass_mfg/docs/README.rst
+++ b/tools/mass_mfg/docs/README.rst
@@ -120,7 +120,7 @@ The mfg\_gen.py utility is using the generated CSV Configuration file and Master
 +------------------------+----------------------------------------------------------------------------------------------+
 |   Arguments            |                                     Description                                              |                                   
 +========================+==============================================================================================+
-| --size PART_SIZE       |  Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000                         |
+| --size PART_SIZE       |  Size of NVS Partition in bytes (must be multiple of 4096)                                   |
 +------------------------+----------------------------------------------------------------------------------------------+
 | --conf CONFIG_FILE     |  the input configuration csv file                                                            |
 +------------------------+----------------------------------------------------------------------------------------------+
diff --git a/tools/mass_mfg/mfg_gen.py b/tools/mass_mfg/mfg_gen.py
index 07b2b28e6..c5502161c 100755
--- a/tools/mass_mfg/mfg_gen.py
+++ b/tools/mass_mfg/mfg_gen.py
@@ -16,7 +16,7 @@
 #
 
 from __future__ import print_function
-from builtins import map, range
+from builtins import range
 from future.moves.itertools import zip_longest
 import sys
 import os
@@ -27,6 +27,7 @@ import distutils.dir_util
 sys.path.insert(0, os.getenv('IDF_PATH') + "/components/nvs_flash/nvs_partition_generator/")
 import nvs_partition_gen
 
+
 def verify_values_exist(input_values_file, keys_in_values_file):
     """ Verify all keys have corresponding values in values file
     """
@@ -288,7 +289,7 @@ file_identifier=None,output_dir_path=None):
             parser.add_argument("--size",
                                 dest='part_size',
                                 required=True,
-                                help='Size of NVS Partition in hex (must be multiple of 4096). Eg. 0x1000')
+                                help='Size of NVS Partition in bytes (must be multiple of 4096)')
 
             parser.add_argument('--conf',
                                 dest='config_file',