Merge branch 'bugfix/hard_error_partition_offset' into 'master'

partition_table: Hard error if CSV contains invalid/overlapping offsets

See merge request idf/esp-idf!2755

components/partition_table/Kconfig.projbuild

@@ -45,7 +45,8 @@ config PARTITION_TABLE_OFFSET
 
         This number should be a multiple of 0x1000.
 
-        Note that partition offsets in the partition table CSV file may need to be changed if this value is set to a higher value.
+        Note that partition offsets in the partition table CSV file may need to be changed if this value is set to a higher value. To have
+        each partition offset adapt to the configured partition table offset, leave all partition offsets blank in the CSV file.
 
 config PARTITION_TABLE_MD5
     bool "Generate an MD5 checksum for the partition table"

components/partition_table/gen_esp32part.py

@@ -96,19 +96,23 @@ class PartitionTable(list):
             if line.startswith("#") or len(line) == 0:
                 continue
             try:
-                res.append(PartitionDefinition.from_csv(line))
+                res.append(PartitionDefinition.from_csv(line, line_no+1))
             except InputError as e:
                 raise InputError("Error at line %d: %s" % (line_no+1, e))
             except Exception:
-                critical("Unexpected error parsing line %d: %s" % (line_no+1, line))
+                critical("Unexpected error parsing CSV line %d: %s" % (line_no+1, line))
                 raise
 
         # fix up missing offsets & negative sizes
         last_end = offset_part_table + PARTITION_TABLE_SIZE # first offset after partition table
         for e in res:
-            if offset_part_table != 0 and e.offset is not None and e.offset < last_end:
-                critical("WARNING: 0x%x address in the partition table is below 0x%x" % (e.offset, last_end))
-                e.offset = None
+            if e.offset is not None and e.offset < last_end:
+                if e == res[0]:
+                    raise InputError("CSV Error: First partition offset 0x%x overlaps end of partition table 0x%x"
+                                     % (e.offset, last_end))
+                else:
+                    raise InputError("CSV Error: Partitions overlap. Partition at line %d sets offset 0x%x. Previous partition ends 0x%x"
+                                     % (e.line_no, e.offset, last_end))
             if e.offset is None:
                 pad_to = 0x10000 if e.type == APP_TYPE else 4
                 if last_end % pad_to != 0:
@@ -246,12 +250,13 @@ class PartitionDefinition(object):
         self.encrypted = False
 
     @classmethod
-    def from_csv(cls, line):
+    def from_csv(cls, line, line_no):
         """ Parse a line from the CSV """
         line_w_defaults = line + ",,,,"  # lazy way to support default fields
         fields = [ f.strip() for f in line_w_defaults.split(",") ]
 
         res = PartitionDefinition()
+        res.line_no = line_no
         res.name = fields[0]
         res.type = res.parse_type(fields[1])
         res.subtype = res.parse_subtype(fields[2])

components/partition_table/partitions_singleapp.csv

@@ -1,5 +1,5 @@
 # Name,   Type, SubType, Offset,  Size, Flags
 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
-nvs,      data, nvs,     0x9000,  0x6000,
-phy_init, data, phy,     0xf000,  0x1000,
-factory,  app,  factory, 0x10000, 1M,
+nvs,      data, nvs,     ,        0x6000,
+phy_init, data, phy,     ,        0x1000,
+factory,  app,  factory, ,        1M,

components/partition_table/partitions_singleapp_coredump.csv

@@ -1,6 +1,6 @@
 # Name,   Type, SubType, Offset,  Size
 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
-nvs,      data, nvs,     0x9000,  0x6000
-phy_init, data, phy,     0xf000,  0x1000
-factory,  app,  factory, 0x10000, 1M
-coredump, data, coredump,,        64K
+nvs,      data, nvs,           ,  0x6000
+phy_init, data, phy,           ,  0x1000
+factory,  app,  factory,       ,  1M
+coredump, data, coredump,      ,  64K

components/partition_table/partitions_two_ota.csv

@@ -1,8 +1,8 @@
 # Name,   Type, SubType, Offset,   Size, Flags
 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
-nvs,      data, nvs,     0x9000,  0x4000
-otadata,  data, ota,     0xd000,  0x2000
-phy_init, data, phy,     0xf000,  0x1000
-factory,  app,  factory, 0x10000,  1M
-ota_0,    app,  ota_0,   ,         1M
-ota_1,    app,  ota_1,   ,         1M
+nvs,      data, nvs,     ,        0x4000,
+otadata,  data, ota,     ,        0x2000,
+phy_init, data, phy,     ,        0x1000,
+factory,  app,  factory, ,        1M,
+ota_0,    app,  ota_0,   ,        1M,
+ota_1,    app,  ota_1,   ,        1M,

components/partition_table/partitions_two_ota_coredump.csv

@@ -1,9 +1,9 @@
 # Name,   Type, SubType, Offset,   Size
 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
-nvs,      data, nvs,     0x9000,  0x4000
-otadata,  data, ota,     0xd000,  0x2000
-phy_init, data, phy,     0xf000,  0x1000
-factory,  0,    0,       0x10000,  1M
+nvs,      data, nvs,     ,         0x4000
+otadata,  data, ota,     ,         0x2000
+phy_init, data, phy,     ,         0x1000
+factory,  0,    0,       ,         1M
 coredump, data, coredump,,         64K
 ota_0,    0,    ota_0,   ,         1M
 ota_1,    0,    ota_1,   ,         1M

components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py

@@ -162,8 +162,8 @@ second, data, 0x15,         ,  1M
 first, app, factory, 0x100000, 2M
 second, app, ota_0,  0x200000, 1M
 """
-        t = PartitionTable.from_csv(csv)
         with self.assertRaisesRegexp(InputError, "overlap"):
+            t = PartitionTable.from_csv(csv)
             t.verify()
 
 class BinaryOutputTests(unittest.TestCase):

docs/en/api-guides/partition-tables.rst

@@ -4,7 +4,7 @@ Partition Tables
 Overview
 --------
 
-A single ESP32's flash can contain multiple apps, as well as many different kinds of data (calibration data, filesystems, parameter storage, etc). For this reason a partition table is flashed to offset 0x8000 in the flash.
+A single ESP32's flash can contain multiple apps, as well as many different kinds of data (calibration data, filesystems, parameter storage, etc). For this reason a partition table is flashed to (:envvar:`default offset <CONFIG_PARTITION_TABLE_OFFSET>`) 0x8000 in the flash.
 
 Partition table length is 0xC00 bytes (maximum 95 partition table entries). An MD5 checksum is appended after the table data. If the partition table is signed due to `secure boot`, the signature is appended after the partition table.
 
@@ -23,10 +23,10 @@ Built-in Partition Tables
 Here is the summary printed for the "Single factory app, no OTA" configuration::
 
   # Espressif ESP32 Partition Table
-  # Name,   Type, SubType, Offset,  Size
-  nvs,      data, nvs,     0x9000,  0x6000
-  phy_init, data, phy,     0xf000,  0x1000
-  factory,  app,  factory, 0x10000, 1M
+  # Name,   Type, SubType, Offset,  Size, Flags
+  nvs,      data, nvs,     0x9000,  0x6000,
+  phy_init, data, phy,     0xf000,  0x1000,
+  factory,  app,  factory, 0x10000, 1M,
 
 * At a 0x10000 (64KB) offset in the flash is the app labelled "factory". The bootloader will run this app by default.
 * There are also two data regions defined in the partition table for storing NVS library partition and PHY init data.
@@ -34,13 +34,13 @@ Here is the summary printed for the "Single factory app, no OTA" configuration::
 Here is the summary printed for the "Factory app, two OTA definitions" configuration::
 
   # Espressif ESP32 Partition Table
-  # Name,   Type, SubType, Offset,  Size
-  nvs,      data, nvs,     0x9000,  0x4000
-  otadata,  data, ota,     0xd000,  0x2000
-  phy_init, data, phy,     0xf000,  0x1000
-  factory,  0,    0,       0x10000, 1M
-  ota_0,    0,    ota_0,   ,        1M
-  ota_1,    0,    ota_1,   ,        1M
+  # Name,   Type, SubType, Offset,  Size, Flags
+  nvs,      data, nvs,     0x9000,  0x4000,
+  otadata,  data, ota,     0xd000,  0x2000,
+  phy_init, data, phy,     0xf000,  0x1000,
+  factory,  0,    0,       0x10000, 1M,
+  ota_0,    0,    ota_0,  0x110000, 1M,
+  ota_1,    0,    ota_1,  0x210000, 1M,
 
 * There are now three app partition definitions.
 * The type of all three are set as "app", but the subtype varies between the factory app at 0x10000 and the next two "OTA" apps.
@@ -53,17 +53,17 @@ If you choose "Custom partition table CSV" in menuconfig then you can also enter
 
 The CSV format is the same format as printed in the summaries shown above. However, not all fields are required in the CSV. For example, here is the "input" CSV for the OTA partition table::
 
-  # Name,   Type, SubType, Offset,   Size
-  nvs,      data, nvs,     0x9000,  0x4000
-  otadata,  data, ota,     0xd000,  0x2000
-  phy_init, data, phy,     0xf000,  0x1000
-  factory,  app,  factory, 0x10000,  1M
-  ota_0,    app,  ota_0,   ,         1M
-  ota_1,    app,  ota_1,   ,         1M
+  # Name,   Type, SubType, Offset, Size, Flags
+  nvs,      data, nvs,     ,       0x4000,
+  otadata,  data, ota,     ,       0x2000,
+  phy_init, data, phy,     ,       0x1000,
+  factory,  app,  factory, ,       1M,
+  ota_0,    app,  ota_0,   ,       1M,
+  ota_1,    app,  ota_1,   ,       1M,
 
 * Whitespace between fields is ignored, and so is any line starting with # (comments).
 * Each non-comment line in the CSV file is a partition definition.
-* Only the offset for the first partition is supplied. The gen_esp32part.py tool fills in each remaining offset to start after the preceding partition.
+* The "Offset" field for each partition is empty. The gen_esp32part.py tool fills in each blank offset, starting after the partition table and making sure each partition is aligned correctly.
 
 Name field
 ~~~~~~~~~~
@@ -121,12 +121,21 @@ Other data subtypes are reserved for future esp-idf uses.
 Offset & Size
 ~~~~~~~~~~~~~
 
-Only the first offset field is required (we recommend using 0x10000). Partitions with blank offsets will start after the previous partition.
+Partitions with blank offsets will start after the previous partition, or after the partition table in the case of the first partition.
 
 App partitions have to be at offsets aligned to 0x10000 (64K). If you leave the offset field blank, the tool will automatically align the partition. If you specify an unaligned offset for an app partition, the tool will return an error.
 
 Sizes and offsets can be specified as decimal numbers, hex numbers with the prefix 0x, or size multipliers K or M (1024 and 1024*1024 bytes).
 
+If you want the partitions in the partition table to work with any starting offset (:envvar:`CONFIG_PARTITION_TABLE_OFFSET`) of the table itself, leave the offset field (in CSV file) for all partitions blank. Similarly, if changing the partition table offset then be aware that all blank partition offsets may change to match, and that any fixed offsets may now collide with the partition table (causing an error).
+
+Flags
+~~~~~
+
+Only one flag is currently supported, ``encrypted``. If this field is set to ``encrypted``, this partition will be encrypted if :doc:`/security/flash-encryption` is enabled.
+
+(Note that ``app`` type partitions will always be encrypted, regardless of whether this flag is set or not.)
+
 Generating Binary Partition Table
 ---------------------------------
 

docs/en/security/flash-encryption.rst

@@ -23,7 +23,7 @@ Background
   - Secure boot bootloader digest (if secure boot is enabled)
   - Partition Table
   - All "app" type partitions
-  - Any partition marked with the "encrypt" flag in the partition table
+  - Any partition marked with the "encrypted" flag in the partition table
 
 	It may be desirable for some data partitions to remain unencrypted for ease of access, or to use flash-friendly update algorithms that are ineffective if the data is encrypted. "NVS" partitions for non-volatile storage cannot be encrypted.