partition_table: implement new parttool functionality

components/partition_table/gen_esp32part.py

@@ -35,6 +35,9 @@ MAX_PARTITION_LENGTH = 0xC00   # 3K for partition data (96 entries) leaves 1K in
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
 PARTITION_TABLE_SIZE  = 0x1000  # Size of partition table
 
+MIN_PARTITION_SUBTYPE_APP_OTA = 0x10
+NUM_PARTITION_SUBTYPE_APP_OTA = 16
+
 __version__ = '1.2'
 
 APP_TYPE = 0x00
@@ -254,8 +257,8 @@ class PartitionDefinition(object):
     }
 
     # add subtypes for the 16 OTA slot values ("ota_XX, etc.")
-    for ota_slot in range(16):
+    for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA):
-        SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
+        SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot
 
     def __init__(self):
         self.name = ""

components/partition_table/parttool.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 #
-# parttool returns info about the required partition.
+# parttool is used to perform partition level operations - reading,
-#
+# writing, erasing and getting info about the partition.
-# This utility is used by the make system to get information 
-# about the start addresses: partition table, factory area, phy area.
 #
 # Copyright 2018 Espressif Systems (Shanghai) PTE LTD
 #
@@ -21,113 +19,259 @@
 from __future__ import print_function, division
 import argparse
 import os
-import re
-import struct
 import sys
-import hashlib
+import subprocess
-import binascii
+import tempfile
 import gen_esp32part as gen
 
 __version__ = '1.0'
 
+IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components"))
+
+ESPTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py")
+
 quiet = False
 
 def status(msg):
     """ Print status message to stderr """
     if not quiet:
-        critical(msg)
+        print(msg)
 
-def critical(msg):
-    """ Print critical message to stderr """
-    if not quiet:
-        sys.stderr.write(msg)
-        sys.stderr.write('\n')
-               
-def main():
-    global quiet
-    parser = argparse.ArgumentParser(description='Returns info about the required partition.')
 
-    parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
+def _invoke_esptool(esptool_args, args):
+    m_esptool_args = [sys.executable, ESPTOOL_PY]
 
-    parser.add_argument('--partition-table-offset', help='The offset of the partition table in flash. Only consulted if partition table is in CSV format.', type=str, default='0x8000')
+    if args.port != "":
+        m_esptool_args.extend(["--port", args.port])
 
-    search_type = parser.add_mutually_exclusive_group()
+    m_esptool_args.extend(esptool_args)
-    search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
-    search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
-    search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+
-                             'using the same fallback logic as the IDF bootloader', action="store_true")
 
-    parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None)
+    if quiet:
+        with open(os.devnull, "w") as fnull:
+            subprocess.check_call(m_esptool_args, stdout=fnull, stderr=fnull)
+    else:
+        subprocess.check_call(m_esptool_args)
 
-    parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true")
-    parser.add_argument('--size', help='Return size of required partition', action="store_true")
 
-    parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.',
+def _get_partition_table(args):
-                        type=argparse.FileType('rb'), default=sys.stdin)
+    partition_table = None
-
-    args = parser.parse_args()
-
-    if args.type is not None and args.subtype is None:
-        status("If --type is specified, --subtype is required")
-        return 2
-    if args.type is None and args.subtype is not None:
-        status("--subtype is only used with --type")
-        return 2
-
-    quiet = args.quiet
 
     gen.offset_part_table = int(args.partition_table_offset, 0)
 
-    input = args.input.read()
+    if args.partition_table_file:
-    input_is_binary = input[0:2] == gen.PartitionDefinition.MAGIC_BYTES
+        status("Reading partition table from partition table file...")
-    if input_is_binary:
+
-        status("Parsing binary partition input...")
+        try:
-        table = gen.PartitionTable.from_binary(input)
+            with open(args.partition_table_file, "rb") as partition_table_file:
+                partition_table = gen.PartitionTable.from_binary(partition_table_file.read())
+                status("Partition table read from binary file {}".format(partition_table_file.name))
+        except (gen.InputError, TypeError):
+            with open(args.partition_table_file, "r") as partition_table_file:
+                partition_table_file.seek(0)
+                partition_table = gen.PartitionTable.from_csv(partition_table_file.read())
+                status("Partition table read from CSV file {}".format(partition_table_file.name))
     else:
-        input = input.decode()
+        port_info = (" on port " + args.port if args.port else "")
-        status("Parsing CSV input...")
+        status("Reading partition table from device{}...".format(port_info))
-        table = gen.PartitionTable.from_csv(input)
+        with tempfile.NamedTemporaryFile() as partition_table_file:
+            invoke_args = ["read_flash", str(gen.offset_part_table), str(gen.MAX_PARTITION_LENGTH), partition_table_file.name]
+            _invoke_esptool(invoke_args, args)
+            partition_table = gen.PartitionTable.from_binary(partition_table_file.read())
+        status("Partition table read from device" + port_info)
 
-    found_partition = None
+    return partition_table
 
-    if args.default_boot_partition:
+
-        search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ]
+def _get_partition(args):
+    partition_table = _get_partition_table(args)
+
+    partition = None
+
+    if args.partition_name:
+        partition = partition_table.find_by_name(args.partition_name)
+    elif args.partition_type and args.partition_subtype:
+        partition = partition_table.find_by_type(args.partition_type, args.partition_subtype)
+    elif args.partition_boot_default:
+        search = [ "factory" ] + [ "ota_{}".format(d) for d in range(16) ]
         for subtype in search:
-            found_partition = table.find_by_type("app", subtype)
+            partition = partition_table.find_by_type("app", subtype)
-            if found_partition is not None:
+            if partition is not None:
                 break
-    elif args.partition_name is not None:
-        found_partition = table.find_by_name(args.partition_name)
-    elif args.type is not None:
-        found_partition = table.find_by_type(args.type, args.subtype)
     else:
-        raise RuntimeError("invalid partition selection choice")
+        raise RuntimeError("Invalid partition selection arguments. Specify --partition-name OR \
+                            --partition-type and --partition-subtype OR --partition--boot-default.")
 
-    if found_partition is None:
+    if partition:
-        return 1  # nothing found
+        status("Found partition {}".format(str(partition)))
 
-    if args.offset:
+    return partition
-        print('0x%x' % (found_partition.offset))
-    if args.size:
-        print('0x%x' % (found_partition.size))
-
-    return 0
-
-class InputError(RuntimeError):
-    def __init__(self, e):
-        super(InputError, self).__init__(e)
 
 
-class ValidationError(InputError):
+def _get_and_check_partition(args):
-    def __init__(self, partition, message):
+    partition = None
-        super(ValidationError, self).__init__(
+
-            "Partition %s invalid: %s" % (partition.name, message))
+    partition = _get_partition(args)
+
+    if not partition:
+        raise RuntimeError("Unable to find specified partition.")
+
+    return partition
+
+
+def write_partition(args):
+    erase_partition(args)
+
+    partition = _get_and_check_partition(args)
+
+    status("Checking input file size...")
+
+    with open(args.input, "rb") as input_file:
+        content_len = len(input_file.read())
+
+        if content_len != partition.size:
+            status("File size (0x{:x}) does not match partition size (0x{:x})".format(content_len, partition.size))
+        else:
+            status("File size matches partition size (0x{:x})".format(partition.size))
+
+    _invoke_esptool(["write_flash", str(partition.offset),  args.input], args)
+
+    status("Written contents of file '{}' to device at offset 0x{:x}".format(args.input, partition.offset))
+
+
+def read_partition(args):
+    partition = _get_and_check_partition(args)
+    _invoke_esptool(["read_flash", str(partition.offset), str(partition.size), args.output], args)
+    status("Read partition contents from device at offset 0x{:x} to file '{}'".format(partition.offset, args.output))
+
+
+def erase_partition(args):
+    partition = _get_and_check_partition(args)
+    _invoke_esptool(["erase_region", str(partition.offset),  str(partition.size)], args)
+    status("Erased partition at offset 0x{:x} on device".format(partition.offset))
+
+
+def get_partition_info(args):
+    partition = None
+
+    if args.table:
+        partition_table =  _get_partition_table(args)
+
+        if args.table.endswith(".csv"):
+            partition_table = partition_table.to_csv()
+        else:
+            partition_table = partition_table.to_binary()
+
+        with open(args.table, "wb") as table_file:
+            table_file.write(partition_table)
+            status("Partition table written to " + table_file.name)
+    else:
+        partition = _get_partition(args)
+
+        if partition:
+            info_dict = {
+                "offset" : '0x{:x}'.format(partition.offset),
+                "size": '0x{:x}'.format(partition.size)
+            }
+
+            infos = []
+
+            try:
+                for info in args.info:
+                    infos += [info_dict[info]]
+            except KeyError:
+                raise RuntimeError("Request for unknown partition info {}".format(info))
+
+            status("Requested partition information [{}]:".format( ", ".join(args.info)))
+            print(" ".join(infos))
+        else:
+            status("Partition not found")
+
+
+def generate_blank_partition_file(args):
+    output = None
+    stdout_binary = None
+
+    partition = _get_and_check_partition(args)
+    output =  b"\xFF" * partition.size
+
+    try:
+        stdout_binary = sys.stdout.buffer  # Python 3
+    except AttributeError:
+        stdout_binary = sys.stdout
+
+    with stdout_binary if args.output == "" else open(args.output, 'wb') as f:
+        f.write(output)
+        status("Blank partition file '{}' generated".format(args.output))
+
+
+def main():
+    global quiet
+
+    parser = argparse.ArgumentParser("ESP-IDF Partitions Tool")
+
+    parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true")
+
+    # There are two possible sources for the partition table: a device attached to the host
+    # or a partition table CSV/binary file. These sources are mutually exclusive.
+    partition_table_info_source_args = parser.add_mutually_exclusive_group()
+
+    partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="")
+    partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from")
+
+    parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from",  default="0x8000")
+
+    # Specify what partition to perform the operation on. This can either be specified using the
+    # partition name or the first partition that matches the specified type/subtype
+    partition_selection_args = parser.add_mutually_exclusive_group()
+
+    partition_selection_args.add_argument("--partition-name", "-n", help="name of the partition")
+    partition_selection_args.add_argument("--partition-type", "-t", help="type of the partition")
+    partition_selection_args.add_argument('--partition-boot-default', "-d", help='select the default boot partition, '+
+                             'using the same fallback logic as the IDF bootloader', action="store_true")
+
+    parser.add_argument("--partition-subtype", "-s", help="subtype of the partition")
+
+    subparsers = parser.add_subparsers(dest="operation", help="run parttool -h for additional help")
+
+    # Specify the supported operations
+    read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file")
+    read_part_subparser.add_argument("--output", help="file to dump the read partition contents to")
+
+    write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device")
+    write_part_subparser.add_argument("--input", help="file whose contents are to be written to the partition offset")
+
+    subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device")
+
+    print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information")
+    print_partition_info_subparser_info_type = print_partition_info_subparser.add_mutually_exclusive_group()
+    print_partition_info_subparser_info_type.add_argument("--info", help="type of partition information to get", nargs="+")
+    print_partition_info_subparser_info_type.add_argument("--table", help="dump the partition table to a file")
+
+    generate_blank_subparser = subparsers.add_parser("generate_blank_partition_file", help="generate a blank (all 0xFF) partition file of the specified partition that can be flashed to the device")
+    generate_blank_subparser.add_argument("--output", help="blank partition file filename")
+
+    args = parser.parse_args()
+
+    quiet = args.quiet
+
+    # No operation specified, display help and exit
+    if args.operation is None:
+        if not quiet:
+            parser.print_help()
+        sys.exit(1)
+
+    # Else execute the operation
+    operation_func = globals()[args.operation]
+
+    if quiet:
+        # If exceptions occur, suppress and exit quietly
+        try:
+            operation_func(args)
+        except Exception:
+            sys.exit(2)
+    else:
+        operation_func(args)
 
 
 if __name__ == '__main__':
-    try:
+    main()
-        r = main()
-        sys.exit(r)
-    except InputError as e:
-        print(e, file=sys.stderr)
-        sys.exit(2)

components/partition_table/project_include.cmake

@@ -28,13 +28,13 @@ endif()
 set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH})
 
 # Parse the partition table to get variable partition offsets & sizes which must be known at CMake runtime
-function(get_partition_info variable get_part_info_args)
+function(get_partition_info variable get_part_info_args part_info)
     separate_arguments(get_part_info_args)
     execute_process(COMMAND ${PYTHON}
         ${COMPONENT_PATH}/parttool.py -q
         --partition-table-offset ${PARTITION_TABLE_OFFSET}
-        ${get_part_info_args}
+        --partition-table-file ${PARTITION_CSV_PATH}
-        ${PARTITION_CSV_PATH}
+        ${get_part_info_args} get_partition_info --info ${part_info}
         OUTPUT_VARIABLE result
         RESULT_VARIABLE exit_code
         OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -46,15 +46,19 @@ function(get_partition_info variable get_part_info_args)
 endfunction()
 
 if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
-    get_partition_info(PHY_PARTITION_OFFSET "--type data --subtype phy --offset")
+    get_partition_info(PHY_PARTITION_OFFSET
+                "--partition-type data --partition-subtype phy" "offset")
     set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin")
 endif()
 
-get_partition_info(APP_PARTITION_OFFSET "--default-boot-partition --offset")
+get_partition_info(APP_PARTITION_OFFSET
+                "--partition-boot-default" "offset")
 
-get_partition_info(OTADATA_PARTITION_OFFSET "--type data --subtype ota --offset")
+get_partition_info(OTADATA_PARTITION_OFFSET
+                "--partition-type data --partition-subtype ota" "offset")
 
-get_partition_info(OTADATA_PARTITION_SIZE "--type data --subtype ota --size")
+get_partition_info(OTADATA_PARTITION_SIZE
+                "--partition-type data --partition-subtype ota" "size")
 
 endif()
 

components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py

@@ -398,13 +398,13 @@ app,app, factory, 32K, 1M
 
 class PartToolTests(Py23TestCase):
 
-    def _run_parttool(self, csvcontents, args):
+    def _run_parttool(self, csvcontents, args, info):
         csvpath = tempfile.mktemp()
         with open(csvpath, "w") as f:
             f.write(csvcontents)
         try:
-            output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + [ csvpath ],
+            output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") 
-                                           stderr=subprocess.STDOUT)
+                    + ["--partition-table-file", csvpath , "get_partition_info", "--info", info], stderr=subprocess.STDOUT)
             self.assertNotIn(b"WARNING", output)
             m = re.search(b"0x[0-9a-fA-F]+", output)
             return m.group(0) if m else ""
@@ -418,16 +418,16 @@ otadata,  data, ota,     0xd000,  0x2000
 phy_init, data, phy,     0xf000,  0x1000
 factory,  app, factory, 0x10000,  1M
         """
-        rpt = lambda args: self._run_parttool(csv, args)
+        rpt = lambda args, info: self._run_parttool(csv, args, info)
 
         self.assertEqual(
-            rpt("--type data --subtype nvs --offset"), b"0x9000")
+            rpt("--partition-type=data --partition-subtype=nvs -q", "offset"), b"0x9000")
         self.assertEqual(
-            rpt("--type data --subtype nvs --size"), b"0x4000")
+            rpt("--partition-type=data --partition-subtype=nvs -q", "size"), b"0x4000")
         self.assertEqual(
-            rpt("--partition-name otadata --offset"), b"0xd000")
+            rpt("--partition-name=otadata -q", "offset"), b"0xd000")
         self.assertEqual(
-            rpt("--default-boot-partition --offset"), b"0x10000")
+            rpt("--partition-boot-default -q", "offset"), b"0x10000")
 
     def test_fallback(self):
         csv = """
@@ -437,15 +437,15 @@ phy_init, data, phy,     0xf000,  0x1000
 ota_0,  app,    ota_0,   0x30000,  1M
 ota_1,  app,    ota_1,          ,  1M
         """
-        rpt = lambda args: self._run_parttool(csv, args)
+        rpt = lambda args, info: self._run_parttool(csv, args, info)
 
         self.assertEqual(
-            rpt("--type app --subtype ota_1 --offset"), b"0x130000")
+            rpt("--partition-type=app --partition-subtype=ota_1 -q", "offset"), b"0x130000")
         self.assertEqual(
-            rpt("--default-boot-partition --offset"), b"0x30000")  # ota_0
+            rpt("--partition-boot-default -q", "offset"), b"0x30000")  # ota_0
         csv_mod = csv.replace("ota_0", "ota_2")
         self.assertEqual(
-            self._run_parttool(csv_mod, "--default-boot-partition --offset"),
+            self._run_parttool(csv_mod, "--partition-boot-default -q", "offset"),
             b"0x130000")  # now default is ota_1
 
 if __name__ =="__main__":