Merge branch 'bugfix/app_partition_fallback' into 'master'

partition_table: Add parttool.py fallback option, some small cleanups

See merge request idf/esp-idf!2593
This commit is contained in:
Angus Gratton 2018-06-25 08:43:42 +08:00
commit 3a3a12d48d
9 changed files with 218 additions and 97 deletions

View file

@ -72,7 +72,7 @@ $(COMPONENT_LIB): $(OBJ_FILES)
lib: $(COMPONENT_LIB)
partition_table.bin: partition_table.csv
python ../../partition_table/gen_esp32part.py --verify $< $@
python ../../partition_table/gen_esp32part.py $< $@
$(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) $(WEAR_LEVELLING_HOST_DIR)/$(WEAR_LEVELLING_LIB) partition_table.bin
g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -L$(WEAR_LEVELLING_HOST_DIR) -l:$(WEAR_LEVELLING_LIB) -g -m32

View file

@ -57,7 +57,7 @@ all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info
partition_table_get_info: $(PARTITION_TABLE_BIN)
$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype phy --offset $(PARTITION_TABLE_BIN)))
$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --type app --subtype factory --offset $(PARTITION_TABLE_BIN)))
$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --default-boot-partition --offset $(PARTITION_TABLE_BIN)))
export APP_OFFSET
export PHY_DATA_OFFSET

View file

@ -33,7 +33,32 @@ MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in
MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
PARTITION_TABLE_SIZE = 0x1000 # Size of partition table
__version__ = '1.1'
__version__ = '1.2'
APP_TYPE = 0x00
DATA_TYPE = 0x01
TYPES = {
"app" : APP_TYPE,
"data" : DATA_TYPE,
}
# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
SUBTYPES = {
APP_TYPE : {
"factory" : 0x00,
"test" : 0x20,
},
DATA_TYPE : {
"ota" : 0x00,
"phy" : 0x01,
"nvs" : 0x02,
"coredump" : 0x03,
"esphttpd" : 0x80,
"fat" : 0x81,
"spiffs" : 0x82,
},
}
quiet = False
md5sum = True
@ -46,9 +71,8 @@ def status(msg):
def critical(msg):
""" Print critical message to stderr """
if not quiet:
sys.stderr.write(msg)
sys.stderr.write('\n')
sys.stderr.write(msg)
sys.stderr.write('\n')
class PartitionTable(list):
def __init__(self):
@ -85,7 +109,7 @@ class PartitionTable(list):
critical("WARNING: 0x%x address in the partition table is below 0x%x" % (e.offset, last_end))
e.offset = None
if e.offset is None:
pad_to = 0x10000 if e.type == PartitionDefinition.APP_TYPE else 4
pad_to = 0x10000 if e.type == APP_TYPE else 4
if last_end % pad_to != 0:
last_end += pad_to - (last_end % pad_to)
e.offset = last_end
@ -106,6 +130,36 @@ class PartitionTable(list):
else:
return super(PartitionTable, self).__getitem__(item)
def find_by_type(self, ptype, subtype):
""" Return a partition by type & subtype, returns
None if not found """
# convert ptype & subtypes names (if supplied this way) to integer values
try:
ptype = TYPES[ptype]
except KeyError:
try:
ptypes = int(ptype, 0)
except TypeError:
pass
try:
subtype = SUBTYPES[int(ptype)][subtype]
except KeyError:
try:
ptypes = int(ptype, 0)
except TypeError:
pass
for p in self:
if p.type == ptype and p.subtype == subtype:
return p
return None
def find_by_name(self, name):
for p in self:
if p.name == name:
return p
return None
def verify(self):
# verify each partition individually
for p in self:
@ -165,30 +219,6 @@ class PartitionTable(list):
return "\n".join(rows) + "\n"
class PartitionDefinition(object):
APP_TYPE = 0x00
DATA_TYPE = 0x01
TYPES = {
"app" : APP_TYPE,
"data" : DATA_TYPE,
}
# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h
SUBTYPES = {
APP_TYPE : {
"factory" : 0x00,
"test" : 0x20,
},
DATA_TYPE : {
"ota" : 0x00,
"phy" : 0x01,
"nvs" : 0x02,
"coredump" : 0x03,
"esphttpd" : 0x80,
"fat" : 0x81,
"spiffs" : 0x82,
},
}
MAGIC_BYTES = b"\xAA\x50"
ALIGNMENT = {
@ -202,7 +232,7 @@ class PartitionDefinition(object):
"encrypted" : 0
}
# add subtypes for the 16 OTA slot values ("ota_XXX, etc.")
# add subtypes for the 16 OTA slot values ("ota_XX, etc.")
for ota_slot in range(16):
SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
@ -270,12 +300,12 @@ class PartitionDefinition(object):
def parse_type(self, strval):
if strval == "":
raise InputError("Field 'type' can't be left empty.")
return parse_int(strval, self.TYPES)
return parse_int(strval, TYPES)
def parse_subtype(self, strval):
if strval == "":
return 0 # default
return parse_int(strval, self.SUBTYPES.get(self.type, {}))
return parse_int(strval, SUBTYPES.get(self.type, {}))
def parse_address(self, strval):
if strval == "":
@ -295,6 +325,14 @@ class PartitionDefinition(object):
if self.size is None:
raise ValidationError(self, "Size field is not set")
if self.name in TYPES and TYPES.get(self.name, "") != self.type:
critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's type (0x%x). Mistake in partition table?" % (self.name, self.type))
all_subtype_names = []
for names in (t.keys() for t in SUBTYPES.values()):
all_subtype_names += names
if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype:
critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype))
STRUCT_FORMAT = "<2sBBLL16sL"
@classmethod
@ -348,8 +386,8 @@ class PartitionDefinition(object):
return ":".join(self.get_flags_list())
return ",".join([ self.name,
lookup_keyword(self.type, self.TYPES),
lookup_keyword(self.subtype, self.SUBTYPES.get(self.type, {})),
lookup_keyword(self.type, TYPES),
lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})),
addr_format(self.offset, False),
addr_format(self.size, True),
generate_text_flags()])
@ -381,14 +419,14 @@ def main():
parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash',
nargs='?', choices=[ '1MB', '2MB', '4MB', '8MB', '16MB' ])
parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true')
parser.add_argument('--verify', '-v', help='Verify partition table fields', default=True, action='store_false')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true')
parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is enabled by default and this flag does nothing.", action='store_true')
parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true')
parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000')
parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted, unless the --display argument is also passed (in which case only the summary is printed.)',
nargs='?',
default='-')
parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb'))
parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.',
nargs='?', default='-')
args = parser.parse_args()
@ -405,7 +443,7 @@ def main():
status("Parsing CSV input...")
table = PartitionTable.from_csv(input)
if args.verify:
if not args.no_verify:
status("Verifying table...")
table.verify()
@ -423,7 +461,11 @@ def main():
f.write(output)
else:
output = table.to_binary()
with sys.stdout.buffer if args.output == '-' else open(args.output, 'wb') as f:
try:
stdout_binary = sys.stdout.buffer # Python 3
except AttributeError:
stdout_binary = sys.stdout
with stdout_binary if args.output == '-' else open(args.output, 'wb') as f:
f.write(output)

View file

@ -3,6 +3,6 @@
nvs, data, nvs, 0x9000, 0x4000
otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
factory, 0, 0, 0x10000, 1M
ota_0, 0, ota_0, , 1M
ota_1, 0, ota_1, , 1M
factory, app, factory, 0x10000, 1M
ota_0, app, ota_0, , 1M
ota_1, app, ota_1, , 1M

1 # Name, Type, SubType, Offset, Size, Flags
3 nvs, data, nvs, 0x9000, 0x4000
4 otadata, data, ota, 0xd000, 0x2000
5 phy_init, data, phy, 0xf000, 0x1000
6 factory, 0, 0, 0x10000, 1M factory, app, factory, 0x10000, 1M
7 ota_0, 0, ota_0, , 1M ota_0, app, ota_0, , 1M
8 ota_1, 0, ota_1, , 1M ota_1, app, ota_1, , 1M

View file

@ -48,17 +48,30 @@ def main():
parser = argparse.ArgumentParser(description='Returns info about the required partition.')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
parser.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
parser.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
search_type = parser.add_mutually_exclusive_group()
search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+
'using the same fallback logic as the IDF bootloader', action="store_true")
parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None)
parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true", default=None)
parser.add_argument('--size', help='Return size of required partition', action="store_true", default=None)
parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true")
parser.add_argument('--size', help='Return size of required partition', action="store_true")
parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.', type=argparse.FileType('rb'), default=sys.stdin)
parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.',
type=argparse.FileType('rb'), default=sys.stdin)
args = parser.parse_args()
if args.type is not None and args.subtype is None:
status("If --type is specified, --subtype is required")
return 2
if args.type is None and args.subtype is not None:
status("--subtype is only used with --type")
return 2
quiet = args.quiet
input = args.input.read()
@ -71,36 +84,30 @@ def main():
status("Parsing CSV input...")
table = gen.PartitionTable.from_csv(input)
if args.partition_name is not None:
offset = 0
size = 0
for p in table:
if p.name == args.partition_name:
offset = p.offset
size = p.size
break;
if args.offset is not None:
print('0x%x ' % (offset))
if args.size is not None:
print('0x%x' % (size))
return 0
found_partition = None
if args.type is not None and args.subtype is not None:
offset = 0
size = 0
TYPES = gen.PartitionDefinition.TYPES
SUBTYPES = gen.PartitionDefinition.SUBTYPES
for p in table:
if p.type == TYPES[args.type]:
if p.subtype == SUBTYPES[TYPES[args.type]][args.subtype]:
offset = p.offset
size = p.size
break;
if args.offset is not None:
print('0x%x ' % (offset))
if args.size is not None:
print('0x%x' % (size))
return 0
if args.default_boot_partition:
search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ]
for subtype in search:
found_partition = table.find_by_type("app", subtype)
if found_partition is not None:
break
elif args.partition_name is not None:
found_partition = table.find_by_name(args.partition_name)
elif args.type is not None:
found_partition = table.find_by_type(args.type, args.subtype)
else:
raise RuntimeError("invalid partition selection choice")
if found_partition is None:
return 1 # nothing found
if args.offset:
print('0x%x ' % (found_partition.offset))
if args.size:
print('0x%x' % (found_partition.size))
return 0
class InputError(RuntimeError):
def __init__(self, e):
@ -115,7 +122,8 @@ class ValidationError(InputError):
if __name__ == '__main__':
try:
main()
r = main()
sys.exit(r)
except InputError as e:
print(e, file=sys.stderr)
sys.exit(2)

View file

@ -7,6 +7,7 @@ import sys
import subprocess
import tempfile
import os
import StringIO
sys.path.append("..")
from gen_esp32part import *
@ -236,10 +237,10 @@ class BinaryParserTests(unittest.TestCase):
t.verify()
self.assertEqual(3, len(t))
self.assertEqual(t[0].type, PartitionDefinition.APP_TYPE)
self.assertEqual(t[0].type, APP_TYPE)
self.assertEqual(t[0].name, "factory")
self.assertEqual(t[1].type, PartitionDefinition.DATA_TYPE)
self.assertEqual(t[1].type, DATA_TYPE)
self.assertEqual(t[1].name, "data")
self.assertEqual(t[2].type, 0x10)
@ -319,16 +320,18 @@ class CommandLineTests(unittest.TestCase):
f.write(LONGER_BINARY_TABLE)
# run gen_esp32part.py to convert binary file to CSV
subprocess.check_call([sys.executable, "../gen_esp32part.py",
binpath, csvpath])
output = subprocess.check_output([sys.executable, "../gen_esp32part.py",
binpath, csvpath], stderr=subprocess.STDOUT)
# reopen the CSV and check the generated binary is identical
self.assertNotIn("WARNING", output)
with open(csvpath, 'r') as f:
from_csv = PartitionTable.from_csv(f.read())
self.assertEqual(_strip_trailing_ffs(from_csv.to_binary()), LONGER_BINARY_TABLE)
# run gen_esp32part.py to conver the CSV to binary again
subprocess.check_call([sys.executable, "../gen_esp32part.py",
csvpath, binpath])
output = subprocess.check_output([sys.executable, "../gen_esp32part.py",
csvpath, binpath], stderr=subprocess.STDOUT)
self.assertNotIn("WARNING", output)
# assert that file reads back as identical
with open(binpath, 'rb') as f:
binary_readback = f.read()
@ -356,5 +359,75 @@ app,app, factory, 32K, 1M
t.verify()
def test_warnings(self):
try:
sys.stderr = StringIO.StringIO() # capture stderr
csv_1 = "app, 1, 2, 32K, 1M\n"
PartitionTable.from_csv(csv_1).verify()
self.assertIn("WARNING", sys.stderr.getvalue())
self.assertIn("partition type", sys.stderr.getvalue())
sys.stderr = StringIO.StringIO()
csv_2 = "ota_0, app, ota_1, , 1M\n"
PartitionTable.from_csv(csv_2).verify()
self.assertIn("WARNING", sys.stderr.getvalue())
self.assertIn("partition subtype", sys.stderr.getvalue())
finally:
sys.stderr = sys.__stderr__
class PartToolTests(unittest.TestCase):
def _run_parttool(self, csvcontents, args):
csvpath = tempfile.mktemp()
with open(csvpath, "w") as f:
f.write(csvcontents)
try:
output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + [ csvpath ],
stderr=subprocess.STDOUT)
self.assertNotIn("WARNING", output)
m = re.search("0x[0-9a-fA-F]+", output)
return m.group(0) if m else ""
finally:
os.remove(csvpath)
def test_find_basic(self):
csv = """
nvs, data, nvs, 0x9000, 0x4000
otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
factory, app, factory, 0x10000, 1M
"""
rpt = lambda args: self._run_parttool(csv, args)
self.assertEqual(
rpt("--type data --subtype nvs --offset"), "0x9000")
self.assertEqual(
rpt("--type data --subtype nvs --size"), "0x4000")
self.assertEqual(
rpt("--partition-name otadata --offset"), "0xd000")
self.assertEqual(
rpt("--default-boot-partition --offset"), "0x10000")
def test_fallback(self):
csv = """
nvs, data, nvs, 0x9000, 0x4000
otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
ota_0, app, ota_0, 0x30000, 1M
ota_1, app, ota_1, , 1M
"""
rpt = lambda args: self._run_parttool(csv, args)
self.assertEqual(
rpt("--type app --subtype ota_1 --offset"), "0x130000")
self.assertEqual(
rpt("--default-boot-partition --offset"), "0x30000") # ota_0
csv_mod = csv.replace("ota_0", "ota_2")
self.assertEqual(
self._run_parttool(csv_mod, "--default-boot-partition --offset"),
"0x130000") # now default is ota_1
if __name__ =="__main__":
unittest.main()

View file

@ -67,7 +67,7 @@ $(COMPONENT_LIB): $(OBJ_FILES)
lib: $(COMPONENT_LIB)
partitions_table.bin: partitions_table.csv
python ../../partition_table/gen_esp32part.py --verify $< $@
python ../../partition_table/gen_esp32part.py $< $@
$(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) partitions_table.bin
g++ $(LDFLAGS) -o $(TEST_PROGRAM) $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -g -m32

View file

@ -66,7 +66,7 @@ force:
lib: $(COMPONENT_LIB)
partition_table.bin: partition_table.csv
python ../../../components/partition_table/gen_esp32part.py --verify $< $@
python ../../../components/partition_table/gen_esp32part.py $< $@
$(TEST_PROGRAM): lib $(TEST_OBJ_FILES) $(SPI_FLASH_SIM_DIR)/$(SPI_FLASH_LIB) partition_table.bin
g++ $(LDFLAGS) -o $@ $(TEST_OBJ_FILES) -L$(abspath .) -l:$(COMPONENT_LIB) -L$(SPI_FLASH_SIM_DIR) -l:$(SPI_FLASH_LIB) -g -m32

View file

@ -136,18 +136,16 @@ If you configure the partition table CSV name in ``make menuconfig`` and then ``
To convert CSV to Binary manually::
python gen_esp32part.py --verify input_partitions.csv binary_partitions.bin
python gen_esp32part.py input_partitions.csv binary_partitions.bin
To convert binary format back to CSV::
python gen_esp32part.py --verify binary_partitions.bin input_partitions.csv
python gen_esp32part.py binary_partitions.bin input_partitions.csv
To display the contents of a binary partition table on stdout (this is how the summaries displayed when running `make partition_table` are generated::
python gen_esp32part.py binary_partitions.bin
``gen_esp32part.py`` takes one optional argument, ``--verify``, which will also verify the partition table during conversion (checking for overlapping partitions, unaligned partitions, etc.)
MD5 checksum
~~~~~~~~~~~~