From e1e6c1ae0a701e012fc125e878e5c76160bcd4c2 Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Tue, 4 Dec 2018 13:06:46 +0100 Subject: [PATCH] components: Correct the Python coding style --- .flake8 | 13 +- components/espcoredump/espcoredump.py | 270 +++++++++--------- .../espcoredump/test/test_espcoredump.py | 17 +- components/partition_table/gen_esp32part.py | 131 +++++---- .../gen_esp32part_tests.py | 161 ++++++----- components/ulp/esp32ulp_mapgen.py | 34 +-- 6 files changed, 326 insertions(+), 300 deletions(-) diff --git a/.flake8 b/.flake8 index 171b2bd42..e6b8695b7 100644 --- a/.flake8 +++ b/.flake8 @@ -150,22 +150,15 @@ exclude = components/unity/unity, examples/build_system/cmake/import_lib/main/lib/tinyxml2 # autogenerated scripts - examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py, - # temporary list (should be empty) - components/app_update/dump_otadata.py, - components/app_update/gen_empty_partition.py, - components/espcoredump/espcoredump.py, - components/espcoredump/test/test_espcoredump.py, - components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py, - components/partition_table/gen_esp32part.py, - components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py, components/protocomm/python/constants_pb2.py, components/protocomm/python/sec0_pb2.py, components/protocomm/python/sec1_pb2.py, components/protocomm/python/session_pb2.py, - components/ulp/esp32ulp_mapgen.py, components/wifi_provisioning/python/wifi_config_pb2.py, components/wifi_provisioning/python/wifi_constants_pb2.py, + examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py, + # temporary list (should be empty) + components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py, tools/ci/apply_bot_filter.py, tools/cmake/convert_to_cmake.py, tools/esp_app_trace/apptrace_proc.py, diff --git a/components/espcoredump/espcoredump.py b/components/espcoredump/espcoredump.py index 8a6c4e754..47dabed38 100755 --- a/components/espcoredump/espcoredump.py +++ b/components/espcoredump/espcoredump.py @@ -5,6 +5,7 @@ from __future__ import print_function from __future__ import unicode_literals from __future__ import division +import sys try: from builtins import zip from builtins import str @@ -15,13 +16,11 @@ except ImportError: print('Import has failed probably because of the missing "future" package. Please install all the packages for ' 'interpreter {} from the $IDF_PATH/requirements.txt file.'.format(sys.executable)) sys.exit(1) -import sys import os import argparse import subprocess import tempfile import struct -import array import errno import base64 import binascii @@ -53,17 +52,18 @@ class ESPCoreDumpError(RuntimeError): """ super(ESPCoreDumpError, self).__init__(message) + class BinStruct(object): """Binary structure representation Subclasses must specify actual structure layout using 'fields' and 'format' members. - For example, the following subclass represents structure with two fields: - f1 of size 2 bytes and 4 bytes f2. Little endian. + For example, the following subclass represents structure with two fields: + f1 of size 2 bytes and 4 bytes f2. Little endian. class SomeStruct(BinStruct): fields = ("f1", "f2") format = "= ps.addr and addr < (ps.addr + seg_len): - raise ESPCoreDumpError("Can not add overlapping region [%x..%x] to ELF file. Conflict with existing [%x..%x]." % - (addr, addr + data_sz - 1, ps.addr, ps.addr + seg_len - 1)) + raise ESPCoreDumpError("Can not add overlapping region [%x..%x] to ELF file. Conflict with existing [%x..%x]." % + (addr, addr + data_sz - 1, ps.addr, ps.addr + seg_len - 1)) if (addr + data_sz) > ps.addr and (addr + data_sz) <= (ps.addr + seg_len): - raise ESPCoreDumpError("Can not add overlapping region [%x..%x] to ELF file. Conflict with existing [%x..%x]." % - (addr, addr + data_sz - 1, ps.addr, ps.addr + seg_len - 1)) + raise ESPCoreDumpError("Can not add overlapping region [%x..%x] to ELF file. Conflict with existing [%x..%x]." % + (addr, addr + data_sz - 1, ps.addr, ps.addr + seg_len - 1)) # append self.program_segments.append(ESPCoreDumpSegment(addr, data, type, flags)) @@ -434,11 +434,11 @@ class ESPCoreDumpElfFile(esptool.ELFFile): phdr.p_type = self.program_segments[i].type phdr.p_offset = cur_off phdr.p_vaddr = self.program_segments[i].addr - phdr.p_paddr = phdr.p_vaddr # TODO + phdr.p_paddr = phdr.p_vaddr # TODO phdr.p_filesz = len(self.program_segments[i].data) - phdr.p_memsz = phdr.p_filesz # TODO + phdr.p_memsz = phdr.p_filesz # TODO phdr.p_flags = self.program_segments[i].flags - phdr.p_align = 0 # TODO + phdr.p_align = 0 # TODO f.write(phdr.dump()) cur_off += phdr.p_filesz # write program segments @@ -463,7 +463,7 @@ class ESPCoreDumpLoader(object): ESP32_COREDUMP_HDR_SZ = struct.calcsize(ESP32_COREDUMP_HDR_FMT) ESP32_COREDUMP_TSK_HDR_FMT = '<3L' ESP32_COREDUMP_TSK_HDR_SZ = struct.calcsize(ESP32_COREDUMP_TSK_HDR_FMT) - + def __init__(self): """Base constructor for core dump loader """ @@ -475,51 +475,51 @@ class ESPCoreDumpLoader(object): # from "gdb/xtensa-tdep.h" # typedef struct # { - #0 xtensa_elf_greg_t pc; - #1 xtensa_elf_greg_t ps; - #2 xtensa_elf_greg_t lbeg; - #3 xtensa_elf_greg_t lend; - #4 xtensa_elf_greg_t lcount; - #5 xtensa_elf_greg_t sar; - #6 xtensa_elf_greg_t windowstart; - #7 xtensa_elf_greg_t windowbase; - #8..63 xtensa_elf_greg_t reserved[8+48]; - #64 xtensa_elf_greg_t ar[64]; + # 0 xtensa_elf_greg_t pc; + # 1 xtensa_elf_greg_t ps; + # 2 xtensa_elf_greg_t lbeg; + # 3 xtensa_elf_greg_t lend; + # 4 xtensa_elf_greg_t lcount; + # 5 xtensa_elf_greg_t sar; + # 6 xtensa_elf_greg_t windowstart; + # 7 xtensa_elf_greg_t windowbase; + # 8..63 xtensa_elf_greg_t reserved[8+48]; + # 64 xtensa_elf_greg_t ar[64]; # } xtensa_elf_gregset_t; - REG_PC_IDX=0 - REG_PS_IDX=1 - REG_LB_IDX=2 - REG_LE_IDX=3 - REG_LC_IDX=4 - REG_SAR_IDX=5 - REG_WS_IDX=6 - REG_WB_IDX=7 - REG_AR_START_IDX=64 - REG_AR_NUM=64 - # FIXME: acc to xtensa_elf_gregset_t number of regs must be 128, + REG_PC_IDX = 0 + REG_PS_IDX = 1 + REG_LB_IDX = 2 + REG_LE_IDX = 3 + REG_LC_IDX = 4 + REG_SAR_IDX = 5 + # REG_WS_IDX = 6 + # REG_WB_IDX = 7 + REG_AR_START_IDX = 64 + # REG_AR_NUM = 64 + # FIXME: acc to xtensa_elf_gregset_t number of regs must be 128, # but gdb complanis when it less then 129 - REG_NUM=129 + REG_NUM = 129 - XT_SOL_EXIT=0 - XT_SOL_PC=1 - XT_SOL_PS=2 - XT_SOL_NEXT=3 - XT_SOL_AR_START=4 - XT_SOL_AR_NUM=4 - XT_SOL_FRMSZ=8 + # XT_SOL_EXIT = 0 + XT_SOL_PC = 1 + XT_SOL_PS = 2 + # XT_SOL_NEXT = 3 + XT_SOL_AR_START = 4 + XT_SOL_AR_NUM = 4 + # XT_SOL_FRMSZ = 8 - XT_STK_EXIT=0 - XT_STK_PC=1 - XT_STK_PS=2 - XT_STK_AR_START=3 - XT_STK_AR_NUM=16 - XT_STK_SAR=19 - XT_STK_EXCCAUSE=20 - XT_STK_EXCVADDR=21 - XT_STK_LBEG=22 - XT_STK_LEND=23 - XT_STK_LCOUNT=24 - XT_STK_FRMSZ=25 + XT_STK_EXIT = 0 + XT_STK_PC = 1 + XT_STK_PS = 2 + XT_STK_AR_START = 3 + XT_STK_AR_NUM = 16 + XT_STK_SAR = 19 + # XT_STK_EXCCAUSE = 20 + # XT_STK_EXCVADDR = 21 + XT_STK_LBEG = 22 + XT_STK_LEND = 23 + XT_STK_LCOUNT = 24 + XT_STK_FRMSZ = 25 regs = [0] * REG_NUM # TODO: support for growing up stacks @@ -541,7 +541,7 @@ class ESPCoreDumpLoader(object): regs[REG_LB_IDX] = stack[XT_STK_LBEG] regs[REG_LE_IDX] = stack[XT_STK_LEND] regs[REG_LC_IDX] = stack[XT_STK_LCOUNT] - # FIXME: crashed and some running tasks (e.g. prvIdleTask) have EXCM bit set + # FIXME: crashed and some running tasks (e.g. prvIdleTask) have EXCM bit set # and GDB can not unwind callstack properly (it implies not windowed call0) if regs[REG_PS_IDX] & (1 << 5): regs[REG_PS_IDX] &= ~(1 << 4) @@ -550,14 +550,14 @@ class ESPCoreDumpLoader(object): regs[REG_PS_IDX] = stack[XT_SOL_PS] for i in range(XT_SOL_AR_NUM): regs[REG_AR_START_IDX + i] = stack[XT_SOL_AR_START + i] - nxt = stack[XT_SOL_NEXT] - + # nxt = stack[XT_SOL_NEXT] + # TODO: remove magic hack with saved PC to get proper value regs[REG_PC_IDX] = ((regs[REG_PC_IDX] & 0x3FFFFFFF) | 0x40000000) if regs[REG_PC_IDX] & 0x80000000: - regs[REG_PC_IDX] = (regs[REG_PC_IDX] & 0x3fffffff) | 0x40000000; + regs[REG_PC_IDX] = (regs[REG_PC_IDX] & 0x3fffffff) | 0x40000000 if regs[REG_AR_START_IDX + 0] & 0x80000000: - regs[REG_AR_START_IDX + 0] = (regs[REG_AR_START_IDX + 0] & 0x3fffffff) | 0x40000000; + regs[REG_AR_START_IDX + 0] = (regs[REG_AR_START_IDX + 0] & 0x3fffffff) | 0x40000000 return regs def remove_tmp_file(self, fname): @@ -587,7 +587,7 @@ class ESPCoreDumpLoader(object): raise ESPCoreDumpLoaderError("Core dump version '%d' is not supported! Should be up to '%d'." % (coredump_ver, self.ESP32_COREDUMP_VESION)) tcbsz_aligned = tcbsz if tcbsz_aligned % 4: - tcbsz_aligned = 4*(old_div(tcbsz_aligned,4) + 1) + tcbsz_aligned = 4 * (old_div(tcbsz_aligned,4) + 1) core_off += self.ESP32_COREDUMP_HDR_SZ core_elf = ESPCoreDumpElfFile() notes = b'' @@ -600,17 +600,18 @@ class ESPCoreDumpLoader(object): else: stack_len = stack_top - stack_end stack_base = stack_end - + stack_len_aligned = stack_len if stack_len_aligned % 4: - stack_len_aligned = 4*(old_div(stack_len_aligned,4) + 1) - + stack_len_aligned = 4 * (old_div(stack_len_aligned,4) + 1) + core_off += self.ESP32_COREDUMP_TSK_HDR_SZ logging.info("Read TCB %d bytes @ 0x%x" % (tcbsz_aligned, tcb_addr)) data = self.read_data(core_off, tcbsz_aligned) try: if tcbsz != tcbsz_aligned: - core_elf.add_program_segment(tcb_addr, data[:tcbsz - tcbsz_aligned], ESPCoreDumpElfFile.PT_LOAD, ESPCoreDumpSegment.PF_R | ESPCoreDumpSegment.PF_W) + core_elf.add_program_segment(tcb_addr, data[:tcbsz - tcbsz_aligned], + ESPCoreDumpElfFile.PT_LOAD, ESPCoreDumpSegment.PF_R | ESPCoreDumpSegment.PF_W) else: core_elf.add_program_segment(tcb_addr, data, ESPCoreDumpElfFile.PT_LOAD, ESPCoreDumpSegment.PF_R | ESPCoreDumpSegment.PF_W) except ESPCoreDumpError as e: @@ -632,8 +633,8 @@ class ESPCoreDumpLoader(object): print(e) return None prstatus = XtensaPrStatus() - prstatus.pr_cursig = 0 # TODO: set sig only for current/failed task - prstatus.pr_pid = i # TODO: use pid assigned by OS + prstatus.pr_cursig = 0 # TODO: set sig only for current/failed task + prstatus.pr_pid = i # TODO: use pid assigned by OS note = Elf32NoteDesc("CORE", 1, prstatus.dump() + struct.pack("<%dL" % len(task_regs), *task_regs)).dump() notes += note @@ -650,7 +651,7 @@ class ESPCoreDumpLoader(object): core_elf.add_program_segment(ps.addr, ps.data, ESPCoreDumpElfFile.PT_LOAD, ps.flags) except ESPCoreDumpError as e: logging.warning("Skip ROM segment %d bytes @ 0x%x. (Reason: %s)" % (len(ps.data), ps.addr, e)) - + core_elf.e_type = ESPCoreDumpElfFile.ET_CORE core_elf.e_machine = ESPCoreDumpElfFile.EM_XTENSA if core_fname: @@ -673,7 +674,7 @@ class ESPCoreDumpLoader(object): class ESPCoreDumpFileLoader(ESPCoreDumpLoader): """Core dump file loader class """ - def __init__(self, path, b64 = False): + def __init__(self, path, b64=False): """Constructor for core dump file loader """ super(ESPCoreDumpFileLoader, self).__init__() @@ -725,7 +726,7 @@ class ESPCoreDumpFlashLoader(ESPCoreDumpLoader): if e == '.pyc': self.path = self.path[:-1] else: - self.path = tool_path + self.path = tool_path self.port = port self.baud = baud self.chip = chip @@ -780,7 +781,7 @@ class ESPCoreDumpFlashLoader(ESPCoreDumpLoader): data = self.read_data(0, self.dump_sz - self.ESP32_COREDUMP_FLASH_CRC_SZ) data_crc = binascii.crc32(data) & 0xffffffff if dump_crc != data_crc: - raise ESPCoreDumpLoaderError("Invalid core dump CRC %x, should be %x" % (data_crc, dump_crc)) + raise ESPCoreDumpLoaderError("Invalid core dump CRC %x, should be %x" % (data_crc, dump_crc)) return super(ESPCoreDumpFlashLoader, self).create_corefile(core_fname) @@ -876,8 +877,9 @@ class GDBMIStreamConsoleHandler(GDBMIOutStreamHandler): """ TAG = '~' + def load_aux_elf(elf_path): - """ Loads auxilary ELF file and composes GDB command to read its symbols + """ Loads auxilary ELF file and composes GDB command to read its symbols """ elf = None sym_cmd = '' @@ -888,6 +890,7 @@ def load_aux_elf(elf_path): sym_cmd = 'add-symbol-file %s 0x%x' % (elf_path, s.addr) return (elf, sym_cmd) + def dbg_corefile(args): """ Command to load core dump from file or flash and run GDB debug session with it """ @@ -911,18 +914,18 @@ def dbg_corefile(args): loader.cleanup() return - p = subprocess.Popen( - bufsize = 0, - args = [args.gdb, - '--nw', # ignore .gdbinit - '--core=%s' % core_fname, # core file, - '-ex', rom_sym_cmd, - args.prog], - stdin = None, stdout = None, stderr = None, - close_fds = CLOSE_FDS - ) + p = subprocess.Popen(bufsize=0, + args=[args.gdb, + '--nw', # ignore .gdbinit + '--core=%s' % core_fname, # core file, + '-ex', rom_sym_cmd, + args.prog + ], + stdin=None, stdout=None, stderr=None, + close_fds=CLOSE_FDS + ) p.wait() - + if loader: if not args.core and not args.save_core: loader.remove_tmp_file(core_fname) @@ -931,13 +934,14 @@ def dbg_corefile(args): def info_corefile(args): - """ Command to load core dump from file or flash and print it's data in user friendly form + """ Command to load core dump from file or flash and print it's data in user friendly form """ global CLOSE_FDS + def gdbmi_console_stream_handler(ln): sys.stdout.write(ln) sys.stdout.flush() - + def gdbmi_read2prompt(f, out_handlers=None): while True: ln = f.readline().decode('utf-8').rstrip(' \r\n') @@ -953,20 +957,18 @@ def info_corefile(args): def gdbmi_start(handlers, gdb_cmds): gdb_args = [args.gdb, - '--quiet', # inhibit dumping info at start-up - '--nx', # inhibit window interface - '--nw', # ignore .gdbinit - '--interpreter=mi2', # use GDB/MI v2 - '--core=%s' % core_fname] # core file + '--quiet', # inhibit dumping info at start-up + '--nx', # inhibit window interface + '--nw', # ignore .gdbinit + '--interpreter=mi2', # use GDB/MI v2 + '--core=%s' % core_fname] # core file for c in gdb_cmds: gdb_args += ['-ex', c] gdb_args.append(args.prog) - p = subprocess.Popen( - bufsize = 0, - args = gdb_args, - stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, - close_fds = CLOSE_FDS - ) + p = subprocess.Popen(bufsize=0, + args=gdb_args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + close_fds=CLOSE_FDS) gdbmi_read2prompt(p.stdout, handlers) return p @@ -1093,13 +1095,13 @@ def info_corefile(args): p.wait() p.stdin.close() p.stdout.close() - + if loader: if not args.core and not args.save_core: loader.remove_tmp_file(core_fname) loader.cleanup() print('Done!') - + def main(): parser = argparse.ArgumentParser(description='espcoredump.py v%s - ESP32 Core Dump Utility' % __version__, prog='espcoredump') @@ -1130,9 +1132,12 @@ def main(): parser_debug_coredump.add_argument('--debug', '-d', help='Log level (0..3)', type=int, default=2) parser_debug_coredump.add_argument('--gdb', '-g', help='Path to gdb', default='xtensa-esp32-elf-gdb') parser_debug_coredump.add_argument('--core', '-c', help='Path to core dump file (if skipped core dump will be read from flash)', type=str) - parser_debug_coredump.add_argument('--core-format', '-t', help='(elf, raw or b64). File specified with "-c" is an ELF ("elf"), raw (raw) or base64-encoded (b64) binary', type=str, default='elf') - parser_debug_coredump.add_argument('--off', '-o', help='Ofsset of coredump partition in flash (type "make partition_table" to see).', type=int, default=0x110000) - parser_debug_coredump.add_argument('--save-core', '-s', help='Save core to file. Othwerwise temporary core file will be deleted. Ignored with "-c"', type=str) + parser_debug_coredump.add_argument('--core-format', '-t', help='(elf, raw or b64). File specified with "-c" is an ELF ("elf"), ' + 'raw (raw) or base64-encoded (b64) binary', type=str, default='elf') + parser_debug_coredump.add_argument('--off', '-o', help='Ofsset of coredump partition in flash ' + '(type "make partition_table" to see).', type=int, default=0x110000) + parser_debug_coredump.add_argument('--save-core', '-s', help='Save core to file. Othwerwise temporary core file will be deleted. ' + 'Ignored with "-c"', type=str) parser_debug_coredump.add_argument('--rom-elf', '-r', help='Path to ROM ELF file.', type=str, default='esp32_rom.elf') parser_debug_coredump.add_argument('prog', help='Path to program\'s ELF binary', type=str) @@ -1142,9 +1147,12 @@ def main(): parser_info_coredump.add_argument('--debug', '-d', help='Log level (0..3)', type=int, default=0) parser_info_coredump.add_argument('--gdb', '-g', help='Path to gdb', default='xtensa-esp32-elf-gdb') parser_info_coredump.add_argument('--core', '-c', help='Path to core dump file (if skipped core dump will be read from flash)', type=str) - parser_info_coredump.add_argument('--core-format', '-t', help='(elf, raw or b64). File specified with "-c" is an ELF ("elf"), raw (raw) or base64-encoded (b64) binary', type=str, default='elf') - parser_info_coredump.add_argument('--off', '-o', help='Ofsset of coredump partition in flash (type "make partition_table" to see).', type=int, default=0x110000) - parser_info_coredump.add_argument('--save-core', '-s', help='Save core to file. Othwerwise temporary core file will be deleted. Does not work with "-c"', type=str) + parser_info_coredump.add_argument('--core-format', '-t', help='(elf, raw or b64). File specified with "-c" is an ELF ("elf"), ' + 'raw (raw) or base64-encoded (b64) binary', type=str, default='elf') + parser_info_coredump.add_argument('--off', '-o', help='Offset of coredump partition in flash (type ' + '"make partition_table" to see).', type=int, default=0x110000) + parser_info_coredump.add_argument('--save-core', '-s', help='Save core to file. Othwerwise temporary core file will be deleted. ' + 'Does not work with "-c"', type=str) parser_info_coredump.add_argument('--rom-elf', '-r', help='Path to ROM ELF file.', type=str, default='esp32_rom.elf') parser_info_coredump.add_argument('--print-mem', '-m', help='Print memory dump', action='store_true') parser_info_coredump.add_argument('prog', help='Path to program\'s ELF binary', type=str) diff --git a/components/espcoredump/test/test_espcoredump.py b/components/espcoredump/test/test_espcoredump.py index 6b7da2612..8d2ea9322 100755 --- a/components/espcoredump/test/test_espcoredump.py +++ b/components/espcoredump/test/test_espcoredump.py @@ -18,10 +18,14 @@ import sys import os import unittest -idf_path = os.getenv('IDF_PATH') -if idf_path: - sys.path.insert(0, os.path.join(idf_path, 'components', 'espcoredump')) -import espcoredump +try: + import espcoredump +except ImportError: + idf_path = os.getenv('IDF_PATH') + if idf_path: + sys.path.insert(0, os.path.join(idf_path, 'components', 'espcoredump')) + import espcoredump + class TestESPCoreDumpFileLoader(unittest.TestCase): def setUp(self): @@ -34,15 +38,16 @@ class TestESPCoreDumpFileLoader(unittest.TestCase): def testESPCoreDumpFileLoaderWithoutB64(self): t = espcoredump.ESPCoreDumpFileLoader(path='coredump.b64', b64=False) - self.assertIsInstance(t, espcoredump.ESPCoreDumpFileLoader) # invoke for coverage of open() + self.assertIsInstance(t, espcoredump.ESPCoreDumpFileLoader) # invoke for coverage of open() t.cleanup() def test_cannot_remove_dir(self): - self.dloader.remove_tmp_file(fname='.') # silent failure (but covers exception inside) + self.dloader.remove_tmp_file(fname='.') # silent failure (but covers exception inside) def test_create_corefile(self): self.assertEqual(self.dloader.create_corefile(core_fname=self.tmp_file, off=0, rom_elf=None), self.tmp_file) + if __name__ == '__main__': # The purpose of these tests is to increase the code coverage at places which are sensitive to issues related to # Python 2&3 compatibility. diff --git a/components/partition_table/gen_esp32part.py b/components/partition_table/gen_esp32part.py index 53a65efc0..c7b8d5021 100755 --- a/components/partition_table/gen_esp32part.py +++ b/components/partition_table/gen_esp32part.py @@ -32,7 +32,7 @@ import binascii import errno MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature -MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum +MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum PARTITION_TABLE_SIZE = 0x1000 # Size of partition table MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 @@ -44,25 +44,25 @@ APP_TYPE = 0x00 DATA_TYPE = 0x01 TYPES = { - "app" : APP_TYPE, - "data" : DATA_TYPE, + "app": APP_TYPE, + "data": DATA_TYPE, } # Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h SUBTYPES = { - APP_TYPE : { - "factory" : 0x00, - "test" : 0x20, + APP_TYPE: { + "factory": 0x00, + "test": 0x20, }, - DATA_TYPE : { - "ota" : 0x00, - "phy" : 0x01, - "nvs" : 0x02, - "coredump" : 0x03, - "nvs_keys" : 0x04, - "esphttpd" : 0x80, - "fat" : 0x81, - "spiffs" : 0x82, + DATA_TYPE: { + "ota": 0x00, + "phy": 0x01, + "nvs": 0x02, + "coredump": 0x03, + "nvs_keys": 0x04, + "esphttpd": 0x80, + "fat": 0x81, + "spiffs": 0x82, }, } @@ -71,16 +71,19 @@ md5sum = True secure = False offset_part_table = 0 + def status(msg): """ Print status message to stderr """ if not quiet: critical(msg) + def critical(msg): """ Print critical message to stderr """ sys.stderr.write(msg) sys.stderr.write('\n') + class PartitionTable(list): def __init__(self): super(PartitionTable, self).__init__(self) @@ -102,15 +105,15 @@ class PartitionTable(list): if line.startswith("#") or len(line) == 0: continue try: - res.append(PartitionDefinition.from_csv(line, line_no+1)) + res.append(PartitionDefinition.from_csv(line, line_no + 1)) except InputError as e: - raise InputError("Error at line %d: %s" % (line_no+1, e)) + raise InputError("Error at line %d: %s" % (line_no + 1, e)) except Exception: - critical("Unexpected error parsing CSV line %d: %s" % (line_no+1, line)) + critical("Unexpected error parsing CSV line %d: %s" % (line_no + 1, line)) raise # fix up missing offsets & negative sizes - last_end = offset_part_table + PARTITION_TABLE_SIZE # first offset after partition table + last_end = offset_part_table + PARTITION_TABLE_SIZE # first offset after partition table for e in res: if e.offset is not None and e.offset < last_end: if e == res[0]: @@ -149,14 +152,14 @@ class PartitionTable(list): ptype = TYPES[ptype] except KeyError: try: - ptypes = int(ptype, 0) + ptype = int(ptype, 0) except TypeError: pass try: subtype = SUBTYPES[int(ptype)][subtype] except KeyError: try: - ptypes = int(ptype, 0) + ptype = int(ptype, 0) except TypeError: pass @@ -175,11 +178,11 @@ class PartitionTable(list): # verify each partition individually for p in self: p.verify() - + # check on duplicate name - names = [ p.name for p in self ] - duplicates = set( n for n in names if names.count(n) > 1 ) - + names = [p.name for p in self] + duplicates = set(n for n in names if names.count(n) > 1) + # print sorted duplicate partitions by name if len(duplicates) != 0: print("A list of partitions that have the same name:") @@ -187,14 +190,14 @@ class PartitionTable(list): if len(duplicates.intersection([p.name])) != 0: print("%s" % (p.to_csv())) raise InputError("Partition names must be unique") - + # check for overlaps last = None for p in sorted(self, key=lambda x:x.offset): if p.offset < offset_part_table + PARTITION_TABLE_SIZE: raise InputError("Partition offset 0x%x is below 0x%x" % (p.offset, offset_part_table + PARTITION_TABLE_SIZE)) if last is not None and p.offset < last.offset + last.size: - raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset+last.size-1)) + raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset + last.size - 1)) last = p def flash_size(self): @@ -209,17 +212,17 @@ class PartitionTable(list): @classmethod def from_binary(cls, b): - md5 = hashlib.md5(); + md5 = hashlib.md5() result = cls() for o in range(0,len(b),32): - data = b[o:o+32] + data = b[o:o + 32] if len(data) != 32: raise InputError("Partition table length must be a multiple of 32 bytes") - if data == b'\xFF'*32: + if data == b'\xFF' * 32: return result # got end marker - if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]: #check only the magic number part + if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]: # check only the magic number part if data[16:] == md5.digest(): - continue # the next iteration will check for the end marker + continue # the next iteration will check for the end marker else: raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:]))) else: @@ -231,29 +234,30 @@ class PartitionTable(list): result = b"".join(e.to_binary() for e in self) if md5sum: result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() - if len(result )>= MAX_PARTITION_LENGTH: + if len(result) >= MAX_PARTITION_LENGTH: raise InputError("Binary partition table length (%d) longer than max" % len(result)) result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing return result def to_csv(self, simple_formatting=False): - rows = [ "# Espressif ESP32 Partition Table", - "# Name, Type, SubType, Offset, Size, Flags" ] - rows += [ x.to_csv(simple_formatting) for x in self ] + rows = ["# Espressif ESP32 Partition Table", + "# Name, Type, SubType, Offset, Size, Flags"] + rows += [x.to_csv(simple_formatting) for x in self] return "\n".join(rows) + "\n" + class PartitionDefinition(object): MAGIC_BYTES = b"\xAA\x50" ALIGNMENT = { - APP_TYPE : 0x10000, - DATA_TYPE : 0x04, + APP_TYPE: 0x10000, + DATA_TYPE: 0x04, } # dictionary maps flag name (as used in CSV flags list, property name) # to bit set in flags words in binary format FLAGS = { - "encrypted" : 0 + "encrypted": 0 } # add subtypes for the 16 OTA slot values ("ota_XX, etc.") @@ -272,7 +276,7 @@ class PartitionDefinition(object): def from_csv(cls, line, line_no): """ Parse a line from the CSV """ line_w_defaults = line + ",,,," # lazy way to support default fields - fields = [ f.strip() for f in line_w_defaults.split(",") ] + fields = [f.strip() for f in line_w_defaults.split(",")] res = PartitionDefinition() res.line_no = line_no @@ -302,7 +306,7 @@ class PartitionDefinition(object): def maybe_hex(x): return "0x%x" % x if x is not None else "None" return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, - maybe_hex(self.offset), maybe_hex(self.size)) + maybe_hex(self.offset), maybe_hex(self.size)) def __str__(self): return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1) @@ -329,7 +333,7 @@ class PartitionDefinition(object): def parse_subtype(self, strval): if strval == "": - return 0 # default + return 0 # default return parse_int(strval, SUBTYPES.get(self.type, {})) def parse_address(self, strval): @@ -353,12 +357,14 @@ class PartitionDefinition(object): raise ValidationError(self, "Size field is not set") if self.name in TYPES and TYPES.get(self.name, "") != self.type: - critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's type (0x%x). Mistake in partition table?" % (self.name, self.type)) + critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's " + "type (0x%x). Mistake in partition table?" % (self.name, self.type)) all_subtype_names = [] for names in (t.keys() for t in SUBTYPES.values()): all_subtype_names += names if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype: - critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype)) + critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has " + "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype)) STRUCT_FORMAT = b"<2sBBLL16sL" @@ -369,21 +375,21 @@ class PartitionDefinition(object): res = cls() (magic, res.type, res.subtype, res.offset, res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) - if b"\x00" in res.name: # strip null byte padding from name string + if b"\x00" in res.name: # strip null byte padding from name string res.name = res.name[:res.name.index(b"\x00")] res.name = res.name.decode() if magic != cls.MAGIC_BYTES: raise InputError("Invalid magic bytes (%r) for partition definition" % magic) for flag,bit in cls.FLAGS.items(): - if flags & (1< > " ); + description = ("This application generates .h and .ld files for symbols defined in input file. " + "The input symbols file can be generated using nm utility like this: " + "esp32-ulp-nm -g -f posix > ") parser = OptionParser(description=description) parser.add_option("-s", "--symfile", dest="symfile", @@ -44,11 +45,10 @@ def main(): parser.print_help() return 1 - with open(options.outputfile + ".h", 'w') as f_h, \ - open(options.outputfile + ".ld", 'w') as f_ld, \ - open(options.symfile) as f_sym: \ + with open(options.outputfile + ".h", 'w') as f_h, open(options.outputfile + ".ld", 'w') as f_ld, open(options.symfile) as f_sym: gen_ld_h_from_sym(f_sym, f_ld, f_h) return 0 + if __name__ == "__main__": - exit(main()); + exit(main())