Crash_Override
/
OVMS3-idf
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'feature/partition_tools' into 'master' 

Partition level tools

See merge request idf/esp-idf!3355
This commit is contained in:
Angus Gratton 2018-11-30 14:15:51 +08:00
parent c22512d27f b926764385
commit a34a27010b
35 changed files with 1385 additions and 347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.flake8
View File

@ -203,11 +203,13 @@ exclude =
examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
examples/provisioning/softap_prov/softap_prov_test.py,
examples/provisioning/softap_prov/utils/wifi_tools.py,
examples/storage/parttool/example_test.py,
examples/system/cpp_exceptions/example_test.py,
examples/system/esp_event/default_event_loop/example_test.py,
examples/system/esp_event/user_event_loops/example_test.py,
examples/system/esp_timer/example_test.py,
examples/system/light_sleep/example_test.py,
examples/system/ota/otatool/example_test.py,
examples/wifi/iperf/iperf_test.py,
examples/wifi/iperf/test_report.py,
tools/check_python_dependencies.py,

17
components/app_update/CMakeLists.txt
View File

@ -7,12 +7,21 @@ set(COMPONENT_PRIV_REQUIRES bootloader_support)
register_component()
# Add custom target for generating empty otadata partition for flashing
if(OTADATA_PARTITION_OFFSET AND IDF_BUILD_ARTIFACTS)
if(OTADATA_PARTITION_OFFSET AND OTADATA_PARTITION_SIZE)
add_custom_command(OUTPUT "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}"
COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/gen_empty_partition.py
--size ${OTADATA_PARTITION_SIZE} "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}")
COMMAND ${PYTHON} ${IDF_PATH}/components/partition_table/parttool.py
--partition-type data --partition-subtype ota -q
--partition-table-file ${PARTITION_CSV_PATH} generate_blank_partition_file
--output "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}")
add_custom_target(blank_ota_data ALL DEPENDS "${IDF_BUILD_ARTIFACTS_DIR}/${BLANK_OTADATA_FILE}")
add_dependencies(flash blank_ota_data)
endif()
set(otatool_py ${PYTHON} ${COMPONENT_PATH}/otatool.py)
add_custom_target(read_otadata DEPENDS "${PARTITION_CSV_PATH}"
COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} read_otadata)
add_custom_target(erase_otadata DEPENDS "${PARTITION_CSV_PATH}"
COMMAND ${otatool_py} --partition-table-file ${PARTITION_CSV_PATH} erase_otadata)

62
components/app_update/Makefile.projbuild
View File

@ -1,60 +1,38 @@
# Generate partition binary
#
.PHONY: dump_otadata erase_ota blank_ota_data
.PHONY: blank_ota_data erase_otadata read_otadata
GEN_EMPTY_PART := $(PYTHON) $(COMPONENT_PATH)/gen_empty_partition.py
OTATOOL_PY := $(PYTHON) $(COMPONENT_PATH)/otatool.py
PARTTOOL_PY := $(PYTHON) $(IDF_PATH)/components/partition_table/parttool.py
# Generate blank partition file
BLANK_OTA_DATA_FILE = $(BUILD_DIR_BASE)/ota_data_initial.bin
PARTITION_TABLE_LEN := 0xC00
OTADATA_LEN := 0x2000
$(BLANK_OTA_DATA_FILE): partition_table_get_info $(PARTITION_TABLE_BIN) | check_python_dependencies
$(shell if [ $(OTA_DATA_OFFSET) != "" ] && [ $(OTA_DATA_SIZE) != "" ]; then \
$(PARTTOOL_PY) --partition-type data --partition-subtype ota --partition-table-file $(PARTITION_TABLE_BIN) \
-q generate_blank_partition_file --output $(BLANK_OTA_DATA_FILE); \
fi; )
$(eval BLANK_OTA_DATA_FILE = $(shell if [ $(OTA_DATA_OFFSET) != "" ] && [ $(OTA_DATA_SIZE) != "" ]; then \
echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
PARTITION_TABLE_ONCHIP_BIN_PATH := $(call dequote,$(abspath $(BUILD_DIR_BASE)))
PARTITION_TABLE_ONCHIP_BIN_NAME := "onchip_partition.bin"
OTADATA_ONCHIP_BIN_NAME := "onchip_otadata.bin"
PARTITION_TABLE_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(PARTITION_TABLE_ONCHIP_BIN_NAME))
OTADATA_ONCHIP_BIN := $(PARTITION_TABLE_ONCHIP_BIN_PATH)/$(call dequote,$(OTADATA_ONCHIP_BIN_NAME))
PARTITION_TABLE_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(PARTITION_TABLE_OFFSET) $(PARTITION_TABLE_LEN) $(PARTITION_TABLE_ONCHIP_BIN)
OTADATA_GET_BIN_CMD = $(ESPTOOLPY_SERIAL) read_flash $(OTADATA_OFFSET) $(OTADATA_LEN) $(OTADATA_ONCHIP_BIN)
GEN_OTADATA = $(IDF_PATH)/components/app_update/dump_otadata.py
ERASE_OTADATA_CMD = $(ESPTOOLPY_SERIAL) erase_region $(OTADATA_OFFSET) $(OTADATA_LEN)
blank_ota_data: $(BLANK_OTA_DATA_FILE)
# If there is no otadata partition, both OTA_DATA_OFFSET and BLANK_OTA_DATA_FILE
# expand to empty values.
ESPTOOL_ALL_FLASH_ARGS += $(OTA_DATA_OFFSET) $(BLANK_OTA_DATA_FILE)
$(PARTITION_TABLE_ONCHIP_BIN):
$(PARTITION_TABLE_GET_BIN_CMD)
erase_otadata: $(PARTITION_TABLE_BIN) partition_table_get_info | check_python_dependencies
$(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_BIN) erase_otadata
onchip_otadata_get_info: $(PARTITION_TABLE_ONCHIP_BIN)
$(eval OTADATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_ONCHIP_BIN)))
@echo $(if $(OTADATA_OFFSET), $(shell export OTADATA_OFFSET), $(shell rm -f $(PARTITION_TABLE_ONCHIP_BIN));$(error "ERROR: ESP32 does not have otadata partition."))
read_otadata: $(PARTITION_TABLE_BIN) partition_table_get_info | check_python_dependencies
$(OTATOOL_PY) --partition-table-file $(PARTITION_TABLE_BIN) read_otadata
$(OTADATA_ONCHIP_BIN):
$(OTADATA_GET_BIN_CMD)
dump_otadata: onchip_otadata_get_info $(OTADATA_ONCHIP_BIN) $(PARTITION_TABLE_ONCHIP_BIN)
@echo "otadata retrieved. Contents:"
@echo $(SEPARATOR)
$(GEN_OTADATA) $(OTADATA_ONCHIP_BIN)
@echo $(SEPARATOR)
rm -f $(PARTITION_TABLE_ONCHIP_BIN)
rm -f $(OTADATA_ONCHIP_BIN)
$(BLANK_OTA_DATA_FILE): partition_table_get_info
$(GEN_EMPTY_PART) --size $(OTA_DATA_SIZE) $(BLANK_OTA_DATA_FILE)
$(eval BLANK_OTA_DATA_FILE = $(shell if [ $(OTA_DATA_SIZE) != 0 ]; then echo $(BLANK_OTA_DATA_FILE); else echo " "; fi) )
blank_ota_data: $(BLANK_OTA_DATA_FILE)
erase_ota: partition_table_get_info | check_python_dependencies
@echo $(if $(OTA_DATA_OFFSET), "Erase ota_data [addr=$(OTA_DATA_OFFSET) size=$(OTA_DATA_SIZE)] ...", $(error "ERROR: Partition table does not have ota_data partition."))
$(ESPTOOLPY_SERIAL) erase_region $(OTA_DATA_OFFSET) $(OTA_DATA_SIZE)
erase_ota: erase_otadata
@echo "WARNING: erase_ota is deprecated. Use erase_otadata instead."
all: blank_ota_data
flash: blank_ota_data
clean:
rm -f $(BLANK_OTA_DATA_FILE)
rm -f $(BLANK_OTA_DATA_FILE)

88
components/app_update/dump_otadata.py
View File

@ -1,88 +0,0 @@
#!/usr/bin/env python
#
# gen_otadata prints info about the otadata partition.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function, division
import argparse
import os
import re
import struct
import sys
import hashlib
import binascii
__version__ = '1.0'
quiet = False
def status(msg):
""" Print status message to stderr """
if not quiet:
critical(msg)
def critical(msg):
""" Print critical message to stderr """
if not quiet:
sys.stderr.write(msg)
sys.stderr.write('\n')
def little_endian(buff, offset):
data = buff[offset:offset+4]
data.reverse()
data = ''.join(data)
return data
def main():
global quiet
parser = argparse.ArgumentParser(description='Prints otadata partition in human readable form.')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
search_type = parser.add_mutually_exclusive_group()
parser.add_argument('input', help='Path to binary file containing otadata partition to parse.',
type=argparse.FileType('rb'))
args = parser.parse_args()
quiet = args.quiet
input = args.input.read()
hex_input_0 = binascii.hexlify(input)
hex_input_0 = map(''.join, zip(*[iter(hex_input_0)]*2))
hex_input_1 = binascii.hexlify(input[4096:])
hex_input_1 = map(''.join, zip(*[iter(hex_input_1)]*2))
print("\t%11s\t%8s |\t%8s\t%8s" %("OTA_SEQ", "CRC", "OTA_SEQ", "CRC"))
print("Firmware: 0x%s \t 0x%s |\t0x%s \t 0x%s" % (little_endian(hex_input_0, 0), little_endian(hex_input_0, 28), \
little_endian(hex_input_1, 0), little_endian(hex_input_1, 28)))
class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
class ValidationError(InputError):
def __init__(self, partition, message):
super(ValidationError, self).__init__(
"Partition %s invalid: %s" % (partition.name, message))
if __name__ == '__main__':
try:
r = main()
sys.exit(r)
except InputError as e:
print(e, file=sys.stderr)
sys.exit(2)

82
components/app_update/gen_empty_partition.py
View File

@ -1,82 +0,0 @@
#!/usr/bin/env python
#
# generates an empty binary file
#
# This tool generates an empty binary file of the required size.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function, division
from __future__ import unicode_literals
import argparse
import os
import re
import struct
import sys
import hashlib
import binascii
__version__ = '1.0'
quiet = False
def status(msg):
""" Print status message to stderr """
if not quiet:
critical(msg)
def critical(msg):
""" Print critical message to stderr """
if not quiet:
sys.stderr.write(msg)
sys.stderr.write('\n')
def generate_blanked_file(size, output_path):
output = b"\xFF" * size
try:
stdout_binary = sys.stdout.buffer # Python 3
except AttributeError:
stdout_binary = sys.stdout
with stdout_binary if output_path == '-' else open(output_path, 'wb') as f:
f.write(output)
def main():
global quiet
parser = argparse.ArgumentParser(description='Generates an empty binary file of the required size.')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
parser.add_argument('--size', help='Size of generated the file', type=str, required=True)
parser.add_argument('output', help='Path for binary file.', nargs='?', default='-')
args = parser.parse_args()
quiet = args.quiet
size = int(args.size, 0)
if size > 0 :
generate_blanked_file(size, args.output)
return 0
class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
if __name__ == '__main__':
try:
r = main()
sys.exit(r)
except InputError as e:
print(e, file=sys.stderr)
sys.exit(2)

327
components/app_update/otatool.py Executable file
View File

@ -0,0 +1,327 @@
#!/usr/bin/env python
#
# otatool is used to perform ota-level operations - flashing ota partition
# erasing ota partition and switching ota partition
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function, division
import argparse
import os
import sys
import binascii
import subprocess
import tempfile
import collections
import struct
__version__ = '1.0'
IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components"))
PARTTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "partition_table", "parttool.py")
SPI_FLASH_SEC_SIZE = 0x2000
quiet = False
def status(msg):
if not quiet:
print(msg)
def _invoke_parttool(parttool_args, args, output=False, partition=None):
invoke_args = []
if partition:
invoke_args += [sys.executable, PARTTOOL_PY] + partition
else:
invoke_args += [sys.executable, PARTTOOL_PY, "--partition-type", "data", "--partition-subtype", "ota"]
if quiet:
invoke_args += ["-q"]
if args.port != "":
invoke_args += ["--port", args.port]
if args.partition_table_file:
invoke_args += ["--partition-table-file", args.partition_table_file]
if args.partition_table_offset:
invoke_args += ["--partition-table-offset", args.partition_table_offset]
invoke_args += parttool_args
if output:
return subprocess.check_output(invoke_args)
else:
return subprocess.check_call(invoke_args)
def _get_otadata_contents(args, check=True):
global quiet
if check:
check_args = ["get_partition_info", "--info", "offset", "size"]
quiet = True
output = _invoke_parttool(check_args, args, True).split(b" ")
quiet = args.quiet
if not output:
raise RuntimeError("No ota_data partition found")
with tempfile.NamedTemporaryFile() as otadata_file:
invoke_args = ["read_partition", "--output", otadata_file.name]
_invoke_parttool(invoke_args, args)
return otadata_file.read()
def _get_otadata_status(otadata_contents):
status = []
otadata_status = collections.namedtuple("otadata_status", "seq crc")
for i in range(2):
start = i * (SPI_FLASH_SEC_SIZE >> 1)
seq = bytearray(otadata_contents[start:start + 4])
crc = bytearray(otadata_contents[start + 28:start + 32])
seq = struct.unpack('>I', seq)
crc = struct.unpack('>I', crc)
status.append(otadata_status(seq[0], crc[0]))
return status
def read_otadata(args):
status("Reading ota_data partition contents...")
otadata_info = _get_otadata_contents(args)
otadata_info = _get_otadata_status(otadata_info)
print(otadata_info)
print("\t\t{:11}\t{:8s}|\t{:8s}\t{:8s}".format("OTA_SEQ", "CRC", "OTA_SEQ", "CRC"))
print("Firmware: 0x{:8x} \t 0x{:8x} |\t0x{:8x} \t 0x{:8x}".format(otadata_info[0].seq, otadata_info[0].crc,
otadata_info[1].seq, otadata_info[1].crc))
def erase_otadata(args):
status("Erasing ota_data partition contents...")
_invoke_parttool(["erase_partition"], args)
status("Erased ota_data partition contents")
def switch_otadata(args):
sys.path.append(os.path.join(IDF_COMPONENTS_PATH, "partition_table"))
import gen_esp32part as gen
def is_otadata_status_valid(status):
seq = status.seq % (1 << 32)
crc = hex(binascii.crc32(struct.pack("I", seq), 0xFFFFFFFF) % (1 << 32))
return seq < (int('0xFFFFFFFF', 16) % (1 << 32)) and status.crc == crc
status("Looking for ota app partitions...")
# In order to get the number of ota app partitions, we need the partition table
partition_table = None
with tempfile.NamedTemporaryFile() as partition_table_file:
invoke_args = ["get_partition_info", "--table", partition_table_file.name]
_invoke_parttool(invoke_args, args)
partition_table = partition_table_file.read()
partition_table = gen.PartitionTable.from_binary(partition_table)
ota_partitions = list()
for i in range(gen.NUM_PARTITION_SUBTYPE_APP_OTA):
ota_partition = filter(lambda p: p.subtype == (gen.MIN_PARTITION_SUBTYPE_APP_OTA + i), partition_table)
try:
ota_partitions.append(list(ota_partition)[0])
except IndexError:
break
ota_partitions = sorted(ota_partitions, key=lambda p: p.subtype)
if not ota_partitions:
raise RuntimeError("No ota app partitions found")
status("Verifying partition to switch to exists...")
# Look for the app partition to switch to
ota_partition_next = None
try:
if args.name:
ota_partition_next = filter(lambda p: p.name == args.name, ota_partitions)
else:
ota_partition_next = filter(lambda p: p.subtype - gen.MIN_PARTITION_SUBTYPE_APP_OTA == args.slot, ota_partitions)
ota_partition_next = list(ota_partition_next)[0]
except IndexError:
raise RuntimeError("Partition to switch to not found")
otadata_contents = _get_otadata_contents(args)
otadata_status = _get_otadata_status(otadata_contents)
# Find the copy to base the computation for ota sequence number on
otadata_compute_base = -1
# Both are valid, take the max as computation base
if is_otadata_status_valid(otadata_status[0]) and is_otadata_status_valid(otadata_status[1]):
if otadata_status[0].seq >= otadata_status[1].seq:
otadata_compute_base = 0
else:
otadata_compute_base = 1
# Only one copy is valid, use that
elif is_otadata_status_valid(otadata_status[0]):
otadata_compute_base = 0
elif is_otadata_status_valid(otadata_status[1]):
otadata_compute_base = 1
# Both are invalid (could be initial state - all 0xFF's)
else:
pass
ota_seq_next = 0
ota_partitions_num = len(ota_partitions)
target_seq = (ota_partition_next.subtype & 0x0F) + 1
# Find the next ota sequence number
if otadata_compute_base == 0 or otadata_compute_base == 1:
base_seq = otadata_status[otadata_compute_base].seq % (1 << 32)
i = 0
while base_seq > target_seq % ota_partitions_num + i * ota_partitions_num:
i += 1
ota_seq_next = target_seq % ota_partitions_num + i * ota_partitions_num
else:
ota_seq_next = target_seq
# Create binary data from computed values
ota_seq_next = struct.pack("I", ota_seq_next)
ota_seq_crc_next = binascii.crc32(ota_seq_next, 0xFFFFFFFF) % (1 << 32)
ota_seq_crc_next = struct.pack("I", ota_seq_crc_next)
with tempfile.NamedTemporaryFile() as otadata_next_file:
start = (1 if otadata_compute_base == 0 else 0) * (SPI_FLASH_SEC_SIZE >> 1)
otadata_next_file.write(otadata_contents)
otadata_next_file.seek(start)
otadata_next_file.write(ota_seq_next)
otadata_next_file.seek(start + 28)
otadata_next_file.write(ota_seq_crc_next)
otadata_next_file.flush()
_invoke_parttool(["write_partition", "--input", otadata_next_file.name], args)
status("Updated ota_data partition")
def _get_partition_specifier(args):
if args.name:
return ["--partition-name", args.name]
else:
return ["--partition-type", "app", "--partition-subtype", "ota_" + str(args.slot)]
def read_ota_partition(args):
invoke_args = ["read_partition", "--output", args.output]
_invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args))
status("Read ota partition contents to file {}".format(args.output))
def write_ota_partition(args):
invoke_args = ["write_partition", "--input", args.input]
_invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args))
status("Written contents of file {} to ota partition".format(args.input))
def erase_ota_partition(args):
invoke_args = ["erase_partition"]
_invoke_parttool(invoke_args, args, partition=_get_partition_specifier(args))
status("Erased contents of ota partition")
def main():
global quiet
parser = argparse.ArgumentParser("ESP-IDF OTA Partitions Tool")
parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true")
# There are two possible sources for the partition table: a device attached to the host
# or a partition table CSV/binary file. These sources are mutually exclusive.
partition_table_info_source_args = parser.add_mutually_exclusive_group()
partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="")
partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from", default="")
parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000")
subparsers = parser.add_subparsers(dest="operation", help="run otatool -h for additional help")
# Specify the supported operations
subparsers.add_parser("read_otadata", help="read otadata partition")
subparsers.add_parser("erase_otadata", help="erase otadata partition")
slot_or_name_parser = argparse.ArgumentParser(add_help=False)
slot_or_name_parser_args = slot_or_name_parser.add_mutually_exclusive_group()
slot_or_name_parser_args.add_argument("--slot", help="slot number of the ota partition", type=int)
slot_or_name_parser_args.add_argument("--name", help="name of the ota partition")
subparsers.add_parser("switch_otadata", help="switch otadata partition", parents=[slot_or_name_parser])
read_ota_partition_subparser = subparsers.add_parser("read_ota_partition", help="read contents of an ota partition", parents=[slot_or_name_parser])
read_ota_partition_subparser.add_argument("--output", help="file to write the contents of the ota partition to")
write_ota_partition_subparser = subparsers.add_parser("write_ota_partition", help="write contents to an ota partition", parents=[slot_or_name_parser])
write_ota_partition_subparser.add_argument("--input", help="file whose contents to write to the ota partition")
subparsers.add_parser("erase_ota_partition", help="erase contents of an ota partition", parents=[slot_or_name_parser])
args = parser.parse_args()
quiet = args.quiet
# No operation specified, display help and exit
if args.operation is None:
if not quiet:
parser.print_help()
sys.exit(1)
# Else execute the operation
operation_func = globals()[args.operation]
if quiet:
# If exceptions occur, suppress and exit quietly
try:
operation_func(args)
except Exception:
sys.exit(2)
else:
operation_func(args)
if __name__ == '__main__':
main()

3
components/app_update/project_include.cmake
View File

@ -3,6 +3,7 @@
# partition table
# (NB: because of component dependency, we know partition_table
# project_include.cmake has already been included.)
if(OTADATA_PARTITION_OFFSET AND IDF_BUILD_ARTIFACTS)
if(OTADATA_PARTITION_OFFSET AND OTADATA_PARTITION_SIZE AND IDF_BUILD_ARTIFACTS)
set(BLANK_OTADATA_FILE "ota_data_initial.bin")
endif()

12
components/partition_table/Makefile.projbuild
View File

@ -63,10 +63,14 @@ $(PARTITION_TABLE_BIN_UNSIGNED): $(PARTITION_TABLE_CSV_PATH) $(SDKCONFIG_MAKEFIL
all_binaries: $(PARTITION_TABLE_BIN) partition_table_get_info
partition_table_get_info: $(PARTITION_TABLE_BIN)
$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --type data --subtype phy --offset $(PARTITION_TABLE_BIN)))
$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --default-boot-partition --offset $(PARTITION_TABLE_BIN)))
$(eval OTA_DATA_SIZE := $(shell $(GET_PART_INFO) --type data --subtype ota --size $(PARTITION_TABLE_BIN) || echo 0))
$(eval OTA_DATA_OFFSET := $(shell $(GET_PART_INFO) --type data --subtype ota --offset $(PARTITION_TABLE_BIN)))
$(eval PHY_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype phy \
--partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset))
$(eval APP_OFFSET:=$(shell $(GET_PART_INFO) --partition-boot-default \
--partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset))
$(eval OTA_DATA_OFFSET:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \
--partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info offset))
$(eval OTA_DATA_SIZE:=$(shell $(GET_PART_INFO) --partition-type data --partition-subtype ota \
--partition-table-file $(PARTITION_TABLE_BIN) get_partition_info --info size))
export APP_OFFSET
export PHY_DATA_OFFSET

7
components/partition_table/gen_esp32part.py
View File

@ -35,6 +35,9 @@ MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in
MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum
PARTITION_TABLE_SIZE = 0x1000 # Size of partition table
MIN_PARTITION_SUBTYPE_APP_OTA = 0x10
NUM_PARTITION_SUBTYPE_APP_OTA = 16
__version__ = '1.2'
APP_TYPE = 0x00
@ -254,8 +257,8 @@ class PartitionDefinition(object):
}
# add subtypes for the 16 OTA slot values ("ota_XX, etc.")
for ota_slot in range(16):
SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = 0x10 + ota_slot
for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA):
SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot
def __init__(self):
self.name = ""

306
components/partition_table/parttool.py
View File

@ -1,9 +1,7 @@
#!/usr/bin/env python
#
# parttool returns info about the required partition.
#
# This utility is used by the make system to get information
# about the start addresses: partition table, factory area, phy area.
# parttool is used to perform partition level operations - reading,
# writing, erasing and getting info about the partition.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
@ -21,113 +19,259 @@
from __future__ import print_function, division
import argparse
import os
import re
import struct
import sys
import hashlib
import binascii
import subprocess
import tempfile
import gen_esp32part as gen
__version__ = '1.0'
IDF_COMPONENTS_PATH = os.path.expandvars(os.path.join("$IDF_PATH", "components"))
ESPTOOL_PY = os.path.join(IDF_COMPONENTS_PATH, "esptool_py", "esptool", "esptool.py")
quiet = False
def status(msg):
""" Print status message to stderr """
if not quiet:
critical(msg)
print(msg)
def critical(msg):
""" Print critical message to stderr """
if not quiet:
sys.stderr.write(msg)
sys.stderr.write('\n')
def main():
global quiet
parser = argparse.ArgumentParser(description='Returns info about the required partition.')
parser.add_argument('--quiet', '-q', help="Don't print status messages to stderr", action='store_true')
def _invoke_esptool(esptool_args, args):
m_esptool_args = [sys.executable, ESPTOOL_PY]
parser.add_argument('--partition-table-offset', help='The offset of the partition table in flash. Only consulted if partition table is in CSV format.', type=str, default='0x8000')
if args.port != "":
m_esptool_args.extend(["--port", args.port])
search_type = parser.add_mutually_exclusive_group()
search_type.add_argument('--partition-name', '-p', help='The name of the required partition', type=str, default=None)
search_type.add_argument('--type', '-t', help='The type of the required partition', type=str, default=None)
search_type.add_argument('--default-boot-partition', help='Select the default boot partition, '+
'using the same fallback logic as the IDF bootloader', action="store_true")
m_esptool_args.extend(esptool_args)
parser.add_argument('--subtype', '-s', help='The subtype of the required partition', type=str, default=None)
if quiet:
with open(os.devnull, "w") as fnull:
subprocess.check_call(m_esptool_args, stdout=fnull, stderr=fnull)
else:
subprocess.check_call(m_esptool_args)
parser.add_argument('--offset', '-o', help='Return offset of required partition', action="store_true")
parser.add_argument('--size', help='Return size of required partition', action="store_true")
parser.add_argument('input', help='Path to CSV or binary file to parse. Will use stdin if omitted.',
type=argparse.FileType('rb'), default=sys.stdin)
args = parser.parse_args()
if args.type is not None and args.subtype is None:
status("If --type is specified, --subtype is required")
return 2
if args.type is None and args.subtype is not None:
status("--subtype is only used with --type")
return 2
quiet = args.quiet
def _get_partition_table(args):
partition_table = None
gen.offset_part_table = int(args.partition_table_offset, 0)
input = args.input.read()
input_is_binary = input[0:2] == gen.PartitionDefinition.MAGIC_BYTES
if input_is_binary:
status("Parsing binary partition input...")
table = gen.PartitionTable.from_binary(input)
if args.partition_table_file:
status("Reading partition table from partition table file...")
try:
with open(args.partition_table_file, "rb") as partition_table_file:
partition_table = gen.PartitionTable.from_binary(partition_table_file.read())
status("Partition table read from binary file {}".format(partition_table_file.name))
except (gen.InputError, TypeError):
with open(args.partition_table_file, "r") as partition_table_file:
partition_table_file.seek(0)
partition_table = gen.PartitionTable.from_csv(partition_table_file.read())
status("Partition table read from CSV file {}".format(partition_table_file.name))
else:
input = input.decode()
status("Parsing CSV input...")
table = gen.PartitionTable.from_csv(input)
port_info = (" on port " + args.port if args.port else "")
status("Reading partition table from device{}...".format(port_info))
with tempfile.NamedTemporaryFile() as partition_table_file:
invoke_args = ["read_flash", str(gen.offset_part_table), str(gen.MAX_PARTITION_LENGTH), partition_table_file.name]
_invoke_esptool(invoke_args, args)
partition_table = gen.PartitionTable.from_binary(partition_table_file.read())
status("Partition table read from device" + port_info)
found_partition = None
return partition_table
if args.default_boot_partition:
search = [ "factory" ] + [ "ota_%d" % d for d in range(16) ]
def _get_partition(args):
partition_table = _get_partition_table(args)
partition = None
if args.partition_name:
partition = partition_table.find_by_name(args.partition_name)
elif args.partition_type and args.partition_subtype:
partition = partition_table.find_by_type(args.partition_type, args.partition_subtype)
elif args.partition_boot_default:
search = [ "factory" ] + [ "ota_{}".format(d) for d in range(16) ]
for subtype in search:
found_partition = table.find_by_type("app", subtype)
if found_partition is not None:
partition = partition_table.find_by_type("app", subtype)
if partition is not None:
break
elif args.partition_name is not None:
found_partition = table.find_by_name(args.partition_name)
elif args.type is not None:
found_partition = table.find_by_type(args.type, args.subtype)
else:
raise RuntimeError("invalid partition selection choice")
raise RuntimeError("Invalid partition selection arguments. Specify --partition-name OR \
--partition-type and --partition-subtype OR --partition--boot-default.")
if found_partition is None:
return 1 # nothing found
if partition:
status("Found partition {}".format(str(partition)))
if args.offset:
print('0x%x' % (found_partition.offset))
if args.size:
print('0x%x' % (found_partition.size))
return 0
class InputError(RuntimeError):
def __init__(self, e):
super(InputError, self).__init__(e)
return partition
class ValidationError(InputError):
def __init__(self, partition, message):
super(ValidationError, self).__init__(
"Partition %s invalid: %s" % (partition.name, message))
def _get_and_check_partition(args):
partition = None
partition = _get_partition(args)
if not partition:
raise RuntimeError("Unable to find specified partition.")
return partition
def write_partition(args):
erase_partition(args)
partition = _get_and_check_partition(args)
status("Checking input file size...")
with open(args.input, "rb") as input_file:
content_len = len(input_file.read())
if content_len != partition.size:
status("File size (0x{:x}) does not match partition size (0x{:x})".format(content_len, partition.size))
else:
status("File size matches partition size (0x{:x})".format(partition.size))
_invoke_esptool(["write_flash", str(partition.offset), args.input], args)
status("Written contents of file '{}' to device at offset 0x{:x}".format(args.input, partition.offset))
def read_partition(args):
partition = _get_and_check_partition(args)
_invoke_esptool(["read_flash", str(partition.offset), str(partition.size), args.output], args)
status("Read partition contents from device at offset 0x{:x} to file '{}'".format(partition.offset, args.output))
def erase_partition(args):
partition = _get_and_check_partition(args)
_invoke_esptool(["erase_region", str(partition.offset), str(partition.size)], args)
status("Erased partition at offset 0x{:x} on device".format(partition.offset))
def get_partition_info(args):
partition = None
if args.table:
partition_table = _get_partition_table(args)
if args.table.endswith(".csv"):
partition_table = partition_table.to_csv()
else:
partition_table = partition_table.to_binary()
with open(args.table, "wb") as table_file:
table_file.write(partition_table)
status("Partition table written to " + table_file.name)
else:
partition = _get_partition(args)
if partition:
info_dict = {
"offset" : '0x{:x}'.format(partition.offset),
"size": '0x{:x}'.format(partition.size)
}
infos = []
try:
for info in args.info:
infos += [info_dict[info]]
except KeyError:
raise RuntimeError("Request for unknown partition info {}".format(info))
status("Requested partition information [{}]:".format( ", ".join(args.info)))
print(" ".join(infos))
else:
status("Partition not found")
def generate_blank_partition_file(args):
output = None
stdout_binary = None
partition = _get_and_check_partition(args)
output = b"\xFF" * partition.size
try:
stdout_binary = sys.stdout.buffer # Python 3
except AttributeError:
stdout_binary = sys.stdout
with stdout_binary if args.output == "" else open(args.output, 'wb') as f:
f.write(output)
status("Blank partition file '{}' generated".format(args.output))
def main():
global quiet
parser = argparse.ArgumentParser("ESP-IDF Partitions Tool")
parser.add_argument("--quiet", "-q", help="suppress stderr messages", action="store_true")
# There are two possible sources for the partition table: a device attached to the host
# or a partition table CSV/binary file. These sources are mutually exclusive.
partition_table_info_source_args = parser.add_mutually_exclusive_group()
partition_table_info_source_args.add_argument("--port", "-p", help="port where the device to read the partition table from is attached", default="")
partition_table_info_source_args.add_argument("--partition-table-file", "-f", help="file (CSV/binary) to read the partition table from")
parser.add_argument("--partition-table-offset", "-o", help="offset to read the partition table from", default="0x8000")
# Specify what partition to perform the operation on. This can either be specified using the
# partition name or the first partition that matches the specified type/subtype
partition_selection_args = parser.add_mutually_exclusive_group()
partition_selection_args.add_argument("--partition-name", "-n", help="name of the partition")
partition_selection_args.add_argument("--partition-type", "-t", help="type of the partition")
partition_selection_args.add_argument('--partition-boot-default', "-d", help='select the default boot partition, '+
'using the same fallback logic as the IDF bootloader', action="store_true")
parser.add_argument("--partition-subtype", "-s", help="subtype of the partition")
subparsers = parser.add_subparsers(dest="operation", help="run parttool -h for additional help")
# Specify the supported operations
read_part_subparser = subparsers.add_parser("read_partition", help="read partition from device and dump contents into a file")
read_part_subparser.add_argument("--output", help="file to dump the read partition contents to")
write_part_subparser = subparsers.add_parser("write_partition", help="write contents of a binary file to partition on device")
write_part_subparser.add_argument("--input", help="file whose contents are to be written to the partition offset")
subparsers.add_parser("erase_partition", help="erase the contents of a partition on the device")
print_partition_info_subparser = subparsers.add_parser("get_partition_info", help="get partition information")
print_partition_info_subparser_info_type = print_partition_info_subparser.add_mutually_exclusive_group()
print_partition_info_subparser_info_type.add_argument("--info", help="type of partition information to get", nargs="+")
print_partition_info_subparser_info_type.add_argument("--table", help="dump the partition table to a file")
generate_blank_subparser = subparsers.add_parser("generate_blank_partition_file", help="generate a blank (all 0xFF) partition file of the specified partition that can be flashed to the device")
generate_blank_subparser.add_argument("--output", help="blank partition file filename")
args = parser.parse_args()
quiet = args.quiet
# No operation specified, display help and exit
if args.operation is None:
if not quiet:
parser.print_help()
sys.exit(1)
# Else execute the operation
operation_func = globals()[args.operation]
if quiet:
# If exceptions occur, suppress and exit quietly
try:
operation_func(args)
except Exception:
sys.exit(2)
else:
operation_func(args)
if __name__ == '__main__':
try:
r = main()
sys.exit(r)
except InputError as e:
print(e, file=sys.stderr)
sys.exit(2)
main()

18
components/partition_table/project_include.cmake
View File

@ -28,13 +28,13 @@ endif()
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARTITION_CSV_PATH})
# Parse the partition table to get variable partition offsets & sizes which must be known at CMake runtime
function(get_partition_info variable get_part_info_args)
function(get_partition_info variable get_part_info_args part_info)
separate_arguments(get_part_info_args)
execute_process(COMMAND ${PYTHON}
${COMPONENT_PATH}/parttool.py -q
--partition-table-offset ${PARTITION_TABLE_OFFSET}
${get_part_info_args}
${PARTITION_CSV_PATH}
--partition-table-file ${PARTITION_CSV_PATH}
${get_part_info_args} get_partition_info --info ${part_info}
OUTPUT_VARIABLE result
RESULT_VARIABLE exit_code
OUTPUT_STRIP_TRAILING_WHITESPACE)
@ -46,15 +46,19 @@ function(get_partition_info variable get_part_info_args)
endfunction()
if(CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION)
get_partition_info(PHY_PARTITION_OFFSET "--type data --subtype phy --offset")
get_partition_info(PHY_PARTITION_OFFSET
"--partition-type data --partition-subtype phy" "offset")
set(PHY_PARTITION_BIN_FILE "esp32/phy_init_data.bin")
endif()
get_partition_info(APP_PARTITION_OFFSET "--default-boot-partition --offset")
get_partition_info(APP_PARTITION_OFFSET
"--partition-boot-default" "offset")
get_partition_info(OTADATA_PARTITION_OFFSET "--type data --subtype ota --offset")
get_partition_info(OTADATA_PARTITION_OFFSET
"--partition-type data --partition-subtype ota" "offset")
get_partition_info(OTADATA_PARTITION_SIZE "--type data --subtype ota --size")
get_partition_info(OTADATA_PARTITION_SIZE
"--partition-type data --partition-subtype ota" "size")
endif()

24
components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
View File

@ -398,13 +398,13 @@ app,app, factory, 32K, 1M
class PartToolTests(Py23TestCase):
def _run_parttool(self, csvcontents, args):
def _run_parttool(self, csvcontents, args, info):
csvpath = tempfile.mktemp()
with open(csvpath, "w") as f:
f.write(csvcontents)
try:
output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ") + [ csvpath ],
stderr=subprocess.STDOUT)
output = subprocess.check_output([sys.executable, "../parttool.py"] + args.split(" ")
+ ["--partition-table-file", csvpath , "get_partition_info", "--info", info], stderr=subprocess.STDOUT)
self.assertNotIn(b"WARNING", output)
m = re.search(b"0x[0-9a-fA-F]+", output)
return m.group(0) if m else ""
@ -418,16 +418,16 @@ otadata, data, ota, 0xd000, 0x2000
phy_init, data, phy, 0xf000, 0x1000
factory, app, factory, 0x10000, 1M
"""
rpt = lambda args: self._run_parttool(csv, args)
rpt = lambda args, info: self._run_parttool(csv, args, info)
self.assertEqual(
rpt("--type data --subtype nvs --offset"), b"0x9000")
rpt("--partition-type=data --partition-subtype=nvs -q", "offset"), b"0x9000")
self.assertEqual(
rpt("--type data --subtype nvs --size"), b"0x4000")
rpt("--partition-type=data --partition-subtype=nvs -q", "size"), b"0x4000")
self.assertEqual(
rpt("--partition-name otadata --offset"), b"0xd000")
rpt("--partition-name=otadata -q", "offset"), b"0xd000")
self.assertEqual(
rpt("--default-boot-partition --offset"), b"0x10000")
rpt("--partition-boot-default -q", "offset"), b"0x10000")
def test_fallback(self):
csv = """
@ -437,15 +437,15 @@ phy_init, data, phy, 0xf000, 0x1000
ota_0, app, ota_0, 0x30000, 1M
ota_1, app, ota_1, , 1M
"""
rpt = lambda args: self._run_parttool(csv, args)
rpt = lambda args, info: self._run_parttool(csv, args, info)
self.assertEqual(
rpt("--type app --subtype ota_1 --offset"), b"0x130000")
rpt("--partition-type=app --partition-subtype=ota_1 -q", "offset"), b"0x130000")
self.assertEqual(
rpt("--default-boot-partition --offset"), b"0x30000") # ota_0
rpt("--partition-boot-default -q", "offset"), b"0x30000") # ota_0
csv_mod = csv.replace("ota_0", "ota_2")
self.assertEqual(
self._run_parttool(csv_mod, "--default-boot-partition --offset"),
self._run_parttool(csv_mod, "--partition-boot-default -q", "offset"),
b"0x130000") # now default is ota_1
if __name__ =="__main__":

6
examples/storage/parttool/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
# The following lines of boilerplate have to be in your project's CMakeLists
# in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(parttool)

9
examples/storage/parttool/Makefile Normal file
View File

@ -0,0 +1,9 @@
#
# This is a project Makefile. It is assumed the directory this Makefile resides in is a
# project subdirectory.
#
PROJECT_NAME := parttool
include $(IDF_PATH)/make/project.mk

68
examples/storage/parttool/README.md Normal file
View File

@ -0,0 +1,68 @@
# Partitions Tool Example
This example demonstrates common operations the partitions tool [parttool.py](../../../components/partition_table/parttool.py) allows the user to perform:
- reading, writing and erasing partitions,
- retrieving info on a certain partition,
- dumping the entire partition table, and
- generating a blank partition file.
Users taking a look at this example should focus on the contents of the python script [parttool_example.py](parttool_example.py). The script contains programmatic invocations of [parttool.py](../../../components/partition_table/parttool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications.
The example performs the operations mentioned above in a straightforward manner: it performs writes to partitions and then verifies correct content
by reading it back. For partitions, contents are compared to the originally written file. For the partition table, contents are verified against the partition table CSV
file. An erased partition's contents is compared to a generated blank file.
## How to use example
### Build and Flash
Before running the example script [parttool_example.py](parttool_example.py), it is necessary to build and flash the firmware using the usual means:
```bash
# If using Make
make build flash
# If using CMake
idf.py build flash
```
### Running [parttool_example.py](parttool_example.py)
The example can be executed by running the script [parttool_example.py](parttool_example.py). Either run it directly using
```bash
./parttool_example.py
```
or run it using
```bash
python parttool_example.py
```
The script searches for valid target devices connected to the host and performs the operations on the first one it finds. To perform the operations on a specific device, specify the port it is attached to during script invocation:
```bash
# The target device is attached to /dev/ttyUSB2, for example
python parttool_example.py --port /dev/ttyUSB2
```
## Example output
Running the script produces the following output:
```
Checking if device app binary matches built binary
Checking if device partition table matches partition table csv
Retrieving data partition offset and size
Found data partition at offset 0x110000 with size 0x10000
Writing to data partition
Reading data partition
Erasing data partition
Generating blank data partition file
Reading data partition
Partition tool operations performed successfully!
```

40
examples/storage/parttool/example_test.py Normal file
View File

@ -0,0 +1,40 @@
from __future__ import print_function
import os
import sys
import subprocess
test_fw_path = os.getenv('TEST_FW_PATH')
if test_fw_path and test_fw_path not in sys.path:
sys.path.insert(0, test_fw_path)
import TinyFW
import IDF
@IDF.idf_example_test(env_tag='Example_WIFI')
def test_examples_parttool(env, extra_data):
dut = env.get_dut('parttool', 'examples/storage/parttool')
dut.start_app(False)
# Verify factory firmware
dut.expect("Partitions Tool Example")
dut.expect("Example end")
# Close connection to DUT
dut.receive_thread.exit()
dut.port_inst.close()
# Run the example python script
script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "storage", "parttool", "parttool_example.py")
binary_path = ""
for config in dut.download_config:
if "parttool.bin" in config:
binary_path = config
break
subprocess.check_call([sys.executable, script_path, "--binary", binary_path])
if __name__ == '__main__':
test_examples_parttool()

4
examples/storage/parttool/main/CMakeLists.txt Normal file
View File

@ -0,0 +1,4 @@
set(COMPONENT_SRCS "parttool_main.c")
set(COMPONENT_ADD_INCLUDEDIRS ".")
register_component()

4
examples/storage/parttool/main/component.mk Normal file
View File

@ -0,0 +1,4 @@
#
# "main" pseudo-component makefile.
#
# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

22
examples/storage/parttool/main/parttool_main.c Normal file
View File

@ -0,0 +1,22 @@
/* Partitions Tool Example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include <stdio.h>
#include <string.h>
#include <sys/unistd.h>
#include <sys/stat.h>
#include "esp_err.h"
#include "esp_log.h"
static const char *TAG = "example";
void app_main(void)
{
ESP_LOGI(TAG, "Partitions Tool Example");
ESP_LOGI(TAG, "Example end");
}

6
examples/storage/parttool/partitions_example.csv Normal file
View File

@ -0,0 +1,6 @@
# Name, Type, SubType, Offset, Size, Flags
# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
nvs, data, nvs, 0x9000, 0x6000,
phy_init, data, phy, 0xf000, 0x1000,
factory, app, factory, 0x10000, 1M,
storage, data, spiffs, , 0x10000,
1 # Name, Type, SubType, Offset, Size, Flags
2 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
3 nvs, data, nvs, 0x9000, 0x6000,
4 phy_init, data, phy, 0xf000, 0x1000,
5 factory, app, factory, 0x10000, 1M,
6 storage, data, spiffs, , 0x10000,

207
examples/storage/parttool/parttool_example.py Executable file
View File

@ -0,0 +1,207 @@
#!/usr/bin/env python
#
# Demonstrates the use of parttool.py, a tool for performing partition level
# operations.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import subprocess
import argparse
IDF_PATH = os.path.expandvars("$IDF_PATH")
PARTTOOL_PY = os.path.join(IDF_PATH, "components", "partition_table", "parttool.py")
PARTITION_TABLE_OFFSET = 0x8000
INVOKE_ARGS = [sys.executable, PARTTOOL_PY, "-q", "--partition-table-offset", str(PARTITION_TABLE_OFFSET)]
def sized_file_compare(file1, file2):
with open(file1, "rb") as f1:
with open(file2, "rb") as f2:
f1 = f1.read()
f2 = f2.read()
if len(f1) < len(f2):
f2 = f2[:len(f1)]
else:
f1 = f1[:len(f2)]
return f1 == f2
def check(condition, message):
if not condition:
print("Error: " + message)
sys.exit(1)
def write_data_partition(size):
print("Writing to data partition")
with open("write.bin", "wb") as f:
# Create a file to write to the data partition with randomly generated content
f.write(os.urandom(int(size, 16)))
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage write_partition --input write.bin
#
# to write the contents of a file to a partition in the device.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "write_partition", "--input", f.name]
subprocess.check_call(invoke_args)
return f.name
def read_data_partition():
print("Reading data partition")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage read_partition --output read.bin
#
# to read the contents of a partition in the device, which is then written to a file.
f = "read.bin"
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "read_partition", "--output", f]
subprocess.check_call(invoke_args)
return f
def get_data_partition_info():
print("Retrieving data partition offset and size")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage get_partition_info --info offset size
#
# to get the offset and size of a partition named 'storage'.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "get_partition_info", "--info", "offset", "size"]
(offset, size) = subprocess.check_output(invoke_args).strip().split(b" ")
return (offset, size)
def check_app(args):
print("Checking if device app binary matches built binary")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-type app --partition-subtype factory read_partition --output app.bin"
#
# to read the app binary and write it to a file. The read app binary is compared to the built binary in the build folder.
invoke_args = INVOKE_ARGS + ["--partition-type", "app", "--partition-subtype", "factory", "read_partition", "--output", "app.bin"]
subprocess.check_call(invoke_args)
app_same = sized_file_compare("app.bin", args.binary)
check(app_same, "Device app binary does not match built binary")
def check_partition_table():
sys.path.append(os.path.join(IDF_PATH, "components", "partition_table"))
import gen_esp32part as gen
print("Checking if device partition table matches partition table csv")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 get_partition_info --table table.bin
#
# to read the device partition table and write it to a file. The read partition table is compared to
# the partition table csv.
invoke_args = INVOKE_ARGS + ["get_partition_info", "--table", "table.bin"]
subprocess.check_call(invoke_args)
with open("table.bin", "rb") as read:
partition_table_csv = os.path.join(IDF_PATH, "examples", "storage", "parttool", "partitions_example.csv")
with open(partition_table_csv, "r") as csv:
read = gen.PartitionTable.from_binary(read.read())
csv = gen.PartitionTable.from_csv(csv.read())
check(read == csv, "Device partition table does not match csv partition table")
def erase_data_partition():
print("Erasing data partition")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-name storage erase_partition
#
# to erase the 'storage' partition.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "erase_partition"]
subprocess.check_call(invoke_args)
def generate_blank_data_file():
print("Generating blank data partition file")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-name storage generate_blank_partition_file --output blank.bin
#
# to generate a blank partition file and write it to a file. The blank partition file has the same size as the
# 'storage' partition.
f = "blank.bin"
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "generate_blank_partition_file", "--output", f]
subprocess.check_call(invoke_args)
return f
def main():
global INVOKE_ARGS
parser = argparse.ArgumentParser("ESP-IDF Partitions Tool Example")
parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected")
parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "parttool.bin"))
args = parser.parse_args()
if args.port:
INVOKE_ARGS += ["--port", args.port]
# Before proceeding, do checks to verify whether the app and partition table in the device matches the built binary and
# the generated partition table during build
check_app(args)
check_partition_table()
# Get the offset and size of the data partition
(offset, size) = get_data_partition_info()
print("Found data partition at offset %s with size %s" % (offset, size))
# Write a generated file of random bytes to the found data partition
written = write_data_partition(size)
# Read back the contents of the data partition
read = read_data_partition()
# Compare the written and read back data
data_same = sized_file_compare(read, written)
check(data_same, "Read contents of the data partition does not match written data")
# Erase the data partition
erase_data_partition()
# Read back the erase data partition, which should be all 0xFF's after erasure
read = read_data_partition()
# Generate blank partition file (all 0xFF's)
blank = generate_blank_data_file()
# Verify that the partition has been erased by comparing the contents to the generated blank file
data_same = sized_file_compare(read, blank)
check(data_same, "Erased data partition contents does not match blank partition file")
print("\nPartition tool operations performed successfully!")
if __name__ == '__main__':
main()

5
examples/storage/parttool/sdkconfig.defaults Normal file
View File

@ -0,0 +1,5 @@
CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv"
CONFIG_PARTITION_TABLE_CUSTOM_APP_BIN_OFFSET=0x10000
CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv"
CONFIG_APP_OFFSET=0x10000

4
examples/system/ota/README.md
View File

@ -97,8 +97,8 @@ It allows to run the newly loaded app from a factory partition.
make flash
```
After first update, if you want to return back to factory app (or the first OTA partition, if factory partition is not present) then use the command `make erase_ota`.
It erases ota_data partition to initial.
After first update, if you want to return back to factory app (or the first OTA partition, if factory partition is not present) then use the command `make erase_otadata`.
It erases the ota_data partition to initial state. **Take note that this assumes that the partition table of this project is the one that is on the device**.
### Step 5: Run the OTA Example

6
examples/system/ota/otatool/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
# The following lines of boilerplate have to be in your project's CMakeLists
# in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(otatool)

9
examples/system/ota/otatool/Makefile Normal file
View File

@ -0,0 +1,9 @@
#
# This is a project Makefile. It is assumed the directory this Makefile resides in is a
# project subdirectory.
#
PROJECT_NAME := otatool
include $(IDF_PATH)/make/project.mk

70
examples/system/ota/otatool/README.md Normal file
View File

@ -0,0 +1,70 @@
# OTA Tool Example
This example demonstrates common operations the OTA tool [otatool.py](../../../components/app_update/otatool.py) allows the user to perform:
- reading, writing and erasing OTA partitions,
- switching boot partitions, and
- switching to factory partition.
Users taking a look at this example should focus on the contents of the python script [otatool_example.py](otatool_example.py). The script contains programmatic invocations of the tool [otatool.py](../../../components/app_update/otatool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications.
The built application in this example outputs the currently running partition, whose output is used to verify if the tool switched OTA
partitions succesfully. The built application binary is written to all OTA partitions at the start of the example to be able to determine the running
partition for all switches performed.
## How to use example
### Build and Flash
Before running the example script [otatool_example.py](otatool_example.py), it is necessary to build and flash the firmware using the usual means:
```bash
# If using Make
make build flash
# If using CMake
idf.py build flash
```
### Running [otatool_example.py](otatool_example.py)
The example can be executed by running the script [otatool_example.py](otatool_example.py). Either run it directly using
```bash
./otatool_example.py
```
or run it using
```bash
python otatool_example.py
```
The script searches for valid target devices connected to the host and performs the operations on the first one it finds. This could present problems if there
are multiple viable target devices attached to the host. To perform the operations on a specific device, specify the port it is attached to during script invocation:
```bash
# The target device is attached to /dev/ttyUSB2, for example
python otatool_example.py --port /dev/ttyUSB2
```
## Example output
Running the script produces the following output:
```
Writing factory firmware to ota_0
Writing factory firmware to ota_1
Checking written firmware to ota_0 and ota_1 match factory firmware
Switching to ota partition name factory
Switching to ota partition name factory
Switching to ota partition slot 0
Switching to ota partition name ota_1
Switching to ota partition slot 1
Switching to ota partition name ota_0
Switching to ota partition slot 0
Switching to ota partition name factory
Switching to ota partition slot 1
OTA tool operations executed successfully!
```

43
examples/system/ota/otatool/example_test.py Normal file
View File

@ -0,0 +1,43 @@
from __future__ import print_function
import os
import sys
import subprocess
# this is a test case write with tiny-test-fw.
# to run test cases outside tiny-test-fw,
# we need to set environment variable `TEST_FW_PATH`,
# then get and insert `TEST_FW_PATH` to sys path before import FW module
test_fw_path = os.getenv('TEST_FW_PATH')
if test_fw_path and test_fw_path not in sys.path:
sys.path.insert(0, test_fw_path)
import TinyFW
import IDF
@IDF.idf_example_test(env_tag='Example_WIFI')
def test_otatool_example(env, extra_data):
dut = env.get_dut('otatool', 'examples/system/ota/otatool')
# Verify factory firmware
dut.start_app()
dut.expect("OTA Tool Example")
dut.expect("Example end")
# Close connection to DUT
dut.receive_thread.exit()
dut.port_inst.close()
script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "system", "ota", "otatool", "otatool_example.py")
binary_path = ""
for config in dut.download_config:
if "otatool.bin" in config:
binary_path = config
break
subprocess.check_call([sys.executable, script_path, "--binary", binary_path])
if __name__ == '__main__':
test_otatool_example()

4
examples/system/ota/otatool/main/CMakeLists.txt Normal file
View File

@ -0,0 +1,4 @@
set(COMPONENT_SRCS "otatool_main.c")
set(COMPONENT_ADD_INCLUDEDIRS ".")
register_component()

4
examples/system/ota/otatool/main/component.mk Normal file
View File

@ -0,0 +1,4 @@
#
# "main" pseudo-component makefile.
#
# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

26
examples/system/ota/otatool/main/otatool_main.c Normal file
View File

@ -0,0 +1,26 @@
/* OTA Tool example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include "esp_system.h"
#include "esp_log.h"
#include "esp_ota_ops.h"
#include "esp_partition.h"
static const char *TAG = "example";
void app_main()
{
ESP_LOGI(TAG, "OTA Tool Example");
const esp_partition_t *running = esp_ota_get_running_partition();
// Display the running partition
ESP_LOGI(TAG, "Running partition: %s", running->label);
ESP_LOGI(TAG, "Example end");
}

194
examples/system/ota/otatool/otatool_example.py Executable file
View File

@ -0,0 +1,194 @@
#!/usr/bin/env python
#
# Demonstrates the use of otatool.py, a tool for performing ota partition level
# operations.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import subprocess
import argparse
import serial
import re
IDF_PATH = os.path.expandvars("$IDF_PATH")
OTATOOL_PY = os.path.join(IDF_PATH, "components", "app_update", "otatool.py")
ESPTOOL_PY = os.path.join(IDF_PATH, "components", "esptool_py", "esptool", "esptool.py")
INVOKE_ARGS = [sys.executable, OTATOOL_PY, "-q"]
def sized_file_compare(file1, file2):
with open(file1, "rb") as f1:
with open(file2, "rb") as f2:
f1 = f1.read()
f2 = f2.read()
if len(f1) < len(f2):
f2 = f2[:len(f1)]
else:
f1 = f1[:len(f2)]
return f1 == f2
def check(condition, message):
if not condition:
print("Error: " + message)
sys.exit(1)
def flash_example_firmware_to_ota_partitions(args):
# Invokes the command
#
# otatool.py -q write_ota_partition --slot <part_slot> or
# otatool.py -q write_ota_partition --name <part_name>
#
# to write the contents of a file to the specified ota partition (either using name or the slot number)
print("Writing factory firmware to ota_0")
invoke_args = INVOKE_ARGS + ["write_ota_partition", "--slot", "0", "--input", args.binary]
subprocess.check_call(invoke_args)
print("Writing factory firmware to ota_1")
invoke_args = INVOKE_ARGS + ["write_ota_partition", "--name", "ota_1", "--input", args.binary]
subprocess.check_call(invoke_args)
# Verify that the contents of the two ota slots are the same as that of the factory partition
print("Checking written firmware to ota_0 and ota_1 match factory firmware")
# Invokes the command
#
# otatool.py -q read_ota_partition --slot <part_slot> or
# otatool.py -q read_ota_partition --name <part_name>
#
# to read the contents of a specified ota partition (either using name or the slot number) and write to a file
invoke_args = INVOKE_ARGS + ["read_ota_partition", "--slot", "0", "--output", "app_0.bin"]
subprocess.check_call(invoke_args)
invoke_args = INVOKE_ARGS + ["read_ota_partition", "--name", "ota_1", "--output", "app_1.bin"]
subprocess.check_call(invoke_args)
ota_same = sized_file_compare("app_0.bin", args.binary)
check(ota_same, "Slot 0 app does not match factory app")
ota_same = sized_file_compare("app_1.bin", args.binary)
check(ota_same, "Slot 1 app does not match factory app")
def check_running_ota_partition(expected, port=None):
# Monitor the serial output of target device. The firmware outputs the currently
# running partition. It should match the partition the otatool switched to.
if expected == 0 or expected == "ota_0":
expected = b"ota_0"
elif expected == 1 or expected == "ota_1":
expected = b"ota_1"
else:
expected = b"factory"
sys.path.append(os.path.join(IDF_PATH, 'components', 'esptool_py', 'esptool'))
import esptool
baud = os.environ.get("ESPTOOL_BAUD", esptool.ESPLoader.ESP_ROM_BAUD)
if not port:
# Check what esptool.py finds on what port the device is connected to
output = subprocess.check_output([sys.executable, ESPTOOL_PY, "chip_id"])
pattern = r"Serial port ([\S]+)"
pattern = re.compile(pattern.encode())
port = re.search(pattern, output).group(1)
serial_instance = serial.serial_for_url(port.decode("utf-8"), baud, do_not_open=True)
serial_instance.dtr = False
serial_instance.rts = False
serial_instance.rts = True
serial_instance.open()
serial_instance.rts = False
# Read until example end and find the currently running partition string
content = serial_instance.read_until(b"Example end")
pattern = re.compile(b"Running partition: ([a-z0-9_]+)")
running = re.search(pattern, content).group(1)
check(expected == running, "Running partition %s does not match expected %s" % (running, expected))
def switch_partition(part, port):
if isinstance(part, int):
spec = "slot"
else:
spec = "name"
print("Switching to ota partition %s %s" % (spec, str(part)))
if str(part) == "factory":
# Invokes the command
#
# otatool.py -q erase_otadata
#
# to erase the otadata partition, effectively setting boot firmware to
# factory
subprocess.check_call(INVOKE_ARGS + ["erase_otadata"])
else:
# Invokes the command
#
# otatool.py -q switch_otadata --slot <part_slot> or
# otatool.py -q switch_otadata --name <part_name>
#
# to switch to the indicated ota partition (either using name or the slot number)
subprocess.check_call(INVOKE_ARGS + ["switch_otadata", "--" + spec, str(part)])
check_running_ota_partition(part, port)
def main():
global INVOKE_ARGS
parser = argparse.ArgumentParser("ESP-IDF OTA Tool Example")
parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected")
parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "otatool.bin"))
args = parser.parse_args()
if args.port:
INVOKE_ARGS += ["--port", args.port]
# Flash the factory firmware to all ota partitions
flash_example_firmware_to_ota_partitions(args)
# Perform switching ota partitions
switch_partition("factory", args.port)
switch_partition("factory", args.port) # check switching to factory partition twice in a row
switch_partition(0, args.port)
switch_partition("ota_1", args.port)
switch_partition(1, args.port) # check switching to ota_1 partition twice in a row
switch_partition("ota_0", args.port)
switch_partition(0, args.port) # check switching to ota_0 partition twice in a row
switch_partition("factory", args.port)
switch_partition(1, args.port) # check switching to ota_1 partition from factory
print("\nOTA tool operations executed successfully!")
if __name__ == '__main__':
main()

4
examples/system/ota/otatool/sdkconfig.defaults Normal file
View File

@ -0,0 +1,4 @@
# Default sdkconfig parameters to use the OTA
# partition table layout, with a 4MB flash size
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
CONFIG_PARTITION_TABLE_TWO_OTA=y

3
make/project.mk
View File

@ -34,7 +34,8 @@ help:
@echo "make size-components, size-files - Finer-grained memory footprints"
@echo "make size-symbols - Per symbol memory footprint. Requires COMPONENT=<component>"
@echo "make erase_flash - Erase entire flash contents"
@echo "make erase_ota - Erase ota_data partition. After that will boot first bootable partition (factory or OTAx)."
@echo "make erase_otadata - Erase ota_data partition; First bootable partition (factory or OTAx) will be used on next boot."
@echo " This assumes this project's partition table is the one flashed on the device."
@echo "make monitor - Run idf_monitor tool to monitor serial output from app"
@echo "make simple_monitor - Monitor serial output on terminal console"
@echo "make list-components - List all components in the project"

4
tools/ci/executable-list.txt
View File

@ -8,7 +8,7 @@ components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
components/partition_table/gen_esp32part.py
components/partition_table/parttool.py
components/app_update/gen_empty_partition.py
components/app_update/dump_otadata.py
components/app_update/otatool.py
components/partition_table/test_gen_esp32part_host/gen_esp32part_tests.py
components/ulp/esp32ulp_mapgen.py
docs/check_doc_warnings.sh
@ -68,3 +68,5 @@ tools/ldgen/ldgen.py
tools/ldgen/test/test_fragments.py
tools/ldgen/test/test_generation.py
examples/build_system/cmake/idf_as_lib/build.sh
examples/storage/parttool/parttool_example.py
examples/system/ota/otatool/otatool_example.py

42
tools/idf.py
View File

@ -386,27 +386,29 @@ def print_closing_message(args):
ACTIONS = {
# action name : ( function (or alias), dependencies, order-only dependencies )
"all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ),
"build": ( "all", [], [] ), # build is same as 'all' target
"clean": ( clean, [], [ "fullclean" ] ),
"fullclean": ( fullclean, [], [] ),
"reconfigure": ( reconfigure, [], [ "menuconfig" ] ),
"menuconfig": ( build_target, [], [] ),
"all" : ( build_target, [], [ "reconfigure", "menuconfig", "clean", "fullclean" ] ),
"build": ( "all", [], [] ), # build is same as 'all' target
"clean": ( clean, [], [ "fullclean" ] ),
"fullclean": ( fullclean, [], [] ),
"reconfigure": ( reconfigure, [], [ "menuconfig" ] ),
"menuconfig": ( build_target, [], [] ),
"defconfig": ( build_target, [], [] ),
"confserver": ( build_target, [], [] ),
"size": ( build_target, [ "app" ], [] ),
"size-components": ( build_target, [ "app" ], [] ),
"size-files": ( build_target, [ "app" ], [] ),
"bootloader": ( build_target, [], [] ),
"bootloader-clean": ( build_target, [], [] ),
"bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ),
"app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ),
"app-flash": ( flash, [ "app" ], [ "erase_flash"]),
"partition_table": ( build_target, [], [ "reconfigure" ] ),
"partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]),
"flash": ( flash, [ "all" ], [ "erase_flash" ] ),
"erase_flash": ( erase_flash, [], []),
"monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]),
"confserver": ( build_target, [], [] ),
"size": ( build_target, [ "app" ], [] ),
"size-components": ( build_target, [ "app" ], [] ),
"size-files": ( build_target, [ "app" ], [] ),
"bootloader": ( build_target, [], [] ),
"bootloader-clean": ( build_target, [], [] ),
"bootloader-flash": ( flash, [ "bootloader" ], [ "erase_flash"] ),
"app": ( build_target, [], [ "clean", "fullclean", "reconfigure" ] ),
"app-flash": ( flash, [ "app" ], [ "erase_flash"]),
"partition_table": ( build_target, [], [ "reconfigure" ] ),
"partition_table-flash": ( flash, [ "partition_table" ], [ "erase_flash" ]),
"flash": ( flash, [ "all" ], [ "erase_flash" ] ),
"erase_flash": ( erase_flash, [], []),
"monitor": ( monitor, [], [ "flash", "partition_table-flash", "bootloader-flash", "app-flash" ]),
"erase_otadata": ( build_target, [], []),
"read_otadata": ( build_target, [], []),
}
def get_commandline_options():