examples: add otatool and parttool examples

This commit is contained in:
Renz Christian Bagaporo 2018-11-16 05:02:34 +08:00
parent 5e08698039
commit b926764385
21 changed files with 735 additions and 0 deletions

View file

@ -88,11 +88,13 @@ exclude =
examples/provisioning/custom_config/components/custom_provisioning/python/custom_config_pb2.py,
examples/provisioning/softap_prov/softap_prov_test.py,
examples/provisioning/softap_prov/utils/wifi_tools.py,
examples/storage/parttool/example_test.py,
examples/system/cpp_exceptions/example_test.py,
examples/system/esp_event/default_event_loop/example_test.py,
examples/system/esp_event/user_event_loops/example_test.py,
examples/system/esp_timer/example_test.py,
examples/system/light_sleep/example_test.py,
examples/system/ota/otatool/example_test.py,
examples/wifi/iperf/iperf_test.py,
examples/wifi/iperf/test_report.py,
tools/check_python_dependencies.py,

View file

@ -0,0 +1,6 @@
# The following lines of boilerplate have to be in your project's CMakeLists
# in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(parttool)

View file

@ -0,0 +1,9 @@
#
# This is a project Makefile. It is assumed the directory this Makefile resides in is a
# project subdirectory.
#
PROJECT_NAME := parttool
include $(IDF_PATH)/make/project.mk

View file

@ -0,0 +1,68 @@
# Partitions Tool Example
This example demonstrates common operations the partitions tool [parttool.py](../../../components/partition_table/parttool.py) allows the user to perform:
- reading, writing and erasing partitions,
- retrieving info on a certain partition,
- dumping the entire partition table, and
- generating a blank partition file.
Users taking a look at this example should focus on the contents of the python script [parttool_example.py](parttool_example.py). The script contains programmatic invocations of [parttool.py](../../../components/partition_table/parttool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications.
The example performs the operations mentioned above in a straightforward manner: it performs writes to partitions and then verifies correct content
by reading it back. For partitions, contents are compared to the originally written file. For the partition table, contents are verified against the partition table CSV
file. An erased partition's contents is compared to a generated blank file.
## How to use example
### Build and Flash
Before running the example script [parttool_example.py](parttool_example.py), it is necessary to build and flash the firmware using the usual means:
```bash
# If using Make
make build flash
# If using CMake
idf.py build flash
```
### Running [parttool_example.py](parttool_example.py)
The example can be executed by running the script [parttool_example.py](parttool_example.py). Either run it directly using
```bash
./parttool_example.py
```
or run it using
```bash
python parttool_example.py
```
The script searches for valid target devices connected to the host and performs the operations on the first one it finds. To perform the operations on a specific device, specify the port it is attached to during script invocation:
```bash
# The target device is attached to /dev/ttyUSB2, for example
python parttool_example.py --port /dev/ttyUSB2
```
## Example output
Running the script produces the following output:
```
Checking if device app binary matches built binary
Checking if device partition table matches partition table csv
Retrieving data partition offset and size
Found data partition at offset 0x110000 with size 0x10000
Writing to data partition
Reading data partition
Erasing data partition
Generating blank data partition file
Reading data partition
Partition tool operations performed successfully!
```

View file

@ -0,0 +1,40 @@
from __future__ import print_function
import os
import sys
import subprocess
test_fw_path = os.getenv('TEST_FW_PATH')
if test_fw_path and test_fw_path not in sys.path:
sys.path.insert(0, test_fw_path)
import TinyFW
import IDF
@IDF.idf_example_test(env_tag='Example_WIFI')
def test_examples_parttool(env, extra_data):
dut = env.get_dut('parttool', 'examples/storage/parttool')
dut.start_app(False)
# Verify factory firmware
dut.expect("Partitions Tool Example")
dut.expect("Example end")
# Close connection to DUT
dut.receive_thread.exit()
dut.port_inst.close()
# Run the example python script
script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "storage", "parttool", "parttool_example.py")
binary_path = ""
for config in dut.download_config:
if "parttool.bin" in config:
binary_path = config
break
subprocess.check_call([sys.executable, script_path, "--binary", binary_path])
if __name__ == '__main__':
test_examples_parttool()

View file

@ -0,0 +1,4 @@
set(COMPONENT_SRCS "parttool_main.c")
set(COMPONENT_ADD_INCLUDEDIRS ".")
register_component()

View file

@ -0,0 +1,4 @@
#
# "main" pseudo-component makefile.
#
# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

View file

@ -0,0 +1,22 @@
/* Partitions Tool Example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include <stdio.h>
#include <string.h>
#include <sys/unistd.h>
#include <sys/stat.h>
#include "esp_err.h"
#include "esp_log.h"
static const char *TAG = "example";
void app_main(void)
{
ESP_LOGI(TAG, "Partitions Tool Example");
ESP_LOGI(TAG, "Example end");
}

View file

@ -0,0 +1,6 @@
# Name, Type, SubType, Offset, Size, Flags
# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
nvs, data, nvs, 0x9000, 0x6000,
phy_init, data, phy, 0xf000, 0x1000,
factory, app, factory, 0x10000, 1M,
storage, data, spiffs, , 0x10000,
1 # Name, Type, SubType, Offset, Size, Flags
2 # Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild
3 nvs, data, nvs, 0x9000, 0x6000,
4 phy_init, data, phy, 0xf000, 0x1000,
5 factory, app, factory, 0x10000, 1M,
6 storage, data, spiffs, , 0x10000,

View file

@ -0,0 +1,207 @@
#!/usr/bin/env python
#
# Demonstrates the use of parttool.py, a tool for performing partition level
# operations.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import subprocess
import argparse
IDF_PATH = os.path.expandvars("$IDF_PATH")
PARTTOOL_PY = os.path.join(IDF_PATH, "components", "partition_table", "parttool.py")
PARTITION_TABLE_OFFSET = 0x8000
INVOKE_ARGS = [sys.executable, PARTTOOL_PY, "-q", "--partition-table-offset", str(PARTITION_TABLE_OFFSET)]
def sized_file_compare(file1, file2):
with open(file1, "rb") as f1:
with open(file2, "rb") as f2:
f1 = f1.read()
f2 = f2.read()
if len(f1) < len(f2):
f2 = f2[:len(f1)]
else:
f1 = f1[:len(f2)]
return f1 == f2
def check(condition, message):
if not condition:
print("Error: " + message)
sys.exit(1)
def write_data_partition(size):
print("Writing to data partition")
with open("write.bin", "wb") as f:
# Create a file to write to the data partition with randomly generated content
f.write(os.urandom(int(size, 16)))
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage write_partition --input write.bin
#
# to write the contents of a file to a partition in the device.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "write_partition", "--input", f.name]
subprocess.check_call(invoke_args)
return f.name
def read_data_partition():
print("Reading data partition")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage read_partition --output read.bin
#
# to read the contents of a partition in the device, which is then written to a file.
f = "read.bin"
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "read_partition", "--output", f]
subprocess.check_call(invoke_args)
return f
def get_data_partition_info():
print("Retrieving data partition offset and size")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 -q --partition-name storage get_partition_info --info offset size
#
# to get the offset and size of a partition named 'storage'.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "get_partition_info", "--info", "offset", "size"]
(offset, size) = subprocess.check_output(invoke_args).strip().split(b" ")
return (offset, size)
def check_app(args):
print("Checking if device app binary matches built binary")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-type app --partition-subtype factory read_partition --output app.bin"
#
# to read the app binary and write it to a file. The read app binary is compared to the built binary in the build folder.
invoke_args = INVOKE_ARGS + ["--partition-type", "app", "--partition-subtype", "factory", "read_partition", "--output", "app.bin"]
subprocess.check_call(invoke_args)
app_same = sized_file_compare("app.bin", args.binary)
check(app_same, "Device app binary does not match built binary")
def check_partition_table():
sys.path.append(os.path.join(IDF_PATH, "components", "partition_table"))
import gen_esp32part as gen
print("Checking if device partition table matches partition table csv")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 get_partition_info --table table.bin
#
# to read the device partition table and write it to a file. The read partition table is compared to
# the partition table csv.
invoke_args = INVOKE_ARGS + ["get_partition_info", "--table", "table.bin"]
subprocess.check_call(invoke_args)
with open("table.bin", "rb") as read:
partition_table_csv = os.path.join(IDF_PATH, "examples", "storage", "parttool", "partitions_example.csv")
with open(partition_table_csv, "r") as csv:
read = gen.PartitionTable.from_binary(read.read())
csv = gen.PartitionTable.from_csv(csv.read())
check(read == csv, "Device partition table does not match csv partition table")
def erase_data_partition():
print("Erasing data partition")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-name storage erase_partition
#
# to erase the 'storage' partition.
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "erase_partition"]
subprocess.check_call(invoke_args)
def generate_blank_data_file():
print("Generating blank data partition file")
# Invokes the command
#
# parttool.py --partition-table-offset 0x8000 --partition-name storage generate_blank_partition_file --output blank.bin
#
# to generate a blank partition file and write it to a file. The blank partition file has the same size as the
# 'storage' partition.
f = "blank.bin"
invoke_args = INVOKE_ARGS + ["--partition-name", "storage", "generate_blank_partition_file", "--output", f]
subprocess.check_call(invoke_args)
return f
def main():
global INVOKE_ARGS
parser = argparse.ArgumentParser("ESP-IDF Partitions Tool Example")
parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected")
parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "parttool.bin"))
args = parser.parse_args()
if args.port:
INVOKE_ARGS += ["--port", args.port]
# Before proceeding, do checks to verify whether the app and partition table in the device matches the built binary and
# the generated partition table during build
check_app(args)
check_partition_table()
# Get the offset and size of the data partition
(offset, size) = get_data_partition_info()
print("Found data partition at offset %s with size %s" % (offset, size))
# Write a generated file of random bytes to the found data partition
written = write_data_partition(size)
# Read back the contents of the data partition
read = read_data_partition()
# Compare the written and read back data
data_same = sized_file_compare(read, written)
check(data_same, "Read contents of the data partition does not match written data")
# Erase the data partition
erase_data_partition()
# Read back the erase data partition, which should be all 0xFF's after erasure
read = read_data_partition()
# Generate blank partition file (all 0xFF's)
blank = generate_blank_data_file()
# Verify that the partition has been erased by comparing the contents to the generated blank file
data_same = sized_file_compare(read, blank)
check(data_same, "Erased data partition contents does not match blank partition file")
print("\nPartition tool operations performed successfully!")
if __name__ == '__main__':
main()

View file

@ -0,0 +1,5 @@
CONFIG_PARTITION_TABLE_CUSTOM=y
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_example.csv"
CONFIG_PARTITION_TABLE_CUSTOM_APP_BIN_OFFSET=0x10000
CONFIG_PARTITION_TABLE_FILENAME="partitions_example.csv"
CONFIG_APP_OFFSET=0x10000

View file

@ -0,0 +1,6 @@
# The following lines of boilerplate have to be in your project's CMakeLists
# in this exact order for cmake to work correctly
cmake_minimum_required(VERSION 3.5)
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
project(otatool)

View file

@ -0,0 +1,9 @@
#
# This is a project Makefile. It is assumed the directory this Makefile resides in is a
# project subdirectory.
#
PROJECT_NAME := otatool
include $(IDF_PATH)/make/project.mk

View file

@ -0,0 +1,70 @@
# OTA Tool Example
This example demonstrates common operations the OTA tool [otatool.py](../../../components/app_update/otatool.py) allows the user to perform:
- reading, writing and erasing OTA partitions,
- switching boot partitions, and
- switching to factory partition.
Users taking a look at this example should focus on the contents of the python script [otatool_example.py](otatool_example.py). The script contains programmatic invocations of the tool [otatool.py](../../../components/app_update/otatool.py) in Python for the operations mentioned above; and can serve as a guide for users wanting to do the same in their applications.
The built application in this example outputs the currently running partition, whose output is used to verify if the tool switched OTA
partitions succesfully. The built application binary is written to all OTA partitions at the start of the example to be able to determine the running
partition for all switches performed.
## How to use example
### Build and Flash
Before running the example script [otatool_example.py](otatool_example.py), it is necessary to build and flash the firmware using the usual means:
```bash
# If using Make
make build flash
# If using CMake
idf.py build flash
```
### Running [otatool_example.py](otatool_example.py)
The example can be executed by running the script [otatool_example.py](otatool_example.py). Either run it directly using
```bash
./otatool_example.py
```
or run it using
```bash
python otatool_example.py
```
The script searches for valid target devices connected to the host and performs the operations on the first one it finds. This could present problems if there
are multiple viable target devices attached to the host. To perform the operations on a specific device, specify the port it is attached to during script invocation:
```bash
# The target device is attached to /dev/ttyUSB2, for example
python otatool_example.py --port /dev/ttyUSB2
```
## Example output
Running the script produces the following output:
```
Writing factory firmware to ota_0
Writing factory firmware to ota_1
Checking written firmware to ota_0 and ota_1 match factory firmware
Switching to ota partition name factory
Switching to ota partition name factory
Switching to ota partition slot 0
Switching to ota partition name ota_1
Switching to ota partition slot 1
Switching to ota partition name ota_0
Switching to ota partition slot 0
Switching to ota partition name factory
Switching to ota partition slot 1
OTA tool operations executed successfully!
```

View file

@ -0,0 +1,43 @@
from __future__ import print_function
import os
import sys
import subprocess
# this is a test case write with tiny-test-fw.
# to run test cases outside tiny-test-fw,
# we need to set environment variable `TEST_FW_PATH`,
# then get and insert `TEST_FW_PATH` to sys path before import FW module
test_fw_path = os.getenv('TEST_FW_PATH')
if test_fw_path and test_fw_path not in sys.path:
sys.path.insert(0, test_fw_path)
import TinyFW
import IDF
@IDF.idf_example_test(env_tag='Example_WIFI')
def test_otatool_example(env, extra_data):
dut = env.get_dut('otatool', 'examples/system/ota/otatool')
# Verify factory firmware
dut.start_app()
dut.expect("OTA Tool Example")
dut.expect("Example end")
# Close connection to DUT
dut.receive_thread.exit()
dut.port_inst.close()
script_path = os.path.join(os.getenv("IDF_PATH"), "examples", "system", "ota", "otatool", "otatool_example.py")
binary_path = ""
for config in dut.download_config:
if "otatool.bin" in config:
binary_path = config
break
subprocess.check_call([sys.executable, script_path, "--binary", binary_path])
if __name__ == '__main__':
test_otatool_example()

View file

@ -0,0 +1,4 @@
set(COMPONENT_SRCS "otatool_main.c")
set(COMPONENT_ADD_INCLUDEDIRS ".")
register_component()

View file

@ -0,0 +1,4 @@
#
# "main" pseudo-component makefile.
#
# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)

View file

@ -0,0 +1,26 @@
/* OTA Tool example
This example code is in the Public Domain (or CC0 licensed, at your option.)
Unless required by applicable law or agreed to in writing, this
software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied.
*/
#include "esp_system.h"
#include "esp_log.h"
#include "esp_ota_ops.h"
#include "esp_partition.h"
static const char *TAG = "example";
void app_main()
{
ESP_LOGI(TAG, "OTA Tool Example");
const esp_partition_t *running = esp_ota_get_running_partition();
// Display the running partition
ESP_LOGI(TAG, "Running partition: %s", running->label);
ESP_LOGI(TAG, "Example end");
}

View file

@ -0,0 +1,194 @@
#!/usr/bin/env python
#
# Demonstrates the use of otatool.py, a tool for performing ota partition level
# operations.
#
# Copyright 2018 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
import subprocess
import argparse
import serial
import re
IDF_PATH = os.path.expandvars("$IDF_PATH")
OTATOOL_PY = os.path.join(IDF_PATH, "components", "app_update", "otatool.py")
ESPTOOL_PY = os.path.join(IDF_PATH, "components", "esptool_py", "esptool", "esptool.py")
INVOKE_ARGS = [sys.executable, OTATOOL_PY, "-q"]
def sized_file_compare(file1, file2):
with open(file1, "rb") as f1:
with open(file2, "rb") as f2:
f1 = f1.read()
f2 = f2.read()
if len(f1) < len(f2):
f2 = f2[:len(f1)]
else:
f1 = f1[:len(f2)]
return f1 == f2
def check(condition, message):
if not condition:
print("Error: " + message)
sys.exit(1)
def flash_example_firmware_to_ota_partitions(args):
# Invokes the command
#
# otatool.py -q write_ota_partition --slot <part_slot> or
# otatool.py -q write_ota_partition --name <part_name>
#
# to write the contents of a file to the specified ota partition (either using name or the slot number)
print("Writing factory firmware to ota_0")
invoke_args = INVOKE_ARGS + ["write_ota_partition", "--slot", "0", "--input", args.binary]
subprocess.check_call(invoke_args)
print("Writing factory firmware to ota_1")
invoke_args = INVOKE_ARGS + ["write_ota_partition", "--name", "ota_1", "--input", args.binary]
subprocess.check_call(invoke_args)
# Verify that the contents of the two ota slots are the same as that of the factory partition
print("Checking written firmware to ota_0 and ota_1 match factory firmware")
# Invokes the command
#
# otatool.py -q read_ota_partition --slot <part_slot> or
# otatool.py -q read_ota_partition --name <part_name>
#
# to read the contents of a specified ota partition (either using name or the slot number) and write to a file
invoke_args = INVOKE_ARGS + ["read_ota_partition", "--slot", "0", "--output", "app_0.bin"]
subprocess.check_call(invoke_args)
invoke_args = INVOKE_ARGS + ["read_ota_partition", "--name", "ota_1", "--output", "app_1.bin"]
subprocess.check_call(invoke_args)
ota_same = sized_file_compare("app_0.bin", args.binary)
check(ota_same, "Slot 0 app does not match factory app")
ota_same = sized_file_compare("app_1.bin", args.binary)
check(ota_same, "Slot 1 app does not match factory app")
def check_running_ota_partition(expected, port=None):
# Monitor the serial output of target device. The firmware outputs the currently
# running partition. It should match the partition the otatool switched to.
if expected == 0 or expected == "ota_0":
expected = b"ota_0"
elif expected == 1 or expected == "ota_1":
expected = b"ota_1"
else:
expected = b"factory"
sys.path.append(os.path.join(IDF_PATH, 'components', 'esptool_py', 'esptool'))
import esptool
baud = os.environ.get("ESPTOOL_BAUD", esptool.ESPLoader.ESP_ROM_BAUD)
if not port:
# Check what esptool.py finds on what port the device is connected to
output = subprocess.check_output([sys.executable, ESPTOOL_PY, "chip_id"])
pattern = r"Serial port ([\S]+)"
pattern = re.compile(pattern.encode())
port = re.search(pattern, output).group(1)
serial_instance = serial.serial_for_url(port.decode("utf-8"), baud, do_not_open=True)
serial_instance.dtr = False
serial_instance.rts = False
serial_instance.rts = True
serial_instance.open()
serial_instance.rts = False
# Read until example end and find the currently running partition string
content = serial_instance.read_until(b"Example end")
pattern = re.compile(b"Running partition: ([a-z0-9_]+)")
running = re.search(pattern, content).group(1)
check(expected == running, "Running partition %s does not match expected %s" % (running, expected))
def switch_partition(part, port):
if isinstance(part, int):
spec = "slot"
else:
spec = "name"
print("Switching to ota partition %s %s" % (spec, str(part)))
if str(part) == "factory":
# Invokes the command
#
# otatool.py -q erase_otadata
#
# to erase the otadata partition, effectively setting boot firmware to
# factory
subprocess.check_call(INVOKE_ARGS + ["erase_otadata"])
else:
# Invokes the command
#
# otatool.py -q switch_otadata --slot <part_slot> or
# otatool.py -q switch_otadata --name <part_name>
#
# to switch to the indicated ota partition (either using name or the slot number)
subprocess.check_call(INVOKE_ARGS + ["switch_otadata", "--" + spec, str(part)])
check_running_ota_partition(part, port)
def main():
global INVOKE_ARGS
parser = argparse.ArgumentParser("ESP-IDF OTA Tool Example")
parser.add_argument("--port", "-p", help="port where the device to perform operations on is connected")
parser.add_argument("--binary", "-b", help="path to built example binary", default=os.path.join("build", "otatool.bin"))
args = parser.parse_args()
if args.port:
INVOKE_ARGS += ["--port", args.port]
# Flash the factory firmware to all ota partitions
flash_example_firmware_to_ota_partitions(args)
# Perform switching ota partitions
switch_partition("factory", args.port)
switch_partition("factory", args.port) # check switching to factory partition twice in a row
switch_partition(0, args.port)
switch_partition("ota_1", args.port)
switch_partition(1, args.port) # check switching to ota_1 partition twice in a row
switch_partition("ota_0", args.port)
switch_partition(0, args.port) # check switching to ota_0 partition twice in a row
switch_partition("factory", args.port)
switch_partition(1, args.port) # check switching to ota_1 partition from factory
print("\nOTA tool operations executed successfully!")
if __name__ == '__main__':
main()

View file

@ -0,0 +1,4 @@
# Default sdkconfig parameters to use the OTA
# partition table layout, with a 4MB flash size
CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
CONFIG_PARTITION_TABLE_TWO_OTA=y

View file

@ -68,3 +68,5 @@ tools/ldgen/ldgen.py
tools/ldgen/test/test_fragments.py
tools/ldgen/test/test_generation.py
examples/build_system/cmake/idf_as_lib/build.sh
examples/storage/parttool/parttool_example.py
examples/system/ota/otatool/otatool_example.py