20156f9702
Changes argument parsing mechanism from argparse to a new one, that provides better support for extensions and options that are only applicable to specific subcommands, Breaking changes: 1. All global options should go before subcommands, i.e. `idf.py build -C ~/some/project` will not work anymore, only `idf.py -C ~/some/project build` is acceptable 2. To provide multiple values to an option like `--define-cache-entry` it's necessary to repeat option many times, i.e. `idf.py -D entry1 entry2 entry3` will not work, right way is: `idf.py -D entry1 -D entry2 -D entry3` At the moment there are 3 options like this: `--define-cache-entry` in base list and `--test-components` and `--test-exclude-components` in the unit test extensions 3. Drops `defconfig` and `bootloader-clean` subcommands Closes https://github.com/espressif/esp-idf/issues/3570 Closes https://github.com/espressif/esp-idf/issues/3571
298 lines
11 KiB
Python
298 lines
11 KiB
Python
import copy
|
|
import glob
|
|
import os
|
|
import os.path
|
|
import re
|
|
import shutil
|
|
import tempfile
|
|
|
|
|
|
def action_extensions(base_actions, project_path=os.getcwd()):
|
|
""" Describes extensions for unit tests. This function expects that actions "all" and "reconfigure" """
|
|
|
|
PROJECT_NAME = "unit-test-app"
|
|
|
|
# List of unit-test-app configurations.
|
|
# Each file in configs/ directory defines a configuration. The format is the
|
|
# same as sdkconfig file. Configuration is applied on top of sdkconfig.defaults
|
|
# file from the project directory
|
|
CONFIG_NAMES = os.listdir(os.path.join(project_path, "configs"))
|
|
|
|
# Build (intermediate) and output (artifact) directories
|
|
BUILDS_DIR = os.path.join(project_path, "builds")
|
|
BINARIES_DIR = os.path.join(project_path, "output")
|
|
|
|
def ut_apply_config(ut_apply_config_name, ctx, args):
|
|
config_name = re.match(r"ut-apply-config-(.*)", ut_apply_config_name).group(1)
|
|
|
|
def set_config_build_variables(prop, defval=None):
|
|
property_value = re.findall(
|
|
r"^%s=(.+)" % prop, config_file_content, re.MULTILINE
|
|
)
|
|
if property_value:
|
|
property_value = property_value[0]
|
|
else:
|
|
property_value = defval
|
|
|
|
if property_value:
|
|
try:
|
|
args.define_cache_entry.append("%s=" % prop + property_value)
|
|
except AttributeError:
|
|
args.define_cache_entry = ["%s=" % prop + property_value]
|
|
|
|
return property_value
|
|
|
|
sdkconfig_set = None
|
|
|
|
if args.define_cache_entry:
|
|
sdkconfig_set = filter(lambda s: "SDKCONFIG=" in s, args.define_cache_entry)
|
|
|
|
sdkconfig_path = os.path.join(args.project_dir, "sdkconfig")
|
|
|
|
if sdkconfig_set:
|
|
sdkconfig_path = sdkconfig_set[-1].split("=")[1]
|
|
sdkconfig_path = os.path.abspath(sdkconfig_path)
|
|
|
|
try:
|
|
os.remove(sdkconfig_path)
|
|
except OSError:
|
|
pass
|
|
|
|
if config_name in CONFIG_NAMES:
|
|
# Parse the sdkconfig for components to be included/excluded and tests to be run
|
|
config = os.path.join(project_path, "configs", config_name)
|
|
|
|
with open(config, "r") as config_file:
|
|
config_file_content = config_file.read()
|
|
|
|
set_config_build_variables("EXCLUDE_COMPONENTS", "''")
|
|
|
|
test_components = set_config_build_variables("TEST_COMPONENTS", "''")
|
|
|
|
tests_all = None
|
|
if test_components == "''":
|
|
tests_all = "TESTS_ALL=1"
|
|
else:
|
|
tests_all = "TESTS_ALL=0"
|
|
|
|
try:
|
|
args.define_cache_entry.append(tests_all)
|
|
except AttributeError:
|
|
args.define_cache_entry = [tests_all]
|
|
|
|
set_config_build_variables("TEST_EXCLUDE_COMPONENTS", "''")
|
|
|
|
with tempfile.NamedTemporaryFile() as sdkconfig_temp:
|
|
# Use values from the combined defaults and the values from
|
|
# config folder to build config
|
|
sdkconfig_default = os.path.join(project_path, "sdkconfig.defaults")
|
|
|
|
with open(sdkconfig_default, "rb") as sdkconfig_default_file:
|
|
sdkconfig_temp.write(sdkconfig_default_file.read())
|
|
|
|
sdkconfig_config = os.path.join(project_path, "configs", config_name)
|
|
with open(sdkconfig_config, "rb") as sdkconfig_config_file:
|
|
sdkconfig_temp.write(b"\n")
|
|
sdkconfig_temp.write(sdkconfig_config_file.read())
|
|
|
|
sdkconfig_temp.flush()
|
|
|
|
try:
|
|
args.define_cache_entry.append(
|
|
"SDKCONFIG_DEFAULTS=" + sdkconfig_temp.name
|
|
)
|
|
except AttributeError:
|
|
args.define_cache_entry = [
|
|
"SDKCONFIG_DEFAULTS=" + sdkconfig_temp.name
|
|
]
|
|
|
|
reconfigure = base_actions["actions"]["reconfigure"]["callback"]
|
|
reconfigure(None, ctx, args)
|
|
else:
|
|
if not config_name == "all-configs":
|
|
print(
|
|
"unknown unit test app config for action '%s'"
|
|
% ut_apply_config_name
|
|
)
|
|
|
|
# This target builds the configuration. It does not currently track dependencies,
|
|
# but is good enough for CI builds if used together with clean-all-configs.
|
|
# For local builds, use 'apply-config-NAME' target and then use normal 'all'
|
|
# and 'flash' targets.
|
|
def ut_build(ut_build_name, ctx, args):
|
|
# Create a copy of the passed arguments to prevent arg modifications to accrue if
|
|
# all configs are being built
|
|
build_args = copy.copy(args)
|
|
|
|
config_name = re.match(r"ut-build-(.*)", ut_build_name).group(1)
|
|
|
|
if config_name in CONFIG_NAMES:
|
|
build_args.build_dir = os.path.join(BUILDS_DIR, config_name)
|
|
|
|
src = os.path.join(BUILDS_DIR, config_name)
|
|
dest = os.path.join(BINARIES_DIR, config_name)
|
|
|
|
try:
|
|
os.makedirs(dest)
|
|
except OSError:
|
|
pass
|
|
|
|
# Build, tweaking paths to sdkconfig and sdkconfig.defaults
|
|
ut_apply_config("ut-apply-config-" + config_name, ctx, build_args)
|
|
|
|
build_target = base_actions["actions"]["all"]["callback"]
|
|
|
|
build_target("all", ctx, build_args)
|
|
|
|
# Copy artifacts to the output directory
|
|
shutil.copyfile(
|
|
os.path.join(build_args.project_dir, "sdkconfig"),
|
|
os.path.join(dest, "sdkconfig"),
|
|
)
|
|
|
|
binaries = [PROJECT_NAME + x for x in [".elf", ".bin", ".map"]]
|
|
|
|
for binary in binaries:
|
|
shutil.copyfile(os.path.join(src, binary), os.path.join(dest, binary))
|
|
|
|
try:
|
|
os.mkdir(os.path.join(dest, "bootloader"))
|
|
except OSError:
|
|
pass
|
|
|
|
shutil.copyfile(
|
|
os.path.join(src, "bootloader", "bootloader.bin"),
|
|
os.path.join(dest, "bootloader", "bootloader.bin"),
|
|
)
|
|
|
|
for partition_table in glob.glob(
|
|
os.path.join(src, "partition_table", "partition-table*.bin")
|
|
):
|
|
try:
|
|
os.mkdir(os.path.join(dest, "partition_table"))
|
|
except OSError:
|
|
pass
|
|
shutil.copyfile(
|
|
partition_table,
|
|
os.path.join(
|
|
dest, "partition_table", os.path.basename(partition_table)
|
|
),
|
|
)
|
|
|
|
shutil.copyfile(
|
|
os.path.join(src, "flasher_args.json"),
|
|
os.path.join(dest, "flasher_args.json"),
|
|
)
|
|
|
|
binaries = glob.glob(os.path.join(src, "*.bin"))
|
|
binaries = [os.path.basename(s) for s in binaries]
|
|
|
|
for binary in binaries:
|
|
shutil.copyfile(os.path.join(src, binary), os.path.join(dest, binary))
|
|
|
|
else:
|
|
if not config_name == "all-configs":
|
|
print("unknown unit test app config for action '%s'" % ut_build_name)
|
|
|
|
def ut_clean(ut_clean_name, ctx, args):
|
|
config_name = re.match(r"ut-clean-(.*)", ut_clean_name).group(1)
|
|
if config_name in CONFIG_NAMES:
|
|
shutil.rmtree(os.path.join(BUILDS_DIR, config_name), ignore_errors=True)
|
|
shutil.rmtree(os.path.join(BINARIES_DIR, config_name), ignore_errors=True)
|
|
else:
|
|
if not config_name == "all-configs":
|
|
print("unknown unit test app config for action '%s'" % ut_clean_name)
|
|
|
|
def test_component_callback(ctx, global_args, tasks):
|
|
""" Convert the values passed to the -T and -E parameter to corresponding cache entry definitions TESTS_ALL and TEST_COMPONENTS """
|
|
test_components = global_args.test_components
|
|
test_exclude_components = global_args.test_exclude_components
|
|
|
|
cache_entries = []
|
|
|
|
if test_components:
|
|
if "all" in test_components:
|
|
cache_entries.append("TESTS_ALL=1")
|
|
cache_entries.append("TEST_COMPONENTS=''")
|
|
else:
|
|
cache_entries.append("TESTS_ALL=0")
|
|
cache_entries.append("TEST_COMPONENTS='%s'" % " ".join(test_components))
|
|
|
|
if test_exclude_components:
|
|
cache_entries.append(
|
|
"TEST_EXCLUDE_COMPONENTS='%s'" % " ".join(test_exclude_components)
|
|
)
|
|
|
|
if cache_entries:
|
|
global_args.define_cache_entry = list(global_args.define_cache_entry)
|
|
global_args.define_cache_entry.extend(cache_entries)
|
|
|
|
# Brute force add reconfigure at the very beginning
|
|
reconfigure_task = ctx.invoke(ctx.command.get_command(ctx, "reconfigure"))
|
|
tasks.insert(0, reconfigure_task)
|
|
|
|
# Add global options
|
|
extensions = {
|
|
"global_options": [
|
|
# For convenience, define a -T and -E argument that gets converted to -D arguments
|
|
{
|
|
"names": ["-T", "--test-components"],
|
|
"help": "Specify the components to test",
|
|
"multiple": True,
|
|
},
|
|
{
|
|
"names": ["-E", "--test-exclude-components"],
|
|
"help": "Specify the components to exclude from testing",
|
|
"multiple": True,
|
|
},
|
|
],
|
|
"global_action_callbacks": [test_component_callback],
|
|
"actions": {},
|
|
}
|
|
|
|
# This generates per-config targets (clean, build, apply-config).
|
|
build_all_config_deps = []
|
|
clean_all_config_deps = []
|
|
|
|
for config in CONFIG_NAMES:
|
|
config_build_action_name = "ut-build-" + config
|
|
config_clean_action_name = "ut-clean-" + config
|
|
config_apply_config_action_name = "ut-apply-config-" + config
|
|
|
|
extensions["actions"][config_build_action_name] = {
|
|
"callback": ut_build,
|
|
"help": "Build unit-test-app with configuration provided in configs/NAME. "
|
|
+ "Build directory will be builds/%s/, " % config_build_action_name
|
|
+ "output binaries will be under output/%s/" % config_build_action_name,
|
|
}
|
|
|
|
extensions["actions"][config_clean_action_name] = {
|
|
"callback": ut_clean,
|
|
"help": "Remove build and output directories for configuration %s."
|
|
% config_clean_action_name,
|
|
}
|
|
|
|
extensions["actions"][config_apply_config_action_name] = {
|
|
"callback": ut_apply_config,
|
|
"help": "Generates configuration based on configs/%s in sdkconfig file."
|
|
% config_apply_config_action_name
|
|
+ "After this, normal all/flash targets can be used. Useful for development/debugging.",
|
|
}
|
|
|
|
build_all_config_deps.append(config_build_action_name)
|
|
clean_all_config_deps.append(config_clean_action_name)
|
|
|
|
extensions["actions"]["ut-build-all-configs"] = {
|
|
"callback": ut_build,
|
|
"help": "Build all configurations defined in configs/ directory.",
|
|
"dependencies": build_all_config_deps,
|
|
}
|
|
|
|
extensions["actions"]["ut-clean-all-configs"] = {
|
|
"callback": ut_clean,
|
|
"help": "Remove build and output directories for all configurations defined in configs/ directory.",
|
|
"dependencies": clean_all_config_deps,
|
|
}
|
|
|
|
return extensions
|