modify unit-test-apps config files, use CONFIG_IDF_TARGET to set target. if not set, then treat them as supported both targets.
286 lines
11 KiB
Python
Executable file
286 lines
11 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# coding=utf-8
|
|
#
|
|
# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps.
|
|
# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import re
|
|
import glob
|
|
import logging
|
|
import typing
|
|
from find_build_apps import (
|
|
BUILD_SYSTEMS,
|
|
BUILD_SYSTEM_CMAKE,
|
|
BuildSystem,
|
|
BuildItem,
|
|
setup_logging,
|
|
ConfigRule,
|
|
config_rules_from_str,
|
|
DEFAULT_TARGET,
|
|
)
|
|
|
|
# Helper functions
|
|
|
|
|
|
def dict_from_sdkconfig(path):
|
|
"""
|
|
Parse the sdkconfig file at 'path', return name:value pairs as a dict
|
|
"""
|
|
regex = re.compile(r"^([^#=]+)=(.+)$")
|
|
result = {}
|
|
with open(path) as f:
|
|
for line in f:
|
|
m = regex.match(line)
|
|
if m:
|
|
val = m.group(2)
|
|
if val.startswith('"') and val.endswith('"'):
|
|
val = val[1:-1]
|
|
result[m.group(1)] = val
|
|
return result
|
|
|
|
|
|
# Main logic: enumerating apps and builds
|
|
|
|
|
|
def find_builds_for_app(
|
|
app_path, work_dir, build_dir, build_log, target_arg, build_system,
|
|
config_rules): # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem]
|
|
"""
|
|
Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
|
|
:param app_path: app directory (can be / usually will be a relative path)
|
|
:param work_dir: directory where the app should be copied before building.
|
|
May contain env. variables and placeholders.
|
|
:param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders.
|
|
:param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go
|
|
into stdout/stderr.
|
|
:param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with
|
|
a different CONFIG_IDF_TARGET value.
|
|
:param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
|
|
:param config_rules: mapping of sdkconfig file name patterns to configuration names
|
|
:return: list of BuildItems representing build configuration of the app
|
|
"""
|
|
build_items = [] # type: typing.List[BuildItem]
|
|
default_config_name = ""
|
|
|
|
for rule in config_rules:
|
|
if not rule.file_name:
|
|
default_config_name = rule.config_name
|
|
continue
|
|
|
|
sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name))
|
|
sdkconfig_paths = sorted(sdkconfig_paths)
|
|
for sdkconfig_path in sdkconfig_paths:
|
|
|
|
# Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
|
|
sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
|
|
target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET")
|
|
if target_from_config is not None and target_from_config != target_arg:
|
|
logging.debug("Skipping sdkconfig {} which requires target {}".format(
|
|
sdkconfig_path, target_from_config))
|
|
continue
|
|
|
|
# Figure out the config name
|
|
config_name = rule.config_name or ""
|
|
if "*" in rule.file_name:
|
|
# convert glob pattern into a regex
|
|
regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)")
|
|
groups = re.match(regex_str, sdkconfig_path)
|
|
assert groups
|
|
config_name = groups.group(1)
|
|
|
|
sdkconfig_path = os.path.relpath(sdkconfig_path, app_path)
|
|
logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format(
|
|
app_path, sdkconfig_path, config_name))
|
|
build_items.append(
|
|
BuildItem(
|
|
app_path,
|
|
work_dir,
|
|
build_dir,
|
|
build_log,
|
|
target_arg,
|
|
sdkconfig_path,
|
|
config_name,
|
|
build_system,
|
|
))
|
|
|
|
if not build_items:
|
|
logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name))
|
|
return [
|
|
BuildItem(
|
|
app_path,
|
|
work_dir,
|
|
build_dir,
|
|
build_log,
|
|
target_arg,
|
|
None,
|
|
default_config_name,
|
|
build_system,
|
|
)
|
|
]
|
|
|
|
return build_items
|
|
|
|
|
|
def find_apps(build_system_class, path, recursive, exclude_list,
|
|
target): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
|
|
"""
|
|
Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
|
|
with the given target.
|
|
:param build_system_class: class derived from BuildSystem, representing the build system in use
|
|
:param path: path where to look for apps
|
|
:param recursive: whether to recursively descend into nested directories if no app is found
|
|
:param exclude_list: list of paths to be excluded from the recursive search
|
|
:param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped.
|
|
:return: list of paths of the apps found
|
|
"""
|
|
build_system_name = build_system_class.NAME
|
|
logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else ""))
|
|
if not recursive:
|
|
if exclude_list:
|
|
logging.warn("--exclude option is ignored when used without --recursive")
|
|
if not build_system_class.is_app(path):
|
|
logging.warn("Path {} specified without --recursive flag, but no {} app found there".format(
|
|
path, build_system_name))
|
|
return []
|
|
return [path]
|
|
|
|
# The remaining part is for recursive == True
|
|
apps_found = [] # type: typing.List[str]
|
|
for root, dirs, _ in os.walk(path, topdown=True):
|
|
logging.debug("Entering {}".format(root))
|
|
if root in exclude_list:
|
|
logging.debug("Skipping {} (excluded)".format(root))
|
|
del dirs[:]
|
|
continue
|
|
|
|
if build_system_class.is_app(root):
|
|
logging.debug("Found {} app in {}".format(build_system_name, root))
|
|
# Don't recurse into app subdirectories
|
|
del dirs[:]
|
|
|
|
supported_targets = build_system_class.supported_targets(root)
|
|
if supported_targets and target not in supported_targets:
|
|
logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets))
|
|
continue
|
|
|
|
apps_found.append(root)
|
|
|
|
return apps_found
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps")
|
|
parser.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
action="count",
|
|
help="Increase the logging level of the script. Can be specified multiple times.",
|
|
)
|
|
parser.add_argument(
|
|
"--log-file",
|
|
type=argparse.FileType("w"),
|
|
help="Write the script log to the specified file, instead of stderr",
|
|
)
|
|
parser.add_argument(
|
|
"--recursive",
|
|
action="store_true",
|
|
help="Look for apps in the specified directories recursively.",
|
|
)
|
|
parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
|
|
parser.add_argument(
|
|
"--work-dir",
|
|
help="If set, the app is first copied into the specified directory, and then built." +
|
|
"If not set, the work directory is the directory of the app.",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
action="append",
|
|
help="Adds configurations (sdkconfig file names) to build. This can either be " +
|
|
"FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " +
|
|
"relative to the project directory, to be used. Optional NAME can be specified, " +
|
|
"which can be used as a name of this configuration. FILEPATTERN is the name of " +
|
|
"the sdkconfig file, relative to the project directory, with at most one wildcard. " +
|
|
"The part captured by the wildcard is used as the name of the configuration.",
|
|
)
|
|
parser.add_argument(
|
|
"--build-dir",
|
|
help="If set, specifies the build directory name. Can expand placeholders. Can be either a " +
|
|
"name relative to the work directory, or an absolute path.",
|
|
)
|
|
parser.add_argument(
|
|
"--build-log",
|
|
help="If specified, the build log will be written to this file. Can expand placeholders.",
|
|
)
|
|
parser.add_argument("--target", help="Build apps for given target.")
|
|
parser.add_argument(
|
|
"--format",
|
|
default="json",
|
|
choices=["json"],
|
|
help="Format to write the list of builds as",
|
|
)
|
|
parser.add_argument(
|
|
"--exclude",
|
|
action="append",
|
|
help="Ignore specified directory (if --recursive is given). Can be used multiple times.",
|
|
)
|
|
parser.add_argument(
|
|
"-o",
|
|
"--output",
|
|
type=argparse.FileType("w"),
|
|
help="Output the list of builds to the specified file",
|
|
)
|
|
parser.add_argument("paths", nargs="+", help="One or more app paths.")
|
|
args = parser.parse_args()
|
|
setup_logging(args)
|
|
|
|
build_system_class = BUILD_SYSTEMS[args.build_system]
|
|
|
|
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
|
|
if not args.target:
|
|
env_target = os.environ.get("IDF_TARGET")
|
|
if env_target:
|
|
logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
|
|
args.target = env_target
|
|
else:
|
|
logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
|
|
args.target = DEFAULT_TARGET
|
|
|
|
# Prepare the list of app paths
|
|
app_paths = [] # type: typing.List[str]
|
|
for path in args.paths:
|
|
app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
|
|
|
|
if not app_paths:
|
|
logging.critical("No {} apps found".format(build_system_class.NAME))
|
|
raise SystemExit(1)
|
|
logging.info("Found {} apps".format(len(app_paths)))
|
|
|
|
app_paths = sorted(app_paths)
|
|
|
|
# Find compatible configurations of each app, collect them as BuildItems
|
|
build_items = [] # type: typing.List[BuildItem]
|
|
config_rules = config_rules_from_str(args.config or [])
|
|
for app_path in app_paths:
|
|
build_items += find_builds_for_app(
|
|
app_path,
|
|
args.work_dir,
|
|
args.build_dir,
|
|
args.build_log,
|
|
args.target,
|
|
args.build_system,
|
|
config_rules,
|
|
)
|
|
logging.info("Found {} builds".format(len(build_items)))
|
|
|
|
# Write out the BuildItems. Only JSON supported now (will add YAML later).
|
|
if args.format != "json":
|
|
raise NotImplementedError()
|
|
out = args.output or sys.stdout
|
|
out.writelines([item.to_json() + "\n" for item in build_items])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|