Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

tools: add {find,build}_apps.py, scripts to build multiple apps

Browse source 
This commit adds a pair of scripts, find_apps.py and build_apps.py.
These scripts are intended to be used in various CI jobs, building
multiple applications with different configurations and targets.

The first script, find_apps.py, is used to prepare the list of builds:
1. It finds apps for the given build system.
2. For each app, it finds configurations (sdkconfig files) which need
   to be built.
3. It filters out the apps and configurations which are not compatible
   with the given target.
4. It outputs the list of builds into stdout or a file. Currently the
   format is a list of lines, each line a JSON string. In the future,
   the tool can be updated to output YAML files.

The lists of builds can be concatenated and processed with standard
command line tools, like sed.

The second script, build_apps.py, executes the builds from the list.
It can execute a subset of builds based on --parallel-count and
--parallel-index arguments.

These two scripts are intended to replace build_examples_make,
build_examples_cmake, and the custom unit-test-app logic (in the
Makefile and idf_ext.py).

Closes IDF-641
This commit is contained in:
Ivan Grokhotkov 2019-09-11 11:23:10 +02:00 committed by Angus Gratton
parent bfbc10e2d8
commit 5d03ae7428
7 changed files with 864 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
tools/build_apps.py Executable file
View file

@ -0,0 +1,129 @@
#!/usr/bin/env python
# coding=utf-8
#
# ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
#
import argparse
import sys
import logging
from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS
def main():
parser = argparse.ArgumentParser(description="ESP-IDF app builder")
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--build-verbose",
action="store_true",
help="Enable verbose output from build system.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--parallel-count",
default=1,
type=int,
help="Number of parallel build jobs. Note that this script doesn't start the jobs, " +
"it needs to be executed multiple times with same value of --parallel-count and " +
"different values of --parallel-index.",
)
parser.add_argument(
"--parallel-index",
default=1,
type=int,
help="Index (1-based) of the job, out of the number specified by --parallel-count.",
)
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to read the list of builds",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Don't actually build, only print the build commands",
)
parser.add_argument(
"--keep-going",
action="store_true",
help="Don't exit immediately when a build fails.",
)
parser.add_argument(
"--output-build-list",
type=argparse.FileType("w"),
help="If specified, the list of builds (with all the placeholders expanded) will be written to this file.",
)
parser.add_argument(
"build_list",
type=argparse.FileType("r"),
nargs="?",
default=sys.stdin,
help="Name of the file to read the list of builds from. If not specified, read from stdin.",
)
args = parser.parse_args()
setup_logging(args)
build_items = [BuildItem.from_json(line) for line in args.build_list]
if not build_items:
logging.error("Empty build list!")
raise SystemExit(1)
num_builds = len(build_items)
num_jobs = args.parallel_count
job_index = args.parallel_index - 1 # convert to 0-based index
num_builds_per_job = (num_builds + num_jobs - 1) // num_jobs
min_job_index = num_builds_per_job * job_index
if min_job_index >= num_builds:
logging.warn("Nothing to do for job {} (build total: {}, per job: {})".format(
job_index + 1, num_builds, num_builds_per_job))
raise SystemExit(0)
max_job_index = min(num_builds_per_job * (job_index + 1) - 1, num_builds - 1)
logging.info("Total {} builds, max. {} builds per job, running builds {}-{}".format(
num_builds, num_builds_per_job, min_job_index + 1, max_job_index + 1))
builds_for_current_job = build_items[min_job_index:max_job_index + 1]
for i, build_info in enumerate(builds_for_current_job):
index = i + min_job_index + 1
build_info.index = index
build_info.dry_run = args.dry_run
build_info.verbose = args.build_verbose
build_info.keep_going = args.keep_going
logging.debug(" Build {}: {}".format(index, repr(build_info)))
if args.output_build_list:
args.output_build_list.write(build_info.to_json_expanded() + "\n")
failed_builds = []
for build_info in builds_for_current_job:
logging.info("Running build {}: {}".format(build_info.index, repr(build_info)))
build_system_class = BUILD_SYSTEMS[build_info.build_system]
try:
build_system_class.build(build_info)
except BuildError as e:
logging.error(e.message)
if args.keep_going:
failed_builds.append(build_info)
else:
raise SystemExit(1)
if failed_builds:
logging.error("The following build have failed:")
for build in failed_builds:
logging.error(" {}".format(build))
raise SystemExit(1)
if __name__ == "__main__":
main()

2
tools/ci/executable-list.txt
View file

@ -27,6 +27,7 @@ examples/system/ota/otatool/get_running_partition.py
examples/system/ota/otatool/otatool_example.py
examples/system/ota/otatool/otatool_example.sh
install.sh
tools/build_apps.py
tools/check_kconfigs.py
tools/check_python_dependencies.py
tools/ci/apply_bot_filter.py
@ -57,6 +58,7 @@ tools/esp_app_trace/logtrace_proc.py
tools/esp_app_trace/sysviewtrace_proc.py
tools/esp_app_trace/test/logtrace/test.sh
tools/esp_app_trace/test/sysview/test.sh
tools/find_apps.py
tools/format.sh
tools/gen_esp_err_to_name.py
tools/idf.py

283
tools/find_apps.py Executable file
View file

@ -0,0 +1,283 @@
#!/usr/bin/env python
# coding=utf-8
#
# ESP-IDF helper script to enumerate the builds of multiple configurations of multiple apps.
# Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds.
import argparse
import os
import sys
import re
import glob
import logging
import typing
from find_build_apps import (
BUILD_SYSTEMS,
BUILD_SYSTEM_CMAKE,
BuildSystem,
BuildItem,
setup_logging,
ConfigRule,
config_rules_from_str,
DEFAULT_TARGET,
)
# Helper functions
def dict_from_sdkconfig(path):
"""
Parse the sdkconfig file at 'path', return name:value pairs as a dict
"""
regex = re.compile(r"^([^#=]+)=(.+)$")
result = {}
with open(path) as f:
for line in f:
m = regex.match(line)
if m:
result[m.group(1)] = m.group(2)
return result
# Main logic: enumerating apps and builds
def find_builds_for_app(
app_path, work_dir, build_dir, build_log, target_arg, build_system,
config_rules): # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem]
"""
Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects
:param app_path: app directory (can be / usually will be a relative path)
:param work_dir: directory where the app should be copied before building.
May contain env. variables and placeholders.
:param build_dir: directory where the build will be done, relative to the work_dir. May contain placeholders.
:param build_log: path of the build log. May contain placeholders. May be None, in which case the log should go
into stdout/stderr.
:param target_arg: the value of IDF_TARGET passed to the script. Used to filter out configurations with
a different CONFIG_IDF_TARGET value.
:param build_system: name of the build system, index into BUILD_SYSTEMS dictionary
:param config_rules: mapping of sdkconfig file name patterns to configuration names
:return: list of BuildItems representing build configuration of the app
"""
build_items = [] # type: typing.List[BuildItem]
default_config_name = ""
for rule in config_rules:
if not rule.file_name:
default_config_name = rule.config_name
continue
sdkconfig_paths = glob.glob(os.path.join(app_path, rule.file_name))
sdkconfig_paths = sorted(sdkconfig_paths)
for sdkconfig_path in sdkconfig_paths:
# Check if the sdkconfig file specifies IDF_TARGET, and if it is matches the --target argument.
sdkconfig_dict = dict_from_sdkconfig(sdkconfig_path)
target_from_config = sdkconfig_dict.get("CONFIG_IDF_TARGET")
if target_from_config is not None and target_from_config != target_arg:
logging.debug("Skipping sdkconfig {} which requires target {}".format(
sdkconfig_path, target_from_config))
continue
# Figure out the config name
config_name = rule.config_name or ""
if "*" in rule.file_name:
# convert glob pattern into a regex
regex_str = r".*" + rule.file_name.replace(".", r"\.").replace("*", r"(.*)")
groups = re.match(regex_str, sdkconfig_path)
assert groups
config_name = groups.group(1)
sdkconfig_path = os.path.relpath(sdkconfig_path, app_path)
logging.debug('Adding build: app {}, sdkconfig {}, config name "{}"'.format(
app_path, sdkconfig_path, config_name))
build_items.append(
BuildItem(
app_path,
work_dir,
build_dir,
build_log,
target_arg,
sdkconfig_path,
config_name,
build_system,
))
if not build_items:
logging.debug('Adding build: app {}, default sdkconfig, config name "{}"'.format(app_path, default_config_name))
return [
BuildItem(
app_path,
work_dir,
build_dir,
build_log,
target_arg,
None,
default_config_name,
build_system,
)
]
return build_items
def find_apps(build_system_class, path, recursive, exclude_list,
target): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str]
"""
Find app directories in path (possibly recursively), which contain apps for the given build system, compatible
with the given target.
:param build_system_class: class derived from BuildSystem, representing the build system in use
:param path: path where to look for apps
:param recursive: whether to recursively descend into nested directories if no app is found
:param exclude_list: list of paths to be excluded from the recursive search
:param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped.
:return: list of paths of the apps found
"""
build_system_name = build_system_class.NAME
logging.debug("Looking for {} apps in {}{}".format(build_system_name, path, " recursively" if recursive else ""))
if not recursive:
if exclude_list:
logging.warn("--exclude option is ignored when used without --recursive")
if not build_system_class.is_app(path):
logging.warn("Path {} specified without --recursive flag, but no {} app found there".format(
path, build_system_name))
return []
return [path]
# The remaining part is for recursive == True
apps_found = [] # type: typing.List[str]
for root, dirs, _ in os.walk(path, topdown=True):
logging.debug("Entering {}".format(root))
if root in exclude_list:
logging.debug("Skipping {} (excluded)".format(root))
del dirs[:]
continue
if build_system_class.is_app(root):
logging.debug("Found {} app in {}".format(build_system_name, root))
# Don't recurse into app subdirectories
del dirs[:]
supported_targets = build_system_class.supported_targets(root)
if supported_targets and target not in supported_targets:
logging.debug("Skipping, app only supports targets: " + ", ".join(supported_targets))
continue
apps_found.append(root)
return apps_found
def main():
parser = argparse.ArgumentParser(description="Tool to generate build steps for IDF apps")
parser.add_argument(
"-v",
"--verbose",
action="count",
help="Increase the logging level of the script. Can be specified multiple times.",
)
parser.add_argument(
"--log-file",
type=argparse.FileType("w"),
help="Write the script log to the specified file, instead of stderr",
)
parser.add_argument(
"--recursive",
action="store_true",
help="Look for apps in the specified directories recursively.",
)
parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE)
parser.add_argument(
"--work-dir",
help="If set, the app is first copied into the specified directory, and then built." +
"If not set, the work directory is the directory of the app.",
)
parser.add_argument(
"--config",
action="append",
help="Adds configurations (sdkconfig file names) to build. This can either be " +
"FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " +
"relative to the project directory, to be used. Optional NAME can be specified, " +
"which can be used as a name of this configuration. FILEPATTERN is the name of " +
"the sdkconfig file, relative to the project directory, with at most one wildcard. " +
"The part captured by the wildcard is used as the name of the configuration.",
)
parser.add_argument(
"--build-dir",
help="If set, specifies the build directory name. Can expand placeholders. Can be either a " +
"name relative to the work directory, or an absolute path.",
)
parser.add_argument(
"--build-log",
help="If specified, the build log will be written to this file. Can expand placeholders.",
)
parser.add_argument("--target", help="Build apps for given target.")
parser.add_argument(
"--format",
default="json",
choices=["json"],
help="Format to write the list of builds as",
)
parser.add_argument(
"--exclude",
action="append",
help="Ignore specified directory (if --recursive is given). Can be used multiple times.",
)
parser.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
help="Output the list of builds to the specified file",
)
parser.add_argument("paths", nargs="+", help="One or more app paths.")
args = parser.parse_args()
setup_logging(args)
build_system_class = BUILD_SYSTEMS[args.build_system]
# If the build target is not set explicitly, get it from the environment or use the default one (esp32)
if not args.target:
env_target = os.environ.get("IDF_TARGET")
if env_target:
logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target))
args.target = env_target
else:
logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET))
args.target = DEFAULT_TARGET
# Prepare the list of app paths
app_paths = [] # type: typing.List[str]
for path in args.paths:
app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target)
if not app_paths:
logging.critical("No {} apps found".format(build_system_class.NAME))
raise SystemExit(1)
logging.info("Found {} apps".format(len(app_paths)))
app_paths = sorted(app_paths)
# Find compatible configurations of each app, collect them as BuildItems
build_items = [] # type: typing.List[BuildItem]
config_rules = config_rules_from_str(args.config or [])
for app_path in app_paths:
build_items += find_builds_for_app(
app_path,
args.work_dir,
args.build_dir,
args.build_log,
args.target,
args.build_system,
config_rules,
)
logging.info("Found {} builds".format(len(build_items)))
# Write out the BuildItems. Only JSON supported now (will add YAML later).
if args.format != "json":
raise NotImplementedError()
out = args.output or sys.stdout
out.writelines([item.to_json() + "\n" for item in build_items])
if __name__ == "__main__":
main()

31
tools/find_build_apps/__init__.py Normal file
View file

@ -0,0 +1,31 @@
from .common import (
BuildItem,
BuildSystem,
BuildError,
ConfigRule,
config_rules_from_str,
setup_logging,
DEFAULT_TARGET,
)
from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE
from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE
BUILD_SYSTEMS = {
BUILD_SYSTEM_MAKE: MakeBuildSystem,
BUILD_SYSTEM_CMAKE: CMakeBuildSystem,
}
__all__ = [
"BuildItem",
"BuildSystem",
"BuildError",
"ConfigRule",
"config_rules_from_str",
"setup_logging",
"DEFAULT_TARGET",
"CMakeBuildSystem",
"BUILD_SYSTEM_CMAKE",
"MakeBuildSystem",
"BUILD_SYSTEM_MAKE",
"BUILD_SYSTEMS",
]

158
tools/find_build_apps/cmake.py Normal file
View file

@ -0,0 +1,158 @@
import os
import sys
import subprocess
import logging
import shutil
import re
from .common import BuildSystem, BuildItem, BuildError
BUILD_SYSTEM_CMAKE = "cmake"
IDF_PY = "idf.py"
# While ESP-IDF component CMakeLists files can be identified by the presence of 'idf_component_register' string,
# there is no equivalent for the project CMakeLists files. This seems to be the best option...
CMAKE_PROJECT_LINE = r"include($ENV{IDF_PATH}/tools/cmake/project.cmake)"
SUPPORTED_TARGETS_REGEX = re.compile(r"set\(\s*SUPPORTED_TARGETS\s+([a-z_0-9\- ]+)\s*\)")
class CMakeBuildSystem(BuildSystem):
NAME = BUILD_SYSTEM_CMAKE
@staticmethod
def build(build_item): # type: (BuildItem) -> None
app_path = build_item.app_dir
work_path = build_item.work_dir or app_path
if not build_item.build_dir:
build_path = os.path.join(work_path, "build")
elif os.path.isabs(build_item.build_dir):
build_path = build_item.build_dir
else:
build_path = os.path.join(work_path, build_item.build_dir)
if work_path != app_path:
if os.path.exists(work_path):
logging.debug("Work directory {} exists, removing".format(work_path))
if not build_item.dry_run:
shutil.rmtree(work_path)
logging.debug("Copying app from {} to {}".format(app_path, work_path))
if not build_item.dry_run:
shutil.copytree(app_path, work_path)
if os.path.exists(build_path):
logging.debug("Build directory {} exists, removing".format(build_path))
if not build_item.dry_run:
shutil.rmtree(build_path)
if not build_item.dry_run:
os.makedirs(build_path)
# Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
# build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
#
# Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
# CMake variable. However here we do this manually to perform environment variable expansion in the
# sdkconfig files.
sdkconfig_defaults_list = ["sdkconfig.defaults"]
if build_item.sdkconfig_path:
sdkconfig_defaults_list.append(build_item.sdkconfig_path)
sdkconfig_file = os.path.join(work_path, "sdkconfig")
if os.path.exists(sdkconfig_file):
logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
if not build_item.dry_run:
os.unlink(sdkconfig_file)
logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
if not build_item.dry_run:
with open(sdkconfig_file, "w") as f_out:
for sdkconfig_name in sdkconfig_defaults_list:
sdkconfig_path = os.path.join(work_path, sdkconfig_name)
if not sdkconfig_path or not os.path.exists(sdkconfig_path):
continue
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
with open(sdkconfig_path, "r") as f_in:
for line in f_in:
f_out.write(os.path.expandvars(line))
# Also save the sdkconfig file in the build directory
shutil.copyfile(
os.path.join(work_path, "sdkconfig"),
os.path.join(build_path, "sdkconfig"),
)
else:
for sdkconfig_name in sdkconfig_defaults_list:
sdkconfig_path = os.path.join(app_path, sdkconfig_name)
if not sdkconfig_path:
continue
logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
if not os.path.exists(sdkconfig_path):
continue
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
# Prepare the build arguments
args = [
# Assume it is the responsibility of the caller to
# set up the environment (run . ./export.sh)
IDF_PY,
"-B",
build_path,
"-C",
work_path,
"-DIDF_TARGET=" + build_item.target,
]
if build_item.verbose:
args.append("-v")
args.append("build")
cmdline = format(" ".join(args))
logging.info("Running {}".format(cmdline))
if build_item.dry_run:
return
log_file = None
build_stdout = sys.stdout
build_stderr = sys.stderr
if build_item.build_log_path:
logging.info("Writing build log to {}".format(build_item.build_log_path))
log_file = open(build_item.build_log_path, "w")
build_stdout = log_file
build_stderr = log_file
try:
subprocess.check_call(args, stdout=build_stdout, stderr=build_stderr)
except subprocess.CalledProcessError as e:
raise BuildError("Build failed with exit code {}".format(e.returncode))
finally:
if log_file:
log_file.close()
@staticmethod
def _read_cmakelists(app_path):
cmakelists_path = os.path.join(app_path, "CMakeLists.txt")
if not os.path.exists(cmakelists_path):
return None
with open(cmakelists_path, "r") as cmakelists_file:
return cmakelists_file.read()
@staticmethod
def is_app(path):
cmakelists_file_content = CMakeBuildSystem._read_cmakelists(path)
if not cmakelists_file_content:
return False
if CMAKE_PROJECT_LINE not in cmakelists_file_content:
return False
return True
@staticmethod
def supported_targets(app_path):
cmakelists_file_content = CMakeBuildSystem._read_cmakelists(app_path)
if not cmakelists_file_content:
return None
match = re.findall(SUPPORTED_TARGETS_REGEX, cmakelists_file_content)
if not match:
return None
if len(match) > 1:
raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
targets = match[0].split(" ")
return targets

231
tools/find_build_apps/common.py Normal file
View file

@ -0,0 +1,231 @@
# coding=utf-8
import sys
import os
from collections import namedtuple
import logging
import json
import typing
DEFAULT_TARGET = "esp32"
TARGET_PLACEHOLDER = "@t"
WILDCARD_PLACEHOLDER = "@w"
NAME_PLACEHOLDER = "@n"
FULL_NAME_PLACEHOLDER = "@f"
INDEX_PLACEHOLDER = "@i"
# ConfigRule represents one --config argument of find_apps.py.
# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
# file_name can also be empty to indicate that the default configuration of the app should be used.
# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
# For example:
# filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
# filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"])
def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.List[ConfigRule]
"""
Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
:param rule_strings: list of rules as strings
:return: list of ConfigRules
"""
rules = [] # type: typing.List[ConfigRule]
for rule_str in rule_strings:
items = rule_str.split("=", 2)
rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
return rules
class BuildItem(object):
"""
Instance of this class represents one build of an application.
The parameters which distinguish the build are passed to the constructor.
"""
def __init__(
self,
app_path,
work_dir,
build_path,
build_log_path,
target,
sdkconfig_path,
config_name,
build_system,
):
# These internal variables store the paths with environment variables and placeholders;
# Public properties with similar names use the _expand method to get the actual paths.
self._app_dir = app_path
self._work_dir = work_dir
self._build_dir = build_path
self._build_log_path = build_log_path
self.sdkconfig_path = sdkconfig_path
self.config_name = config_name
self.target = target
self.build_system = build_system
self._app_name = os.path.basename(os.path.normpath(app_path))
# Some miscellaneous build properties which are set later, at the build stage
self.index = None
self.verbose = False
self.dry_run = False
self.keep_going = False
@property
def app_dir(self):
"""
:return: directory of the app
"""
return self._expand(self._app_dir)
@property
def work_dir(self):
"""
:return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
directory should be used.
"""
return self._expand(self._work_dir)
@property
def build_dir(self):
"""
:return: build directory, either relative to the work directory (if relative path is used) or absolute path.
"""
return self._expand(self._build_dir)
@property
def build_log_path(self):
"""
:return: path of the build log file
"""
return self._expand(self._build_log_path)
def __repr__(self):
return "Build app {} for target {}, sdkconfig {} in {}".format(
self.app_dir,
self.target,
self.sdkconfig_path or "(default)",
self.build_dir,
)
def to_json(self): # type: () -> str
"""
:return: JSON string representing this object
"""
return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)
def to_json_expanded(self): # type: () -> str
"""
:return: JSON string representing this object, with all placeholders in paths expanded
"""
return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)
def _to_json(self, app_dir, work_dir, build_dir, build_log_path): # type: (str, str, str, str) -> str
"""
Internal function, called by to_json and to_json_expanded
"""
return json.dumps({
"build_system": self.build_system,
"app_dir": app_dir,
"work_dir": work_dir,
"build_dir": build_dir,
"build_log_path": build_log_path,
"sdkconfig": self.sdkconfig_path,
"config": self.config_name,
"target": self.target,
"verbose": self.verbose,
})
@staticmethod
def from_json(json_str): # type: (typing.Text) -> BuildItem
"""
:return: Get the BuildItem from a JSON string
"""
d = json.loads(str(json_str))
result = BuildItem(
app_path=d["app_dir"],
work_dir=d["work_dir"],
build_path=d["build_dir"],
build_log_path=d["build_log_path"],
sdkconfig_path=d["sdkconfig"],
config_name=d["config"],
target=d["target"],
build_system=d["build_system"],
)
result.verbose = d["verbose"]
return result
def _expand(self, path): # type: (str) -> str
"""
Internal method, expands any of the placeholders in {app,work,build} paths.
"""
if not path:
return path
if self.index is not None:
path = path.replace(INDEX_PLACEHOLDER, str(self.index))
path = path.replace(TARGET_PLACEHOLDER, self.target)
path = path.replace(NAME_PLACEHOLDER, self._app_name)
if (FULL_NAME_PLACEHOLDER in path): # to avoid recursion to the call to app_dir in the next line:
path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_"))
wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
if wildcard_pos != -1:
if self.config_name:
# if config name is defined, put it in place of the placeholder
path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
else:
# otherwise, remove the placeholder and one character on the left
# (which is usually an underscore, dash, or other delimiter)
left_of_wildcard = max(0, wildcard_pos - 1)
right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
path = path[0:left_of_wildcard] + path[right_of_wildcard:]
path = os.path.expandvars(path)
return path
class BuildSystem(object):
"""
Class representing a build system.
Derived classes implement the methods below.
Objects of these classes aren't instantiated, instead the class (type object) is used.
"""
NAME = "undefined"
@staticmethod
def build(self):
raise NotImplementedError()
@staticmethod
def is_app(path):
raise NotImplementedError()
@staticmethod
def supported_targets(app_path):
raise NotImplementedError()
class BuildError(RuntimeError):
pass
def setup_logging(args):
"""
Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
:param args: namespace obtained from argparse
"""
if not args.verbose:
log_level = logging.WARNING
elif args.verbose == 1:
log_level = logging.INFO
else:
log_level = logging.DEBUG
logging.basicConfig(
format="%(levelname)s: %(message)s",
stream=args.log_file or sys.stderr,
level=log_level,
)

30
tools/find_build_apps/make.py Normal file
View file

@ -0,0 +1,30 @@
import os
from .common import BuildSystem
# Same for the Makefile projects:
MAKE_PROJECT_LINE = r"include $(IDF_PATH)/make/project.mk"
BUILD_SYSTEM_MAKE = "make"
class MakeBuildSystem(BuildSystem):
NAME = BUILD_SYSTEM_MAKE
@staticmethod
def build(build_item):
raise NotImplementedError()
@staticmethod
def is_app(path):
makefile_path = os.path.join(path, "Makefile")
if not os.path.exists(makefile_path):
return False
with open(makefile_path, "r") as makefile:
makefile_content = makefile.read()
if MAKE_PROJECT_LINE not in makefile_content:
return False
return True
@staticmethod
def supported_targets(app_path):
return ["esp32"]