1aacb6e2cf
TEST_GROUPS can be included in the unit-test-app config file, to restrict the list of tests to be executed. However this option is not used at build time, so adding it as a CMake variable along with TEST_COMPONENTS and other options results in a CMake warning. Fix by not passing this variable to CMake, and not including it in the final sdkconfig file.
385 lines
14 KiB
Python
385 lines
14 KiB
Python
# coding=utf-8
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import os
|
|
from abc import abstractmethod
|
|
from collections import namedtuple
|
|
import logging
|
|
import json
|
|
import typing
|
|
from io import open
|
|
|
|
DEFAULT_TARGET = "esp32"
|
|
|
|
TARGET_PLACEHOLDER = "@t"
|
|
WILDCARD_PLACEHOLDER = "@w"
|
|
NAME_PLACEHOLDER = "@n"
|
|
FULL_NAME_PLACEHOLDER = "@f"
|
|
INDEX_PLACEHOLDER = "@i"
|
|
|
|
SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")
|
|
|
|
# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
|
|
SDKCONFIG_TEST_OPTS = [
|
|
"EXCLUDE_COMPONENTS",
|
|
"TEST_EXCLUDE_COMPONENTS",
|
|
"TEST_COMPONENTS",
|
|
]
|
|
|
|
# These keys in sdkconfig.defaults are not propagated to the final sdkconfig file:
|
|
SDKCONFIG_IGNORE_OPTS = [
|
|
"TEST_GROUPS"
|
|
]
|
|
|
|
# ConfigRule represents one --config argument of find_apps.py.
|
|
# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
|
|
# file_name can also be empty to indicate that the default configuration of the app should be used.
|
|
# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
|
|
# For example:
|
|
# filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
|
|
# filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
|
|
ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"])
|
|
|
|
|
|
def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.List[ConfigRule]
|
|
"""
|
|
Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
|
|
:param rule_strings: list of rules as strings
|
|
:return: list of ConfigRules
|
|
"""
|
|
rules = [] # type: typing.List[ConfigRule]
|
|
for rule_str in rule_strings:
|
|
items = rule_str.split("=", 2)
|
|
rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
|
|
return rules
|
|
|
|
|
|
class BuildItem(object):
|
|
"""
|
|
Instance of this class represents one build of an application.
|
|
The parameters which distinguish the build are passed to the constructor.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
app_path,
|
|
work_dir,
|
|
build_path,
|
|
build_log_path,
|
|
target,
|
|
sdkconfig_path,
|
|
config_name,
|
|
build_system,
|
|
):
|
|
# These internal variables store the paths with environment variables and placeholders;
|
|
# Public properties with similar names use the _expand method to get the actual paths.
|
|
self._app_dir = app_path
|
|
self._work_dir = work_dir
|
|
self._build_dir = build_path
|
|
self._build_log_path = build_log_path
|
|
|
|
self.sdkconfig_path = sdkconfig_path
|
|
self.config_name = config_name
|
|
self.target = target
|
|
self.build_system = build_system
|
|
|
|
self._app_name = os.path.basename(os.path.normpath(app_path))
|
|
|
|
# Some miscellaneous build properties which are set later, at the build stage
|
|
self.index = None
|
|
self.verbose = False
|
|
self.dry_run = False
|
|
self.keep_going = False
|
|
|
|
@property
|
|
def app_dir(self):
|
|
"""
|
|
:return: directory of the app
|
|
"""
|
|
return self._expand(self._app_dir)
|
|
|
|
@property
|
|
def work_dir(self):
|
|
"""
|
|
:return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
|
|
directory should be used.
|
|
"""
|
|
return self._expand(self._work_dir)
|
|
|
|
@property
|
|
def build_dir(self):
|
|
"""
|
|
:return: build directory, either relative to the work directory (if relative path is used) or absolute path.
|
|
"""
|
|
return self._expand(self._build_dir)
|
|
|
|
@property
|
|
def build_log_path(self):
|
|
"""
|
|
:return: path of the build log file
|
|
"""
|
|
return self._expand(self._build_log_path)
|
|
|
|
def __repr__(self):
|
|
return "Build app {} for target {}, sdkconfig {} in {}".format(
|
|
self.app_dir,
|
|
self.target,
|
|
self.sdkconfig_path or "(default)",
|
|
self.build_dir,
|
|
)
|
|
|
|
def to_json(self): # type: () -> str
|
|
"""
|
|
:return: JSON string representing this object
|
|
"""
|
|
return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)
|
|
|
|
def to_json_expanded(self): # type: () -> str
|
|
"""
|
|
:return: JSON string representing this object, with all placeholders in paths expanded
|
|
"""
|
|
return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)
|
|
|
|
def _to_json(self, app_dir, work_dir, build_dir, build_log_path): # type: (str, str, str, str) -> str
|
|
"""
|
|
Internal function, called by to_json and to_json_expanded
|
|
"""
|
|
return json.dumps({
|
|
"build_system": self.build_system,
|
|
"app_dir": app_dir,
|
|
"work_dir": work_dir,
|
|
"build_dir": build_dir,
|
|
"build_log_path": build_log_path,
|
|
"sdkconfig": self.sdkconfig_path,
|
|
"config": self.config_name,
|
|
"target": self.target,
|
|
"verbose": self.verbose,
|
|
})
|
|
|
|
@staticmethod
|
|
def from_json(json_str): # type: (typing.Text) -> BuildItem
|
|
"""
|
|
:return: Get the BuildItem from a JSON string
|
|
"""
|
|
d = json.loads(str(json_str))
|
|
result = BuildItem(
|
|
app_path=d["app_dir"],
|
|
work_dir=d["work_dir"],
|
|
build_path=d["build_dir"],
|
|
build_log_path=d["build_log_path"],
|
|
sdkconfig_path=d["sdkconfig"],
|
|
config_name=d["config"],
|
|
target=d["target"],
|
|
build_system=d["build_system"],
|
|
)
|
|
result.verbose = d["verbose"]
|
|
return result
|
|
|
|
def _expand(self, path): # type: (str) -> str
|
|
"""
|
|
Internal method, expands any of the placeholders in {app,work,build} paths.
|
|
"""
|
|
if not path:
|
|
return path
|
|
|
|
if self.index is not None:
|
|
path = path.replace(INDEX_PLACEHOLDER, str(self.index))
|
|
path = path.replace(TARGET_PLACEHOLDER, self.target)
|
|
path = path.replace(NAME_PLACEHOLDER, self._app_name)
|
|
if (FULL_NAME_PLACEHOLDER in path): # to avoid recursion to the call to app_dir in the next line:
|
|
path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_"))
|
|
wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
|
|
if wildcard_pos != -1:
|
|
if self.config_name:
|
|
# if config name is defined, put it in place of the placeholder
|
|
path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
|
|
else:
|
|
# otherwise, remove the placeholder and one character on the left
|
|
# (which is usually an underscore, dash, or other delimiter)
|
|
left_of_wildcard = max(0, wildcard_pos - 1)
|
|
right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
|
|
path = path[0:left_of_wildcard] + path[right_of_wildcard:]
|
|
path = os.path.expandvars(path)
|
|
return path
|
|
|
|
|
|
class BuildSystem(object):
|
|
"""
|
|
Class representing a build system.
|
|
Derived classes implement the methods below.
|
|
Objects of these classes aren't instantiated, instead the class (type object) is used.
|
|
"""
|
|
NAME = "undefined"
|
|
SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)')
|
|
|
|
@classmethod
|
|
def build_prepare(cls, build_item):
|
|
app_path = build_item.app_dir
|
|
work_path = build_item.work_dir or app_path
|
|
if not build_item.build_dir:
|
|
build_path = os.path.join(work_path, "build")
|
|
elif os.path.isabs(build_item.build_dir):
|
|
build_path = build_item.build_dir
|
|
else:
|
|
build_path = os.path.join(work_path, build_item.build_dir)
|
|
|
|
if work_path != app_path:
|
|
if os.path.exists(work_path):
|
|
logging.debug("Work directory {} exists, removing".format(work_path))
|
|
if not build_item.dry_run:
|
|
shutil.rmtree(work_path)
|
|
logging.debug("Copying app from {} to {}".format(app_path, work_path))
|
|
if not build_item.dry_run:
|
|
shutil.copytree(app_path, work_path)
|
|
|
|
if os.path.exists(build_path):
|
|
logging.debug("Build directory {} exists, removing".format(build_path))
|
|
if not build_item.dry_run:
|
|
shutil.rmtree(build_path)
|
|
|
|
if not build_item.dry_run:
|
|
os.makedirs(build_path)
|
|
|
|
# Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
|
|
# build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
|
|
#
|
|
# Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
|
|
# CMake variable. However here we do this manually to perform environment variable expansion in the
|
|
# sdkconfig files.
|
|
sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target]
|
|
if build_item.sdkconfig_path:
|
|
sdkconfig_defaults_list.append(build_item.sdkconfig_path)
|
|
|
|
sdkconfig_file = os.path.join(work_path, "sdkconfig")
|
|
if os.path.exists(sdkconfig_file):
|
|
logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
|
|
if not build_item.dry_run:
|
|
os.unlink(sdkconfig_file)
|
|
|
|
logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
|
|
extra_cmakecache_items = {}
|
|
if not build_item.dry_run:
|
|
with open(sdkconfig_file, "w") as f_out:
|
|
for sdkconfig_name in sdkconfig_defaults_list:
|
|
sdkconfig_path = os.path.join(work_path, sdkconfig_name)
|
|
if not sdkconfig_path or not os.path.exists(sdkconfig_path):
|
|
continue
|
|
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
|
|
with open(sdkconfig_path, "r") as f_in:
|
|
for line in f_in:
|
|
if not line.endswith("\n"):
|
|
line += "\n"
|
|
if cls.NAME == 'cmake':
|
|
m = SDKCONFIG_LINE_REGEX.match(line)
|
|
key = m.group(1) if m else None
|
|
if key in SDKCONFIG_TEST_OPTS:
|
|
extra_cmakecache_items[key] = m.group(2)
|
|
continue
|
|
if key in SDKCONFIG_IGNORE_OPTS:
|
|
continue
|
|
f_out.write(os.path.expandvars(line))
|
|
else:
|
|
for sdkconfig_name in sdkconfig_defaults_list:
|
|
sdkconfig_path = os.path.join(app_path, sdkconfig_name)
|
|
if not sdkconfig_path:
|
|
continue
|
|
logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
|
|
if not os.path.exists(sdkconfig_path):
|
|
continue
|
|
logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
|
|
|
|
# The preparation of build is finished. Implement the build part in sub classes.
|
|
if cls.NAME == 'cmake':
|
|
return build_path, work_path, extra_cmakecache_items
|
|
else:
|
|
return build_path, work_path
|
|
|
|
@staticmethod
|
|
@abstractmethod
|
|
def build(build_item):
|
|
pass
|
|
|
|
@staticmethod
|
|
@abstractmethod
|
|
def is_app(path):
|
|
pass
|
|
|
|
@staticmethod
|
|
def _read_readme(app_path):
|
|
# Markdown supported targets should be:
|
|
# e.g. | Supported Targets | ESP32 |
|
|
# | ----------------- | ----- |
|
|
# reStructuredText supported targets should be:
|
|
# e.g. ================= =====
|
|
# Supported Targets ESP32
|
|
# ================= =====
|
|
def get_md_or_rst(app_path):
|
|
readme_path = os.path.join(app_path, 'README.md')
|
|
if not os.path.exists(readme_path):
|
|
readme_path = os.path.join(app_path, 'README.rst')
|
|
if not os.path.exists(readme_path):
|
|
return None
|
|
return readme_path
|
|
|
|
readme_path = get_md_or_rst(app_path)
|
|
# Handle sub apps situation, e.g. master-slave
|
|
if not readme_path:
|
|
readme_path = get_md_or_rst(os.path.dirname(app_path))
|
|
if not readme_path:
|
|
return None
|
|
with open(readme_path, "r", encoding='utf8') as readme_file:
|
|
return readme_file.read()
|
|
|
|
@staticmethod
|
|
def supported_targets(app_path):
|
|
formal_to_usual = {
|
|
'ESP32': 'esp32',
|
|
'ESP32-S2': 'esp32s2',
|
|
}
|
|
|
|
readme_file_content = BuildSystem._read_readme(app_path)
|
|
if not readme_file_content:
|
|
return None
|
|
match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
|
|
if not match:
|
|
return None
|
|
if len(match) > 1:
|
|
raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
|
|
support_str = match[0].strip()
|
|
|
|
targets = []
|
|
for part in support_str.split('|'):
|
|
for inner in part.split(' '):
|
|
inner = inner.strip()
|
|
if not inner:
|
|
continue
|
|
elif inner in formal_to_usual:
|
|
targets.append(formal_to_usual[inner])
|
|
else:
|
|
raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
|
|
.format(inner, app_path, ', '.join(formal_to_usual.keys())))
|
|
return targets
|
|
|
|
|
|
class BuildError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def setup_logging(args):
|
|
"""
|
|
Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
|
|
:param args: namespace obtained from argparse
|
|
"""
|
|
if not args.verbose:
|
|
log_level = logging.WARNING
|
|
elif args.verbose == 1:
|
|
log_level = logging.INFO
|
|
else:
|
|
log_level = logging.DEBUG
|
|
|
|
logging.basicConfig(
|
|
format="%(levelname)s: %(message)s",
|
|
stream=args.log_file or sys.stderr,
|
|
level=log_level,
|
|
)
|