# coding=utf-8 import re import shutil import sys import os from abc import abstractmethod from collections import namedtuple import logging import json import typing from io import open DEFAULT_TARGET = "esp32" TARGET_PLACEHOLDER = "@t" WILDCARD_PLACEHOLDER = "@w" NAME_PLACEHOLDER = "@n" FULL_NAME_PLACEHOLDER = "@f" INDEX_PLACEHOLDER = "@i" SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$") # If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake SDKCONFIG_TEST_OPTS = [ "EXCLUDE_COMPONENTS", "TEST_EXCLUDE_COMPONENTS", "TEST_COMPONENTS", ] # These keys in sdkconfig.defaults are not propagated to the final sdkconfig file: SDKCONFIG_IGNORE_OPTS = [ "TEST_GROUPS" ] # ConfigRule represents one --config argument of find_apps.py. # file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character). # file_name can also be empty to indicate that the default configuration of the app should be used. # config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used. # For example: # filename='', config_name='default' — represents the default app configuration, and gives it a name 'default' # filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"]) def config_rules_from_str(rule_strings): # type: (typing.List[str]) -> typing.List[ConfigRule] """ Helper function to convert strings like 'file_name=config_name' into ConfigRule objects :param rule_strings: list of rules as strings :return: list of ConfigRules """ rules = [] # type: typing.List[ConfigRule] for rule_str in rule_strings: items = rule_str.split("=", 2) rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None)) return rules class BuildItem(object): """ Instance of this class represents one build of an application. The parameters which distinguish the build are passed to the constructor. """ def __init__( self, app_path, work_dir, build_path, build_log_path, target, sdkconfig_path, config_name, build_system, ): # These internal variables store the paths with environment variables and placeholders; # Public properties with similar names use the _expand method to get the actual paths. self._app_dir = app_path self._work_dir = work_dir self._build_dir = build_path self._build_log_path = build_log_path self.sdkconfig_path = sdkconfig_path self.config_name = config_name self.target = target self.build_system = build_system self._app_name = os.path.basename(os.path.normpath(app_path)) # Some miscellaneous build properties which are set later, at the build stage self.index = None self.verbose = False self.dry_run = False self.keep_going = False @property def app_dir(self): """ :return: directory of the app """ return self._expand(self._app_dir) @property def work_dir(self): """ :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app directory should be used. """ return self._expand(self._work_dir) @property def build_dir(self): """ :return: build directory, either relative to the work directory (if relative path is used) or absolute path. """ return self._expand(self._build_dir) @property def build_log_path(self): """ :return: path of the build log file """ return self._expand(self._build_log_path) def __repr__(self): return "Build app {} for target {}, sdkconfig {} in {}".format( self.app_dir, self.target, self.sdkconfig_path or "(default)", self.build_dir, ) def to_json(self): # type: () -> str """ :return: JSON string representing this object """ return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path) def to_json_expanded(self): # type: () -> str """ :return: JSON string representing this object, with all placeholders in paths expanded """ return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path) def _to_json(self, app_dir, work_dir, build_dir, build_log_path): # type: (str, str, str, str) -> str """ Internal function, called by to_json and to_json_expanded """ return json.dumps({ "build_system": self.build_system, "app_dir": app_dir, "work_dir": work_dir, "build_dir": build_dir, "build_log_path": build_log_path, "sdkconfig": self.sdkconfig_path, "config": self.config_name, "target": self.target, "verbose": self.verbose, }) @staticmethod def from_json(json_str): # type: (typing.Text) -> BuildItem """ :return: Get the BuildItem from a JSON string """ d = json.loads(str(json_str)) result = BuildItem( app_path=d["app_dir"], work_dir=d["work_dir"], build_path=d["build_dir"], build_log_path=d["build_log_path"], sdkconfig_path=d["sdkconfig"], config_name=d["config"], target=d["target"], build_system=d["build_system"], ) result.verbose = d["verbose"] return result def _expand(self, path): # type: (str) -> str """ Internal method, expands any of the placeholders in {app,work,build} paths. """ if not path: return path if self.index is not None: path = path.replace(INDEX_PLACEHOLDER, str(self.index)) path = path.replace(TARGET_PLACEHOLDER, self.target) path = path.replace(NAME_PLACEHOLDER, self._app_name) if (FULL_NAME_PLACEHOLDER in path): # to avoid recursion to the call to app_dir in the next line: path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_")) wildcard_pos = path.find(WILDCARD_PLACEHOLDER) if wildcard_pos != -1: if self.config_name: # if config name is defined, put it in place of the placeholder path = path.replace(WILDCARD_PLACEHOLDER, self.config_name) else: # otherwise, remove the placeholder and one character on the left # (which is usually an underscore, dash, or other delimiter) left_of_wildcard = max(0, wildcard_pos - 1) right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER) path = path[0:left_of_wildcard] + path[right_of_wildcard:] path = os.path.expandvars(path) return path class BuildSystem(object): """ Class representing a build system. Derived classes implement the methods below. Objects of these classes aren't instantiated, instead the class (type object) is used. """ NAME = "undefined" SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)') @classmethod def build_prepare(cls, build_item): app_path = build_item.app_dir work_path = build_item.work_dir or app_path if not build_item.build_dir: build_path = os.path.join(work_path, "build") elif os.path.isabs(build_item.build_dir): build_path = build_item.build_dir else: build_path = os.path.join(work_path, build_item.build_dir) if work_path != app_path: if os.path.exists(work_path): logging.debug("Work directory {} exists, removing".format(work_path)) if not build_item.dry_run: shutil.rmtree(work_path) logging.debug("Copying app from {} to {}".format(app_path, work_path)) if not build_item.dry_run: shutil.copytree(app_path, work_path) if os.path.exists(build_path): logging.debug("Build directory {} exists, removing".format(build_path)) if not build_item.dry_run: shutil.rmtree(build_path) if not build_item.dry_run: os.makedirs(build_path) # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file. # # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS # CMake variable. However here we do this manually to perform environment variable expansion in the # sdkconfig files. sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target] if build_item.sdkconfig_path: sdkconfig_defaults_list.append(build_item.sdkconfig_path) sdkconfig_file = os.path.join(work_path, "sdkconfig") if os.path.exists(sdkconfig_file): logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file)) if not build_item.dry_run: os.unlink(sdkconfig_file) logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file)) extra_cmakecache_items = {} if not build_item.dry_run: with open(sdkconfig_file, "w") as f_out: for sdkconfig_name in sdkconfig_defaults_list: sdkconfig_path = os.path.join(work_path, sdkconfig_name) if not sdkconfig_path or not os.path.exists(sdkconfig_path): continue logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) with open(sdkconfig_path, "r") as f_in: for line in f_in: if not line.endswith("\n"): line += "\n" if cls.NAME == 'cmake': m = SDKCONFIG_LINE_REGEX.match(line) key = m.group(1) if m else None if key in SDKCONFIG_TEST_OPTS: extra_cmakecache_items[key] = m.group(2) continue if key in SDKCONFIG_IGNORE_OPTS: continue f_out.write(os.path.expandvars(line)) else: for sdkconfig_name in sdkconfig_defaults_list: sdkconfig_path = os.path.join(app_path, sdkconfig_name) if not sdkconfig_path: continue logging.debug("Considering sdkconfig {}".format(sdkconfig_path)) if not os.path.exists(sdkconfig_path): continue logging.debug("Appending {} to sdkconfig".format(sdkconfig_name)) # The preparation of build is finished. Implement the build part in sub classes. if cls.NAME == 'cmake': return build_path, work_path, extra_cmakecache_items else: return build_path, work_path @staticmethod @abstractmethod def build(build_item): pass @staticmethod @abstractmethod def is_app(path): pass @staticmethod def _read_readme(app_path): # Markdown supported targets should be: # e.g. | Supported Targets | ESP32 | # | ----------------- | ----- | # reStructuredText supported targets should be: # e.g. ================= ===== # Supported Targets ESP32 # ================= ===== def get_md_or_rst(app_path): readme_path = os.path.join(app_path, 'README.md') if not os.path.exists(readme_path): readme_path = os.path.join(app_path, 'README.rst') if not os.path.exists(readme_path): return None return readme_path readme_path = get_md_or_rst(app_path) # Handle sub apps situation, e.g. master-slave if not readme_path: readme_path = get_md_or_rst(os.path.dirname(app_path)) if not readme_path: return None with open(readme_path, "r", encoding='utf8') as readme_file: return readme_file.read() @staticmethod def supported_targets(app_path): formal_to_usual = { 'ESP32': 'esp32', 'ESP32-S2': 'esp32s2', } readme_file_content = BuildSystem._read_readme(app_path) if not readme_file_content: return None match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) if not match: return None if len(match) > 1: raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) support_str = match[0].strip() targets = [] for part in support_str.split('|'): for inner in part.split(' '): inner = inner.strip() if not inner: continue elif inner in formal_to_usual: targets.append(formal_to_usual[inner]) else: raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" .format(inner, app_path, ', '.join(formal_to_usual.keys()))) return targets class BuildError(RuntimeError): pass def setup_logging(args): """ Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument. :param args: namespace obtained from argparse """ if not args.verbose: log_level = logging.WARNING elif args.verbose == 1: log_level = logging.INFO else: log_level = logging.DEBUG logging.basicConfig( format="%(levelname)s: %(message)s", stream=args.log_file or sys.stderr, level=log_level, )