OVMS3-idf/tools/find_build_apps/common.py

# coding=utf-8
import re
import shutil
import sys
import os
from abc import abstractmethod
from collections import namedtuple
import logging
import json
import typing
from io import open

DEFAULT_TARGET = "esp32"

TARGET_PLACEHOLDER = "@t"
WILDCARD_PLACEHOLDER = "@w"
NAME_PLACEHOLDER = "@n"
FULL_NAME_PLACEHOLDER = "@f"
INDEX_PLACEHOLDER = "@i"

SDKCONFIG_LINE_REGEX = re.compile(r"^([^=]+)=\"?([^\"\n]*)\"?\n*$")

# If these keys are present in sdkconfig.defaults, they will be extracted and passed to CMake
SDKCONFIG_TEST_OPTS = [
    "EXCLUDE_COMPONENTS",
    "TEST_EXCLUDE_COMPONENTS",
    "TEST_COMPONENTS",
]

# These keys in sdkconfig.defaults are not propagated to the final sdkconfig file:
SDKCONFIG_IGNORE_OPTS = [
    "TEST_GROUPS"
]

# ConfigRule represents one --config argument of find_apps.py.
# file_name is the name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
# file_name can also be empty to indicate that the default configuration of the app should be used.
# config_name is the name of the corresponding build configuration, or None if the value of wildcard is to be used.
# For example:
#   filename='', config_name='default' — represents the default app configuration, and gives it a name 'default'
#   filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard value
ConfigRule = namedtuple("ConfigRule", ["file_name", "config_name"])


def config_rules_from_str(rule_strings):  # type: (typing.List[str]) -> typing.List[ConfigRule]
    """
    Helper function to convert strings like 'file_name=config_name' into ConfigRule objects
    :param rule_strings: list of rules as strings
    :return: list of ConfigRules
    """
    rules = []  # type: typing.List[ConfigRule]
    for rule_str in rule_strings:
        items = rule_str.split("=", 2)
        rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else None))
    return rules


class BuildItem(object):
    """
    Instance of this class represents one build of an application.
    The parameters which distinguish the build are passed to the constructor.
    """

    def __init__(
            self,
            app_path,
            work_dir,
            build_path,
            build_log_path,
            target,
            sdkconfig_path,
            config_name,
            build_system,
    ):
        # These internal variables store the paths with environment variables and placeholders;
        # Public properties with similar names use the _expand method to get the actual paths.
        self._app_dir = app_path
        self._work_dir = work_dir
        self._build_dir = build_path
        self._build_log_path = build_log_path

        self.sdkconfig_path = sdkconfig_path
        self.config_name = config_name
        self.target = target
        self.build_system = build_system

        self._app_name = os.path.basename(os.path.normpath(app_path))

        # Some miscellaneous build properties which are set later, at the build stage
        self.index = None
        self.verbose = False
        self.dry_run = False
        self.keep_going = False

    @property
    def app_dir(self):
        """
        :return: directory of the app
        """
        return self._expand(self._app_dir)

    @property
    def work_dir(self):
        """
        :return: directory where the app should be copied to, prior to the build. Can be None, which means that the app
                 directory should be used.
        """
        return self._expand(self._work_dir)

    @property
    def build_dir(self):
        """
        :return: build directory, either relative to the work directory (if relative path is used) or absolute path.
        """
        return self._expand(self._build_dir)

    @property
    def build_log_path(self):
        """
        :return: path of the build log file
        """
        return self._expand(self._build_log_path)

    def __repr__(self):
        return "Build app {} for target {}, sdkconfig {} in {}".format(
            self.app_dir,
            self.target,
            self.sdkconfig_path or "(default)",
            self.build_dir,
        )

    def to_json(self):  # type: () -> str
        """
        :return: JSON string representing this object
        """
        return self._to_json(self._app_dir, self._work_dir, self._build_dir, self._build_log_path)

    def to_json_expanded(self):  # type: () -> str
        """
        :return: JSON string representing this object, with all placeholders in paths expanded
        """
        return self._to_json(self.app_dir, self.work_dir, self.build_dir, self.build_log_path)

    def _to_json(self, app_dir, work_dir, build_dir, build_log_path):  # type: (str, str, str, str) -> str
        """
        Internal function, called by to_json and to_json_expanded
        """
        return json.dumps({
            "build_system": self.build_system,
            "app_dir": app_dir,
            "work_dir": work_dir,
            "build_dir": build_dir,
            "build_log_path": build_log_path,
            "sdkconfig": self.sdkconfig_path,
            "config": self.config_name,
            "target": self.target,
            "verbose": self.verbose,
        })

    @staticmethod
    def from_json(json_str):  # type: (typing.Text) -> BuildItem
        """
        :return: Get the BuildItem from a JSON string
        """
        d = json.loads(str(json_str))
        result = BuildItem(
            app_path=d["app_dir"],
            work_dir=d["work_dir"],
            build_path=d["build_dir"],
            build_log_path=d["build_log_path"],
            sdkconfig_path=d["sdkconfig"],
            config_name=d["config"],
            target=d["target"],
            build_system=d["build_system"],
        )
        result.verbose = d["verbose"]
        return result

    def _expand(self, path):  # type: (str) -> str
        """
        Internal method, expands any of the placeholders in {app,work,build} paths.
        """
        if not path:
            return path

        if self.index is not None:
            path = path.replace(INDEX_PLACEHOLDER, str(self.index))
        path = path.replace(TARGET_PLACEHOLDER, self.target)
        path = path.replace(NAME_PLACEHOLDER, self._app_name)
        if (FULL_NAME_PLACEHOLDER in path):  # to avoid recursion to the call to app_dir in the next line:
            path = path.replace(FULL_NAME_PLACEHOLDER, self.app_dir.replace(os.path.sep, "_"))
        wildcard_pos = path.find(WILDCARD_PLACEHOLDER)
        if wildcard_pos != -1:
            if self.config_name:
                # if config name is defined, put it in place of the placeholder
                path = path.replace(WILDCARD_PLACEHOLDER, self.config_name)
            else:
                # otherwise, remove the placeholder and one character on the left
                # (which is usually an underscore, dash, or other delimiter)
                left_of_wildcard = max(0, wildcard_pos - 1)
                right_of_wildcard = wildcard_pos + len(WILDCARD_PLACEHOLDER)
                path = path[0:left_of_wildcard] + path[right_of_wildcard:]
        path = os.path.expandvars(path)
        return path


class BuildSystem(object):
    """
    Class representing a build system.
    Derived classes implement the methods below.
    Objects of these classes aren't instantiated, instead the class (type object) is used.
    """
    NAME = "undefined"
    SUPPORTED_TARGETS_REGEX = re.compile(r'Supported [Tt]argets((?:[\s|]+(?:ESP[0-9A-Z\-]+))+)')

    @classmethod
    def build_prepare(cls, build_item):
        app_path = build_item.app_dir
        work_path = build_item.work_dir or app_path
        if not build_item.build_dir:
            build_path = os.path.join(work_path, "build")
        elif os.path.isabs(build_item.build_dir):
            build_path = build_item.build_dir
        else:
            build_path = os.path.join(work_path, build_item.build_dir)

        if work_path != app_path:
            if os.path.exists(work_path):
                logging.debug("Work directory {} exists, removing".format(work_path))
                if not build_item.dry_run:
                    shutil.rmtree(work_path)
            logging.debug("Copying app from {} to {}".format(app_path, work_path))
            if not build_item.dry_run:
                shutil.copytree(app_path, work_path)

        if os.path.exists(build_path):
            logging.debug("Build directory {} exists, removing".format(build_path))
            if not build_item.dry_run:
                shutil.rmtree(build_path)

        if not build_item.dry_run:
            os.makedirs(build_path)

        # Prepare the sdkconfig file, from the contents of sdkconfig.defaults (if exists) and the contents of
        # build_info.sdkconfig_path, i.e. the config-specific sdkconfig file.
        #
        # Note: the build system supports taking multiple sdkconfig.defaults files via SDKCONFIG_DEFAULTS
        # CMake variable. However here we do this manually to perform environment variable expansion in the
        # sdkconfig files.
        sdkconfig_defaults_list = ["sdkconfig.defaults", "sdkconfig.defaults." + build_item.target]
        if build_item.sdkconfig_path:
            sdkconfig_defaults_list.append(build_item.sdkconfig_path)

        sdkconfig_file = os.path.join(work_path, "sdkconfig")
        if os.path.exists(sdkconfig_file):
            logging.debug("Removing sdkconfig file: {}".format(sdkconfig_file))
            if not build_item.dry_run:
                os.unlink(sdkconfig_file)

        logging.debug("Creating sdkconfig file: {}".format(sdkconfig_file))
        extra_cmakecache_items = {}
        if not build_item.dry_run:
            with open(sdkconfig_file, "w") as f_out:
                for sdkconfig_name in sdkconfig_defaults_list:
                    sdkconfig_path = os.path.join(work_path, sdkconfig_name)
                    if not sdkconfig_path or not os.path.exists(sdkconfig_path):
                        continue
                    logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))
                    with open(sdkconfig_path, "r") as f_in:
                        for line in f_in:
                            if not line.endswith("\n"):
                                line += "\n"
                            if cls.NAME == 'cmake':
                                m = SDKCONFIG_LINE_REGEX.match(line)
                                key = m.group(1) if m else None
                                if key in SDKCONFIG_TEST_OPTS:
                                    extra_cmakecache_items[key] = m.group(2)
                                    continue
                                if key in SDKCONFIG_IGNORE_OPTS:
                                    continue
                            f_out.write(os.path.expandvars(line))
        else:
            for sdkconfig_name in sdkconfig_defaults_list:
                sdkconfig_path = os.path.join(app_path, sdkconfig_name)
                if not sdkconfig_path:
                    continue
                logging.debug("Considering sdkconfig {}".format(sdkconfig_path))
                if not os.path.exists(sdkconfig_path):
                    continue
                logging.debug("Appending {} to sdkconfig".format(sdkconfig_name))

        # The preparation of build is finished. Implement the build part in sub classes.
        if cls.NAME == 'cmake':
            return build_path, work_path, extra_cmakecache_items
        else:
            return build_path, work_path

    @staticmethod
    @abstractmethod
    def build(build_item):
        pass

    @staticmethod
    @abstractmethod
    def is_app(path):
        pass

    @staticmethod
    def _read_readme(app_path):
        # Markdown supported targets should be:
        # e.g. | Supported Targets | ESP32 |
        #      | ----------------- | ----- |
        # reStructuredText supported targets should be:
        # e.g. ================= =====
        #      Supported Targets ESP32
        #      ================= =====
        def get_md_or_rst(app_path):
            readme_path = os.path.join(app_path, 'README.md')
            if not os.path.exists(readme_path):
                readme_path = os.path.join(app_path, 'README.rst')
                if not os.path.exists(readme_path):
                    return None
            return readme_path

        readme_path = get_md_or_rst(app_path)
        # Handle sub apps situation, e.g. master-slave
        if not readme_path:
            readme_path = get_md_or_rst(os.path.dirname(app_path))
        if not readme_path:
            return None
        with open(readme_path, "r", encoding='utf8') as readme_file:
            return readme_file.read()

    @staticmethod
    def supported_targets(app_path):
        formal_to_usual = {
            'ESP32': 'esp32',
            'ESP32-S2': 'esp32s2',
        }

        readme_file_content = BuildSystem._read_readme(app_path)
        if not readme_file_content:
            return None
        match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content)
        if not match:
            return None
        if len(match) > 1:
            raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path))
        support_str = match[0].strip()

        targets = []
        for part in support_str.split('|'):
            for inner in part.split(' '):
                inner = inner.strip()
                if not inner:
                    continue
                elif inner in formal_to_usual:
                    targets.append(formal_to_usual[inner])
                else:
                    raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'"
                                              .format(inner, app_path, ', '.join(formal_to_usual.keys())))
        return targets


class BuildError(RuntimeError):
    pass


def setup_logging(args):
    """
    Configure logging module according to the number of '--verbose'/'-v' arguments and the --log-file argument.
    :param args: namespace obtained from argparse
    """
    if not args.verbose:
        log_level = logging.WARNING
    elif args.verbose == 1:
        log_level = logging.INFO
    else:
        log_level = logging.DEBUG

    logging.basicConfig(
        format="%(levelname)s: %(message)s",
        stream=args.log_file or sys.stderr,
        level=log_level,
    )