diff --git a/tools/find_apps.py b/tools/find_apps.py index 35e0583de..0d8030e66 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -5,13 +5,13 @@ # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. import argparse +import json import os import sys import re import glob import logging import typing -from typing import Optional from find_build_apps import ( BUILD_SYSTEMS, @@ -47,9 +47,9 @@ def dict_from_sdkconfig(path): # Main logic: enumerating apps and builds -def find_builds_for_app( - app_path, work_dir, build_dir, build_log, target_arg, build_system, - config_rules): # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem] +def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, + build_system, config_rules, build_or_not=True, preserve_artifacts=True): + # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem] """ Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects :param app_path: app directory (can be / usually will be a relative path) @@ -62,6 +62,8 @@ def find_builds_for_app( a different CONFIG_IDF_TARGET value. :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary :param config_rules: mapping of sdkconfig file name patterns to configuration names + :param build_or_not: determine if it will build in build_apps.py + :param preserve_artifacts: determine if the built binary will be uploaded as artifacts. :return: list of BuildItems representing build configuration of the app """ build_items = [] # type: typing.List[BuildItem] @@ -106,6 +108,8 @@ def find_builds_for_app( sdkconfig_path, config_name, build_system, + build_or_not, + preserve_artifacts, )) if not build_items: @@ -120,14 +124,16 @@ def find_builds_for_app( None, default_config_name, build_system, + build_or_not, + preserve_artifacts, ) ] return build_items -def find_apps(build_system_class, path, recursive, exclude_list, target, - build_apps_list_file=None): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str, Optional[str]) -> typing.List[str] +def find_apps(build_system_class, path, recursive, exclude_list, target): + # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str] """ Find app directories in path (possibly recursively), which contain apps for the given build system, compatible with the given target. @@ -151,18 +157,6 @@ def find_apps(build_system_class, path, recursive, exclude_list, target, return [] return [path] - # if this argument is empty, treat it as build_all. - if not build_apps_list_file: - build_all = True - else: - # if this argument is not empty but the file doesn't exists, treat as no supported apps. - if not os.path.exists(build_apps_list_file): - return [] - else: - build_all = False - with open(build_apps_list_file) as fr: - apps_need_be_built = set([line.strip() for line in fr.readlines() if line]) - # The remaining part is for recursive == True apps_found = [] # type: typing.List[str] for root, dirs, _ in os.walk(path, topdown=True): @@ -172,11 +166,6 @@ def find_apps(build_system_class, path, recursive, exclude_list, target, del dirs[:] continue - if not build_all: - if os.path.abspath(root) not in apps_need_be_built: - logging.debug("Skipping, app not listed in {}".format(build_apps_list_file)) - continue - if build_system_class.is_app(root): logging.debug("Found {} app in {}".format(build_system_name, root)) # Don't recurse into app subdirectories @@ -258,7 +247,7 @@ def main(): '--scan-tests-json', default=None, help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n" - "If it's None or the file does not exist, will build all apps and upload all artifacts." + "If the file does not exist, will build all apps and upload all artifacts." ) parser.add_argument("paths", nargs="+", help="One or more app paths.") args = parser.parse_args() @@ -276,30 +265,38 @@ def main(): logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET - # Prepare the list of app paths - app_paths = [] # type: typing.List[str] - for path in args.paths: - app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target, args.build_apps_list_file) + # Prepare the list of app paths, try to read from the scan_tests result. + # If the file exists, then follow the file's app_dir and build/archifacts behavior, won't do find_apps() again. + # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. + if args.scan_tests_json and os.path.exists(args.scan_tests_json): + apps = [json.loads(line) for line in open(args.scan_tests_json)] + else: + app_dirs = [] + for path in args.paths: + app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) + apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs] - build_items = [] # type: typing.List[BuildItem] - if not app_paths: - logging.warning("No {} apps found, skipping...".format(build_system_class.NAME)) - sys.exit(0) + if not apps: + logging.critical("No {} apps found".format(build_system_class.NAME)) + raise SystemExit(1) + logging.info("Found {} apps".format(len(apps))) - logging.info("Found {} apps".format(len(app_paths))) - app_paths = sorted(app_paths) + apps.sort(key=lambda x: x['app_dir']) # Find compatible configurations of each app, collect them as BuildItems + build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) - for app_path in app_paths: + for app in apps: build_items += find_builds_for_app( - app_path, + app['app_dir'], args.work_dir, args.build_dir, args.build_log, args.target, args.build_system, config_rules, + app['build'], + app['preserve'], ) logging.info("Found {} builds".format(len(build_items))) diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 47c4a9d33..0dfb1ba6e 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -71,6 +71,8 @@ class BuildItem(object): sdkconfig_path, config_name, build_system, + build_or_not, + preserve_artifacts, ): # These internal variables store the paths with environment variables and placeholders; # Public properties with similar names use the _expand method to get the actual paths. @@ -84,6 +86,9 @@ class BuildItem(object): self.target = target self.build_system = build_system + self.build = build_or_not + self.preserve = preserve_artifacts + self._app_name = os.path.basename(os.path.normpath(app_path)) # Some miscellaneous build properties which are set later, at the build stage @@ -155,6 +160,8 @@ class BuildItem(object): "config": self.config_name, "target": self.target, "verbose": self.verbose, + "build": self.build, + "preserve": self.preserve, }) @staticmethod @@ -172,6 +179,8 @@ class BuildItem(object): config_name=d["config"], target=d["target"], build_system=d["build_system"], + build_or_not=d["build"], + preserve_artifacts=d["preserve"] ) result.verbose = d["verbose"] return result