From b26d42afe365e24e2d76fc2b5a3ede71cae0eba9 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 20 Apr 2020 14:30:31 +0800 Subject: [PATCH 01/15] Now scan_tests will generate 'scan_.json` also updated CI yaml and shell scripts --- tools/build_apps.py | 9 +- tools/ci/build_examples.sh | 2 + tools/ci/build_test_apps.sh | 2 + tools/ci/check_build_warnings.py | 5 +- tools/ci/config/build.yml | 11 +- tools/ci/config/pre_check.yml | 27 ++++ tools/ci/python_packages/gitlab_api.py | 6 +- .../tiny_test_fw/Utility/CIAssignTest.py | 7 +- .../tiny_test_fw/Utility/SearchCases.py | 1 + .../ttfw_idf/CIAssignExampleTest.py | 2 +- .../ttfw_idf/CIAssignUnitTest.py | 2 +- .../python_packages/ttfw_idf/CIScanTests.py | 134 ++++++++++++++++++ tools/find_apps.py | 59 ++++++-- tools/find_build_apps/__init__.py | 2 + tools/find_build_apps/common.py | 6 + 15 files changed, 244 insertions(+), 31 deletions(-) create mode 100644 tools/ci/python_packages/ttfw_idf/CIScanTests.py diff --git a/tools/build_apps.py b/tools/build_apps.py index 6a49ab06a..412274c61 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -7,7 +7,7 @@ import argparse import sys import logging -from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS +from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS, safe_exit_if_file_is_empty def main(): @@ -33,8 +33,8 @@ def main(): default=1, type=int, help="Number of parallel build jobs. Note that this script doesn't start the jobs, " + - "it needs to be executed multiple times with same value of --parallel-count and " + - "different values of --parallel-index.", + "it needs to be executed multiple times with same value of --parallel-count and " + + "different values of --parallel-index.", ) parser.add_argument( "--parallel-index", @@ -71,11 +71,10 @@ def main(): help="Name of the file to read the list of builds from. If not specified, read from stdin.", ) args = parser.parse_args() - setup_logging(args) + safe_exit_if_file_is_empty(args.build_list.name) build_items = [BuildItem.from_json(line) for line in args.build_list] - if not build_items: logging.error("Empty build list!") raise SystemExit(1) diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 7308861f7..4a5915d10 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -31,6 +31,7 @@ die() { [ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set" [ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set" [ -z ${EXAMPLE_TEST_BUILD_SYSTEM} ] && die "EXAMPLE_TEST_BUILD_SYSTEM is not set" +[ -z ${SCAN_EXAMPLE_TEST_JSON} ] && die "SCAN_EXAMPLE_TEST_JSON is not set" [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} [ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH} @@ -85,6 +86,7 @@ ${IDF_PATH}/tools/find_apps.py examples \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ + --scan-tests-json ${SCAN_EXAMPLE_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index cce9c2dfe..627d1bb78 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -29,6 +29,7 @@ die() { [ -z ${LOG_PATH} ] && die "LOG_PATH is not set" [ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set" [ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set" +[ -z ${SCAN_CUSTOM_TEST_JSON} ] && die "SCAN_CUSTOM_TEST_JSON is not set" [ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} [ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH} @@ -74,6 +75,7 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ + --scan-tests-json ${SCAN_CUSTOM_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index e64cc4066..89c59cf23 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -13,10 +13,10 @@ import logging import re try: - from find_build_apps import BuildItem, setup_logging + from find_build_apps import BuildItem, setup_logging, safe_exit_if_file_is_empty except ImportError: sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - from find_build_apps import BuildItem, setup_logging + from find_build_apps import BuildItem, setup_logging, safe_exit_if_file_is_empty WARNING_REGEX = re.compile(r"(?:error|warning)[^\w]", re.MULTILINE | re.IGNORECASE) @@ -69,6 +69,7 @@ def main(): args = parser.parse_args() setup_logging(args) + safe_exit_if_file_is_empty(args.build_list.name) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 7133ac2a4..6b10837e0 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -126,6 +126,8 @@ build_examples_make: # same as above, but for CMake .build_examples_cmake: &build_examples_cmake extends: .build_examples_template + dependencies: + - scan_tests artifacts: paths: - build_examples/list.json @@ -142,6 +144,8 @@ build_examples_make: LOG_PATH: "${CI_PROJECT_DIR}/log_examples" BUILD_PATH: "${CI_PROJECT_DIR}/build_examples" EXAMPLE_TEST_BUILD_SYSTEM: "cmake" + SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json + build_examples_cmake_esp32: extends: .build_examples_cmake @@ -156,6 +160,8 @@ build_examples_cmake_esp32s2: .build_test_apps: &build_test_apps extends: .build_template stage: build + dependencies: + - scan_tests artifacts: when: always paths: @@ -171,8 +177,9 @@ build_examples_cmake_esp32s2: - $LOG_PATH expire_in: 3 days variables: - LOG_PATH: "$CI_PROJECT_DIR/log_test_apps" - BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps" + LOG_PATH: "${CI_PROJECT_DIR}/log_test_apps" + BUILD_PATH: "${CI_PROJECT_DIR}/build_test_apps" + SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}.json only: variables: - $BOT_TRIGGER_WITH_LABEL == null diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 00eb4922f..a7e971df7 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -190,3 +190,30 @@ check_public_headers: script: - python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf- +.scan_build_tests: + stage: pre_check + image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG + tags: + - assign_test + variables: + CI_SCAN_TESTS_PY: ${CI_PROJECT_DIR}/tools/ci/python_packages/ttfw_idf/CIScanTests.py + TEST_CONFIG_FILE: ${CI_PROJECT_DIR}/tools/ci/config/target-test.yml + +scan_tests: + extends: .scan_build_tests + only: + variables: + - $BOT_LABEL_EXAMPLE_TEST + - $BOT_LABEL_CUSTOM_TEST + artifacts: + paths: + - $EXAMPLE_TEST_OUTPUT_DIR + - $TEST_APPS_OUTPUT_DIR + variables: + EXAMPLE_TEST_DIR: ${CI_PROJECT_DIR}/examples + EXAMPLE_TEST_OUTPUT_DIR: ${CI_PROJECT_DIR}/examples/test_configs + TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps + TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs + script: + - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 561ffb863..3d49dbb40 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -124,7 +124,7 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status="success"): + def find_job_id(self, job_name, pipeline_id=None, job_status="success", suffix=None): """ Get Job ID from job name of specific pipeline @@ -132,6 +132,7 @@ class Gitlab(object): :param pipeline_id: If None, will get pipeline id from CI pre-defined variable. :param job_status: status of job. One pipeline could have multiple jobs with same name after retry. job_status is used to filter these jobs. + :param suffix: suffix of the job. e.g. 'limit' for build only needed apps. :return: a list of job IDs (parallel job will generate multiple jobs) """ job_id_list = [] @@ -144,6 +145,9 @@ class Gitlab(object): if match: if match.group(1) == job_name and job.status == job_status: job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) + elif suffix: + if match.group(1) == "{}_{}".format(job_name, suffix) and job.status == job_status: + job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list @retry_download diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py index 6277f0781..240c7fbc4 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py @@ -189,16 +189,15 @@ class AssignTest(object): job_list.sort(key=lambda x: x["name"]) return job_list - def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None): + def search_cases(self, case_filter=None): """ - :param test_case_path: path contains test case folder :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param. :return: filtered test case list """ _case_filter = self.DEFAULT_FILTER.copy() if case_filter: _case_filter.update(case_filter) - test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern) + test_methods = SearchCases.Search.search_test_cases(self.test_case_path, self.test_case_file_pattern) return CaseConfig.filter_test_cases(test_methods, _case_filter) def _group_cases(self): @@ -287,7 +286,7 @@ class AssignTest(object): failed_to_assign = [] assigned_groups = [] case_filter = self._apply_bot_filter() - self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern) + self.test_cases = self.search_cases(case_filter) self._apply_bot_test_count() test_groups = self._group_cases() diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index 699bb57ec..c36fec8ad 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -50,6 +50,7 @@ class Search(object): for i, test_function in enumerate(test_functions_out): print("\t{}. ".format(i + 1) + test_function.case_info["name"]) + test_function.case_info['app_dir'] = os.path.dirname(file_name) return test_functions_out @classmethod diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 3acdd78d0..99af101c0 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -65,7 +65,7 @@ def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=Exa return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) for build_job_name in case_group.BUILD_JOB_NAMES: - job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id, suffix='limit') for job_info in job_info_list: raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] build_info_list = [json.loads(line) for line in raw_data.splitlines()] diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index bde9a102d..ad71d0dd5 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): def __init__(self, test_case_path, ci_config_file): CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group) - def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None): + def search_cases(self, case_filter=None): """ For unit test case, we don't search for test functions. The unit test cases is stored in a yaml file which is created in job build-idf-test. diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py new file mode 100644 index 000000000..c56664f6b --- /dev/null +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -0,0 +1,134 @@ +import argparse +import json +import os +import re +from collections import defaultdict + +from find_apps import find_apps +from find_build_apps import CMakeBuildSystem +from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup + +VALID_TARGETS = [ + 'esp32', + 'esp32s2', +] + +SPECIAL_REFS = [ + 'master', + re.compile(r'^release/v'), + re.compile(r'^v\d+\.\d+'), +] + + +def _judge_build_all(): + ref = os.getenv('CI_COMMIT_REF_NAME') + pipeline_src = os.getenv('CI_PIPELINE_SOURCE') + if not ref or not pipeline_src: + return False + + # scheduled pipeline will build all + if pipeline_src == 'schedule': + return True + + # master, release/v..., v1.2.3..., and will build all + for special_ref in SPECIAL_REFS: + if isinstance(special_ref, re._pattern_type): + if special_ref.match(ref): + return True + else: + if ref == special_ref: + return True + return False + + +def main(): + parser = argparse.ArgumentParser(description='Scan the required build tests') + actions = parser.add_subparsers(dest='action') + + common = argparse.ArgumentParser(add_help=False) + common.add_argument('paths', type=str, nargs='+', + help="One or more app paths") + common.add_argument('-c', '--ci_config_file', type=str, required=True, + help="gitlab ci config file") + common.add_argument('-o', '--output_path', type=str, required=True, + help="output path of the scan result") + common.add_argument('-p', '--preserve-all', action="store_true", + help='add this flag to preserve artifacts for all apps') + + actions.add_parser('example_test', parents=[common]) + actions.add_parser('test_apps', parents=[common]) + # actions.add_parser('unit_test', parents=[common]) + + args = parser.parse_args() + + test_cases = [] + for path in args.paths: + if args.action == 'example_test': + assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup) + elif args.action == 'test_apps': + CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') + assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup) + # elif args.action == 'unit_test': + # assign = UnitTestAssignTest(args.test_case, args.ci_config_file) + else: + raise SystemExit(1) # which is impossible + + test_cases.extend(assign.search_cases()) + + try: + os.makedirs(args.output_path) + except Exception: + pass + + ''' + { + : { + 'test_case_apps': [], + 'standalone_apps': [], + }, + ... + } + ''' + scan_info_dict = defaultdict(dict) + # store the test cases dir, exclude these folders when scan for standalone apps + exclude_apps = [] + + # TODO change this chip to target after feat/add_multi_target_for_example_test is merged + for target in VALID_TARGETS: + target_dict = scan_info_dict[target] + test_case_apps = target_dict['test_case_apps'] = set() + for case in test_cases: + app_dir = case.case_info['app_dir'] + app_target = case.case_info['chip'] + if app_target.lower() != target.lower(): + continue + test_case_apps.update(find_apps(CMakeBuildSystem, app_dir, True, [], target.lower())) + exclude_apps.append(app_dir) + + for target in VALID_TARGETS: + target_dict = scan_info_dict[target] + standalone_apps = target_dict['standalone_apps'] = set() + for path in args.paths: + standalone_apps.update(find_apps(CMakeBuildSystem, path, True, exclude_apps, target.lower())) + + build_all = _judge_build_all() + for target in VALID_TARGETS: + apps = [] + for app_dir in scan_info_dict[target]['test_case_apps']: + apps.append({ + 'app_dir': app_dir, + 'build': True, + 'preserve': True, + }) + for app_dir in scan_info_dict[target]['standalone_apps']: + apps.append({ + 'app_dir': app_dir, + 'build': build_all, + 'preserve': args.preserve_all and build_all, # you can't preserve the artifacts if you don't build them right? + }) + with open(os.path.join(args.output_path, 'scan_{}.json'.format(target.lower())), 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps]) + + +if __name__ == '__main__': + main() diff --git a/tools/find_apps.py b/tools/find_apps.py index 478ed5f42..35e0583de 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -11,6 +11,8 @@ import re import glob import logging import typing +from typing import Optional + from find_build_apps import ( BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE, @@ -22,8 +24,8 @@ from find_build_apps import ( DEFAULT_TARGET, ) -# Helper functions +# Helper functions def dict_from_sdkconfig(path): """ @@ -124,8 +126,8 @@ def find_builds_for_app( return build_items -def find_apps(build_system_class, path, recursive, exclude_list, - target): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str] +def find_apps(build_system_class, path, recursive, exclude_list, target, + build_apps_list_file=None): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str, Optional[str]) -> typing.List[str] """ Find app directories in path (possibly recursively), which contain apps for the given build system, compatible with the given target. @@ -134,6 +136,8 @@ def find_apps(build_system_class, path, recursive, exclude_list, :param recursive: whether to recursively descend into nested directories if no app is found :param exclude_list: list of paths to be excluded from the recursive search :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped. + :param build_apps_list_file: List of apps need to be built, apps not on this list will be skipped. + None or empty file to build all apps. :return: list of paths of the apps found """ build_system_name = build_system_class.NAME @@ -147,6 +151,18 @@ def find_apps(build_system_class, path, recursive, exclude_list, return [] return [path] + # if this argument is empty, treat it as build_all. + if not build_apps_list_file: + build_all = True + else: + # if this argument is not empty but the file doesn't exists, treat as no supported apps. + if not os.path.exists(build_apps_list_file): + return [] + else: + build_all = False + with open(build_apps_list_file) as fr: + apps_need_be_built = set([line.strip() for line in fr.readlines() if line]) + # The remaining part is for recursive == True apps_found = [] # type: typing.List[str] for root, dirs, _ in os.walk(path, topdown=True): @@ -156,6 +172,11 @@ def find_apps(build_system_class, path, recursive, exclude_list, del dirs[:] continue + if not build_all: + if os.path.abspath(root) not in apps_need_be_built: + logging.debug("Skipping, app not listed in {}".format(build_apps_list_file)) + continue + if build_system_class.is_app(root): logging.debug("Found {} app in {}".format(build_system_name, root)) # Don't recurse into app subdirectories @@ -193,22 +214,22 @@ def main(): parser.add_argument( "--work-dir", help="If set, the app is first copied into the specified directory, and then built." + - "If not set, the work directory is the directory of the app.", + "If not set, the work directory is the directory of the app.", ) parser.add_argument( "--config", action="append", help="Adds configurations (sdkconfig file names) to build. This can either be " + - "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + - "relative to the project directory, to be used. Optional NAME can be specified, " + - "which can be used as a name of this configuration. FILEPATTERN is the name of " + - "the sdkconfig file, relative to the project directory, with at most one wildcard. " + - "The part captured by the wildcard is used as the name of the configuration.", + "FILENAME[=NAME] or FILEPATTERN. FILENAME is the name of the sdkconfig file, " + + "relative to the project directory, to be used. Optional NAME can be specified, " + + "which can be used as a name of this configuration. FILEPATTERN is the name of " + + "the sdkconfig file, relative to the project directory, with at most one wildcard. " + + "The part captured by the wildcard is used as the name of the configuration.", ) parser.add_argument( "--build-dir", help="If set, specifies the build directory name. Can expand placeholders. Can be either a " + - "name relative to the work directory, or an absolute path.", + "name relative to the work directory, or an absolute path.", ) parser.add_argument( "--build-log", @@ -232,6 +253,13 @@ def main(): type=argparse.FileType("w"), help="Output the list of builds to the specified file", ) + parser.add_argument( + '-s', + '--scan-tests-json', + default=None, + help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n" + "If it's None or the file does not exist, will build all apps and upload all artifacts." + ) parser.add_argument("paths", nargs="+", help="One or more app paths.") args = parser.parse_args() setup_logging(args) @@ -251,17 +279,17 @@ def main(): # Prepare the list of app paths app_paths = [] # type: typing.List[str] for path in args.paths: - app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) + app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target, args.build_apps_list_file) + build_items = [] # type: typing.List[BuildItem] if not app_paths: - logging.critical("No {} apps found".format(build_system_class.NAME)) - raise SystemExit(1) - logging.info("Found {} apps".format(len(app_paths))) + logging.warning("No {} apps found, skipping...".format(build_system_class.NAME)) + sys.exit(0) + logging.info("Found {} apps".format(len(app_paths))) app_paths = sorted(app_paths) # Find compatible configurations of each app, collect them as BuildItems - build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app_path in app_paths: build_items += find_builds_for_app( @@ -278,6 +306,7 @@ def main(): # Write out the BuildItems. Only JSON supported now (will add YAML later). if args.format != "json": raise NotImplementedError() + out = args.output or sys.stdout out.writelines([item.to_json() + "\n" for item in build_items]) diff --git a/tools/find_build_apps/__init__.py b/tools/find_build_apps/__init__.py index c87ceebda..891721335 100644 --- a/tools/find_build_apps/__init__.py +++ b/tools/find_build_apps/__init__.py @@ -6,6 +6,7 @@ from .common import ( config_rules_from_str, setup_logging, DEFAULT_TARGET, + safe_exit_if_file_is_empty, ) from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE @@ -28,4 +29,5 @@ __all__ = [ "MakeBuildSystem", "BUILD_SYSTEM_MAKE", "BUILD_SYSTEMS", + "safe_exit_if_file_is_empty", ] diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 81b61859a..47c4a9d33 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -383,3 +383,9 @@ def setup_logging(args): stream=args.log_file or sys.stderr, level=log_level, ) + + +def safe_exit_if_file_is_empty(file_name): + if os.stat(file_name).st_size == 0: + logging.warning('Skipping all...') + sys.exit(0) From ba12a549bb9ad1703e162db4e52e3c38a4fb7224 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 23 Apr 2020 18:29:37 +0800 Subject: [PATCH 02/15] update the find_apps.py, now it will generate `build` and `preserve` keys --- tools/find_apps.py | 69 ++++++++++++++++----------------- tools/find_build_apps/common.py | 9 +++++ 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/tools/find_apps.py b/tools/find_apps.py index 35e0583de..0d8030e66 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -5,13 +5,13 @@ # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. import argparse +import json import os import sys import re import glob import logging import typing -from typing import Optional from find_build_apps import ( BUILD_SYSTEMS, @@ -47,9 +47,9 @@ def dict_from_sdkconfig(path): # Main logic: enumerating apps and builds -def find_builds_for_app( - app_path, work_dir, build_dir, build_log, target_arg, build_system, - config_rules): # type: (str, str, str, str, str, str, typing.List[ConfigRule]) -> typing.List[BuildItem] +def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, + build_system, config_rules, build_or_not=True, preserve_artifacts=True): + # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem] """ Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects :param app_path: app directory (can be / usually will be a relative path) @@ -62,6 +62,8 @@ def find_builds_for_app( a different CONFIG_IDF_TARGET value. :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary :param config_rules: mapping of sdkconfig file name patterns to configuration names + :param build_or_not: determine if it will build in build_apps.py + :param preserve_artifacts: determine if the built binary will be uploaded as artifacts. :return: list of BuildItems representing build configuration of the app """ build_items = [] # type: typing.List[BuildItem] @@ -106,6 +108,8 @@ def find_builds_for_app( sdkconfig_path, config_name, build_system, + build_or_not, + preserve_artifacts, )) if not build_items: @@ -120,14 +124,16 @@ def find_builds_for_app( None, default_config_name, build_system, + build_or_not, + preserve_artifacts, ) ] return build_items -def find_apps(build_system_class, path, recursive, exclude_list, target, - build_apps_list_file=None): # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str, Optional[str]) -> typing.List[str] +def find_apps(build_system_class, path, recursive, exclude_list, target): + # type: (typing.Type[BuildSystem], str, bool, typing.List[str], str) -> typing.List[str] """ Find app directories in path (possibly recursively), which contain apps for the given build system, compatible with the given target. @@ -151,18 +157,6 @@ def find_apps(build_system_class, path, recursive, exclude_list, target, return [] return [path] - # if this argument is empty, treat it as build_all. - if not build_apps_list_file: - build_all = True - else: - # if this argument is not empty but the file doesn't exists, treat as no supported apps. - if not os.path.exists(build_apps_list_file): - return [] - else: - build_all = False - with open(build_apps_list_file) as fr: - apps_need_be_built = set([line.strip() for line in fr.readlines() if line]) - # The remaining part is for recursive == True apps_found = [] # type: typing.List[str] for root, dirs, _ in os.walk(path, topdown=True): @@ -172,11 +166,6 @@ def find_apps(build_system_class, path, recursive, exclude_list, target, del dirs[:] continue - if not build_all: - if os.path.abspath(root) not in apps_need_be_built: - logging.debug("Skipping, app not listed in {}".format(build_apps_list_file)) - continue - if build_system_class.is_app(root): logging.debug("Found {} app in {}".format(build_system_name, root)) # Don't recurse into app subdirectories @@ -258,7 +247,7 @@ def main(): '--scan-tests-json', default=None, help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n" - "If it's None or the file does not exist, will build all apps and upload all artifacts." + "If the file does not exist, will build all apps and upload all artifacts." ) parser.add_argument("paths", nargs="+", help="One or more app paths.") args = parser.parse_args() @@ -276,30 +265,38 @@ def main(): logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) args.target = DEFAULT_TARGET - # Prepare the list of app paths - app_paths = [] # type: typing.List[str] - for path in args.paths: - app_paths += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target, args.build_apps_list_file) + # Prepare the list of app paths, try to read from the scan_tests result. + # If the file exists, then follow the file's app_dir and build/archifacts behavior, won't do find_apps() again. + # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. + if args.scan_tests_json and os.path.exists(args.scan_tests_json): + apps = [json.loads(line) for line in open(args.scan_tests_json)] + else: + app_dirs = [] + for path in args.paths: + app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) + apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs] - build_items = [] # type: typing.List[BuildItem] - if not app_paths: - logging.warning("No {} apps found, skipping...".format(build_system_class.NAME)) - sys.exit(0) + if not apps: + logging.critical("No {} apps found".format(build_system_class.NAME)) + raise SystemExit(1) + logging.info("Found {} apps".format(len(apps))) - logging.info("Found {} apps".format(len(app_paths))) - app_paths = sorted(app_paths) + apps.sort(key=lambda x: x['app_dir']) # Find compatible configurations of each app, collect them as BuildItems + build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) - for app_path in app_paths: + for app in apps: build_items += find_builds_for_app( - app_path, + app['app_dir'], args.work_dir, args.build_dir, args.build_log, args.target, args.build_system, config_rules, + app['build'], + app['preserve'], ) logging.info("Found {} builds".format(len(build_items))) diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 47c4a9d33..0dfb1ba6e 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -71,6 +71,8 @@ class BuildItem(object): sdkconfig_path, config_name, build_system, + build_or_not, + preserve_artifacts, ): # These internal variables store the paths with environment variables and placeholders; # Public properties with similar names use the _expand method to get the actual paths. @@ -84,6 +86,9 @@ class BuildItem(object): self.target = target self.build_system = build_system + self.build = build_or_not + self.preserve = preserve_artifacts + self._app_name = os.path.basename(os.path.normpath(app_path)) # Some miscellaneous build properties which are set later, at the build stage @@ -155,6 +160,8 @@ class BuildItem(object): "config": self.config_name, "target": self.target, "verbose": self.verbose, + "build": self.build, + "preserve": self.preserve, }) @staticmethod @@ -172,6 +179,8 @@ class BuildItem(object): config_name=d["config"], target=d["target"], build_system=d["build_system"], + build_or_not=d["build"], + preserve_artifacts=d["preserve"] ) result.verbose = d["verbose"] return result From 2d57fd7fcad45c62ee31cc37d049ff445a544c4b Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 23 Apr 2020 18:48:51 +0800 Subject: [PATCH 03/15] update the build_apps.py, now it can recognize `build` and `preserve` keys --- tools/build_apps.py | 15 +++++++++++++-- tools/ci/check_build_warnings.py | 5 ++--- tools/find_build_apps/__init__.py | 2 -- tools/find_build_apps/common.py | 6 ------ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/build_apps.py b/tools/build_apps.py index 412274c61..5b0688540 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -5,9 +5,10 @@ # import argparse +import shutil import sys import logging -from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS, safe_exit_if_file_is_empty +from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS def main(): @@ -71,10 +72,11 @@ def main(): help="Name of the file to read the list of builds from. If not specified, read from stdin.", ) args = parser.parse_args() + setup_logging(args) - safe_exit_if_file_is_empty(args.build_list.name) build_items = [BuildItem.from_json(line) for line in args.build_list] + if not build_items: logging.error("Empty build list!") raise SystemExit(1) @@ -106,6 +108,10 @@ def main(): failed_builds = [] for build_info in builds_for_current_job: + if not build_info.build: + logging.info('Skip build detected. Skipping...') + continue + logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) build_system_class = BUILD_SYSTEMS[build_info.build_system] try: @@ -116,6 +122,11 @@ def main(): failed_builds.append(build_info) else: raise SystemExit(1) + else: + if not build_info.preserve: + logging.info('NOT preserve artifacts detected. Deleting...') + shutil.rmtree(build_info.work_dir, ignore_errors=True) + shutil.rmtree(build_info.build_log_path, ignore_errors=True) if failed_builds: logging.error("The following build have failed:") diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index 89c59cf23..e64cc4066 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -13,10 +13,10 @@ import logging import re try: - from find_build_apps import BuildItem, setup_logging, safe_exit_if_file_is_empty + from find_build_apps import BuildItem, setup_logging except ImportError: sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - from find_build_apps import BuildItem, setup_logging, safe_exit_if_file_is_empty + from find_build_apps import BuildItem, setup_logging WARNING_REGEX = re.compile(r"(?:error|warning)[^\w]", re.MULTILINE | re.IGNORECASE) @@ -69,7 +69,6 @@ def main(): args = parser.parse_args() setup_logging(args) - safe_exit_if_file_is_empty(args.build_list.name) build_items = [BuildItem.from_json(line) for line in args.build_list] if not build_items: diff --git a/tools/find_build_apps/__init__.py b/tools/find_build_apps/__init__.py index 891721335..c87ceebda 100644 --- a/tools/find_build_apps/__init__.py +++ b/tools/find_build_apps/__init__.py @@ -6,7 +6,6 @@ from .common import ( config_rules_from_str, setup_logging, DEFAULT_TARGET, - safe_exit_if_file_is_empty, ) from .cmake import CMakeBuildSystem, BUILD_SYSTEM_CMAKE from .make import MakeBuildSystem, BUILD_SYSTEM_MAKE @@ -29,5 +28,4 @@ __all__ = [ "MakeBuildSystem", "BUILD_SYSTEM_MAKE", "BUILD_SYSTEMS", - "safe_exit_if_file_is_empty", ] diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 0dfb1ba6e..dcb39432d 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -392,9 +392,3 @@ def setup_logging(args): stream=args.log_file or sys.stderr, level=log_level, ) - - -def safe_exit_if_file_is_empty(file_name): - if os.stat(file_name).st_size == 0: - logging.warning('Skipping all...') - sys.exit(0) From 007b35ed251d471f03e939fef860615b65eb30b0 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 23 Apr 2020 19:13:09 +0800 Subject: [PATCH 04/15] comment check_build_warnings.py to test CI --- tools/ci/build_examples.sh | 2 +- tools/ci/build_test_apps.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 4a5915d10..bf483d1eb 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -106,4 +106,4 @@ ${IDF_PATH}/tools/build_apps.py \ # Check for build warnings -${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} +#${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index 627d1bb78..87757bc2f 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -100,4 +100,4 @@ ${IDF_PATH}/tools/build_apps.py \ # Check for build warnings -${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} +#${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} From ef85a6075691d1fa1f2a4904bd8ef27727accfc4 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 24 Apr 2020 10:13:40 +0800 Subject: [PATCH 05/15] skip check_build_warning for builditems whose `build` is false --- tools/build_apps.py | 2 +- tools/ci/build_examples.sh | 2 +- tools/ci/build_test_apps.sh | 2 +- tools/ci/check_build_warnings.py | 3 +++ tools/find_apps.py | 2 -- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/build_apps.py b/tools/build_apps.py index 5b0688540..be83347f9 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -125,8 +125,8 @@ def main(): else: if not build_info.preserve: logging.info('NOT preserve artifacts detected. Deleting...') + # we only remove binaries here, log files are still needed by check_build_warnings.py shutil.rmtree(build_info.work_dir, ignore_errors=True) - shutil.rmtree(build_info.build_log_path, ignore_errors=True) if failed_builds: logging.error("The following build have failed:") diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index bf483d1eb..4a5915d10 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -106,4 +106,4 @@ ${IDF_PATH}/tools/build_apps.py \ # Check for build warnings -#${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} +${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index 87757bc2f..627d1bb78 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -100,4 +100,4 @@ ${IDF_PATH}/tools/build_apps.py \ # Check for build warnings -#${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} +${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index e64cc4066..bb01035ac 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -77,6 +77,9 @@ def main(): found_warnings = 0 for build_item in build_items: + if not build_item.build: + logging.debug('Skipping build detected. Skipping checking...') + continue if not build_item.build_log_path: logging.debug("No log file for {}".format(build_item.work_dir)) continue diff --git a/tools/find_apps.py b/tools/find_apps.py index 0d8030e66..a7cfe0872 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -142,8 +142,6 @@ def find_apps(build_system_class, path, recursive, exclude_list, target): :param recursive: whether to recursively descend into nested directories if no app is found :param exclude_list: list of paths to be excluded from the recursive search :param target: desired value of IDF_TARGET; apps incompatible with the given target are skipped. - :param build_apps_list_file: List of apps need to be built, apps not on this list will be skipped. - None or empty file to build all apps. :return: list of paths of the apps found """ build_system_name = build_system_class.NAME From 53aacef29ca3590b139c95fa4eac78ffa590cda0 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 24 Apr 2020 10:39:44 +0800 Subject: [PATCH 06/15] code clean up --- tools/ci/python_packages/gitlab_api.py | 6 +----- .../ci/python_packages/tiny_test_fw/Utility/SearchCases.py | 1 + tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py | 2 +- tools/ci/python_packages/ttfw_idf/CIScanTests.py | 3 --- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 3d49dbb40..561ffb863 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -124,7 +124,7 @@ class Gitlab(object): return raw_data_list - def find_job_id(self, job_name, pipeline_id=None, job_status="success", suffix=None): + def find_job_id(self, job_name, pipeline_id=None, job_status="success"): """ Get Job ID from job name of specific pipeline @@ -132,7 +132,6 @@ class Gitlab(object): :param pipeline_id: If None, will get pipeline id from CI pre-defined variable. :param job_status: status of job. One pipeline could have multiple jobs with same name after retry. job_status is used to filter these jobs. - :param suffix: suffix of the job. e.g. 'limit' for build only needed apps. :return: a list of job IDs (parallel job will generate multiple jobs) """ job_id_list = [] @@ -145,9 +144,6 @@ class Gitlab(object): if match: if match.group(1) == job_name and job.status == job_status: job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) - elif suffix: - if match.group(1) == "{}_{}".format(job_name, suffix) and job.status == job_status: - job_id_list.append({"id": job.id, "parallel_num": match.group(3)}) return job_id_list @retry_download diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index c36fec8ad..d85283b76 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -125,6 +125,7 @@ class Search(object): search all test cases from a folder or file, and then do case replicate. :param test_case: test case file(s) path + :param test_case_file_pattern: unix filename pattern :return: a list of replicated test methods """ test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 99af101c0..3acdd78d0 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -65,7 +65,7 @@ def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=Exa return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) for build_job_name in case_group.BUILD_JOB_NAMES: - job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id, suffix='limit') + job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) for job_info in job_info_list: raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] build_info_list = [json.loads(line) for line in raw_data.splitlines()] diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index c56664f6b..b087aa3e5 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -57,7 +57,6 @@ def main(): actions.add_parser('example_test', parents=[common]) actions.add_parser('test_apps', parents=[common]) - # actions.add_parser('unit_test', parents=[common]) args = parser.parse_args() @@ -68,8 +67,6 @@ def main(): elif args.action == 'test_apps': CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup) - # elif args.action == 'unit_test': - # assign = UnitTestAssignTest(args.test_case, args.ci_config_file) else: raise SystemExit(1) # which is impossible From a238b266c2ef8375b07230428cf5372ccb18646e Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 6 May 2020 12:12:37 +0800 Subject: [PATCH 07/15] fix after rebase --- tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index ad71d0dd5..b04a7654f 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -164,13 +164,14 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): return test_cases test_cases = [] - if os.path.isdir(test_case_path): - for yml_file in find_by_suffix('.yml', test_case_path): + if os.path.isdir(self.test_case_path): + for yml_file in find_by_suffix('.yml', self.test_case_path): test_cases.extend(get_test_cases_from_yml(yml_file)) - elif os.path.isfile(test_case_path): - test_cases.extend(get_test_cases_from_yml(test_case_path)) + elif os.path.isfile(self.test_case_path): + test_cases.extend(get_test_cases_from_yml(self.test_case_path)) else: print("Test case path is invalid. Should only happen when use @bot to skip unit test.") + raise SystemExit(1) # filter keys are lower case. Do map lower case keys with original keys. try: From d7639d5cf86019e1e5a3a5e5cf505685cc1128e7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 19 Jun 2020 14:45:46 +0800 Subject: [PATCH 08/15] add build-all flag and BUILD_ALL_APPS ci environment variable --- .../python_packages/ttfw_idf/CIScanTests.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index b087aa3e5..a6d62a9b1 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -1,4 +1,5 @@ import argparse +import errno import json import os import re @@ -20,7 +21,12 @@ SPECIAL_REFS = [ ] -def _judge_build_all(): +def _judge_build_all(args_build_all): + if args_build_all: + return True + if os.getenv('BUILD_ALL_APPS'): + return True + ref = os.getenv('CI_COMMIT_REF_NAME') pipeline_src = os.getenv('CI_PIPELINE_SOURCE') if not ref or not pipeline_src: @@ -49,11 +55,13 @@ def main(): common.add_argument('paths', type=str, nargs='+', help="One or more app paths") common.add_argument('-c', '--ci_config_file', type=str, required=True, - help="gitlab ci config file") + help="gitlab ci config target-test file") common.add_argument('-o', '--output_path', type=str, required=True, help="output path of the scan result") common.add_argument('-p', '--preserve-all', action="store_true", help='add this flag to preserve artifacts for all apps') + common.add_argument('-b', '--build-all', action="store_true", + help='add this flag to build all apps') actions.add_parser('example_test', parents=[common]) actions.add_parser('test_apps', parents=[common]) @@ -72,10 +80,12 @@ def main(): test_cases.extend(assign.search_cases()) - try: - os.makedirs(args.output_path) - except Exception: - pass + if not os.path.exists(args.output_path): + try: + os.makedirs(args.output_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise e ''' { @@ -90,13 +100,12 @@ def main(): # store the test cases dir, exclude these folders when scan for standalone apps exclude_apps = [] - # TODO change this chip to target after feat/add_multi_target_for_example_test is merged for target in VALID_TARGETS: target_dict = scan_info_dict[target] test_case_apps = target_dict['test_case_apps'] = set() for case in test_cases: app_dir = case.case_info['app_dir'] - app_target = case.case_info['chip'] + app_target = case.case_info['target'] if app_target.lower() != target.lower(): continue test_case_apps.update(find_apps(CMakeBuildSystem, app_dir, True, [], target.lower())) @@ -108,7 +117,7 @@ def main(): for path in args.paths: standalone_apps.update(find_apps(CMakeBuildSystem, path, True, exclude_apps, target.lower())) - build_all = _judge_build_all() + build_all = _judge_build_all(args.build_all) for target in VALID_TARGETS: apps = [] for app_dir in scan_info_dict[target]['test_case_apps']: From 010d7e90238388aba4a484fdf31f3d76e452b101 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 19 Jun 2020 18:58:03 +0800 Subject: [PATCH 09/15] apply to make build system --- tools/ci/config/build.yml | 5 +-- tools/ci/config/pre_check.yml | 3 +- .../python_packages/ttfw_idf/CIScanTests.py | 36 +++++++++++-------- tools/find_build_apps/cmake.py | 30 ++++++++++++++++ tools/find_build_apps/common.py | 29 ++------------- tools/find_build_apps/make.py | 6 +++- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 6b10837e0..200becc2c 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -96,9 +96,6 @@ build_esp_idf_tests_cmake_esp32s2: - mkdir ${BUILD_PATH} - mkdir -p ${LOG_PATH} - ${IDF_PATH}/tools/ci/build_examples.sh - # Check if the tests demand Make built binaries. If not, delete them - - if [ ${EXAMPLE_TEST_BUILD_SYSTEM} == "cmake" ]; then exit 0; fi - - rm -rf ${BUILD_PATH} build_examples_make: extends: .build_examples_template @@ -113,6 +110,7 @@ build_examples_make: BUILD_PATH: "${CI_PROJECT_DIR}/build_examples_make" EXAMPLE_TEST_BUILD_SYSTEM: "make" IDF_TARGET: "esp32" # currently we only support esp32 + SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json only: refs: - master @@ -146,7 +144,6 @@ build_examples_make: EXAMPLE_TEST_BUILD_SYSTEM: "cmake" SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json - build_examples_cmake_esp32: extends: .build_examples_cmake variables: diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index a7e971df7..7d2b80ec6 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -215,5 +215,6 @@ scan_tests: TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs script: - - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index a6d62a9b1..fbac2bb9c 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -6,7 +6,7 @@ import re from collections import defaultdict from find_apps import find_apps -from find_build_apps import CMakeBuildSystem +from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup VALID_TARGETS = [ @@ -54,13 +54,14 @@ def main(): common = argparse.ArgumentParser(add_help=False) common.add_argument('paths', type=str, nargs='+', help="One or more app paths") - common.add_argument('-c', '--ci_config_file', type=str, required=True, + common.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) + common.add_argument('-c', '--ci-config-file', type=str, required=True, help="gitlab ci config target-test file") - common.add_argument('-o', '--output_path', type=str, required=True, + common.add_argument('-o', '--output-path', type=str, required=True, help="output path of the scan result") - common.add_argument('-p', '--preserve-all', action="store_true", + common.add_argument('--preserve', action="store_true", help='add this flag to preserve artifacts for all apps') - common.add_argument('-b', '--build-all', action="store_true", + common.add_argument('--build-all', action="store_true", help='add this flag to build all apps') actions.add_parser('example_test', parents=[common]) @@ -90,8 +91,8 @@ def main(): ''' { : { - 'test_case_apps': [], - 'standalone_apps': [], + 'test_case_apps': [], # which is used in target tests + 'standalone_apps': [], # which is not }, ... } @@ -100,6 +101,9 @@ def main(): # store the test cases dir, exclude these folders when scan for standalone apps exclude_apps = [] + build_system = args.build_system.lower() + build_system_class = BUILD_SYSTEMS[build_system] + for target in VALID_TARGETS: target_dict = scan_info_dict[target] test_case_apps = target_dict['test_case_apps'] = set() @@ -108,32 +112,36 @@ def main(): app_target = case.case_info['target'] if app_target.lower() != target.lower(): continue - test_case_apps.update(find_apps(CMakeBuildSystem, app_dir, True, [], target.lower())) + test_case_apps.update(find_apps(build_system_class, app_dir, True, [], target.lower())) exclude_apps.append(app_dir) for target in VALID_TARGETS: target_dict = scan_info_dict[target] standalone_apps = target_dict['standalone_apps'] = set() for path in args.paths: - standalone_apps.update(find_apps(CMakeBuildSystem, path, True, exclude_apps, target.lower())) + standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower())) build_all = _judge_build_all(args.build_all) + test_case_apps_preserve_default = True if build_system == 'cmake' else False + for target in VALID_TARGETS: apps = [] for app_dir in scan_info_dict[target]['test_case_apps']: apps.append({ 'app_dir': app_dir, 'build': True, - 'preserve': True, + 'preserve': args.preserve or test_case_apps_preserve_default }) for app_dir in scan_info_dict[target]['standalone_apps']: apps.append({ 'app_dir': app_dir, - 'build': build_all, - 'preserve': args.preserve_all and build_all, # you can't preserve the artifacts if you don't build them right? + 'build': build_all if build_system == 'cmake' else True, + 'preserve': (args.preserve and build_all) if build_system == 'cmake' else False }) - with open(os.path.join(args.output_path, 'scan_{}.json'.format(target.lower())), 'w') as fw: - fw.writelines([json.dumps(app) + '\n' for app in apps]) + output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) + if apps: + with open(output_path, 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps]) if __name__ == '__main__': diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py index ab4a842ad..56b57c864 100644 --- a/tools/find_build_apps/cmake.py +++ b/tools/find_build_apps/cmake.py @@ -93,3 +93,33 @@ class CMakeBuildSystem(BuildSystem): if CMAKE_PROJECT_LINE not in cmakelists_file_content: return False return True + + @staticmethod + def supported_targets(app_path): + formal_to_usual = { + 'ESP32': 'esp32', + 'ESP32-S2': 'esp32s2', + } + + readme_file_content = BuildSystem._read_readme(app_path) + if not readme_file_content: + return None + match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) + if not match: + return None + if len(match) > 1: + raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) + support_str = match[0].strip() + + targets = [] + for part in support_str.split('|'): + for inner in part.split(' '): + inner = inner.strip() + if not inner: + continue + elif inner in formal_to_usual: + targets.append(formal_to_usual[inner]) + else: + raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" + .format(inner, app_path, ', '.join(formal_to_usual.keys()))) + return targets diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index dcb39432d..31bbad247 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -341,34 +341,9 @@ class BuildSystem(object): return readme_file.read() @staticmethod + @abstractmethod def supported_targets(app_path): - formal_to_usual = { - 'ESP32': 'esp32', - 'ESP32-S2': 'esp32s2', - } - - readme_file_content = BuildSystem._read_readme(app_path) - if not readme_file_content: - return None - match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) - if not match: - return None - if len(match) > 1: - raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) - support_str = match[0].strip() - - targets = [] - for part in support_str.split('|'): - for inner in part.split(' '): - inner = inner.strip() - if not inner: - continue - elif inner in formal_to_usual: - targets.append(formal_to_usual[inner]) - else: - raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" - .format(inner, app_path, ', '.join(formal_to_usual.keys()))) - return targets + pass class BuildError(RuntimeError): diff --git a/tools/find_build_apps/make.py b/tools/find_build_apps/make.py index a8b153fd0..e7b885732 100644 --- a/tools/find_build_apps/make.py +++ b/tools/find_build_apps/make.py @@ -1,8 +1,8 @@ import logging import os +import shlex import subprocess import sys -import shlex from .common import BuildSystem, BuildError @@ -58,3 +58,7 @@ class MakeBuildSystem(BuildSystem): if MAKE_PROJECT_LINE not in makefile_content: return False return True + + @staticmethod + def supported_targets(app_path): + return ['esp32'] From a882353c320016bc3fe96ad3f9b1aa85838edb97 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 28 Jun 2020 13:55:53 +0800 Subject: [PATCH 10/15] change scan result path --- tools/ci/config/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 200becc2c..3d56725bf 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -82,6 +82,8 @@ build_esp_idf_tests_cmake_esp32s2: artifacts: when: always expire_in: 4 days + variables: + SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}_${EXAMPLE_TEST_BUILD_SYSTEM}.json only: # Here both 'variables' and 'refs' conditions are given. They are combined with "AND" logic. variables: @@ -110,7 +112,6 @@ build_examples_make: BUILD_PATH: "${CI_PROJECT_DIR}/build_examples_make" EXAMPLE_TEST_BUILD_SYSTEM: "make" IDF_TARGET: "esp32" # currently we only support esp32 - SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json only: refs: - master @@ -142,7 +143,6 @@ build_examples_make: LOG_PATH: "${CI_PROJECT_DIR}/log_examples" BUILD_PATH: "${CI_PROJECT_DIR}/build_examples" EXAMPLE_TEST_BUILD_SYSTEM: "cmake" - SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json build_examples_cmake_esp32: extends: .build_examples_cmake From 270e7dec384c04da940f537e322889ab38a6bc73 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 28 Jun 2020 13:56:04 +0800 Subject: [PATCH 11/15] update after MR reviews --- tools/build_apps.py | 9 +- tools/ci/build_examples.sh | 8 +- tools/ci/build_test_apps.sh | 7 +- tools/ci/build_unit_test.sh | 3 +- tools/ci/check_build_warnings.py | 6 +- tools/ci/config/build.yml | 3 +- tools/ci/config/pre_check.yml | 5 +- .../python_packages/ttfw_idf/CIScanTests.py | 30 +++++-- tools/find_apps.py | 83 ++++++++++++------- 9 files changed, 92 insertions(+), 62 deletions(-) diff --git a/tools/build_apps.py b/tools/build_apps.py index be83347f9..9d2969ca3 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -5,9 +5,10 @@ # import argparse +import logging import shutil import sys -import logging + from find_build_apps import BuildItem, BuildError, setup_logging, BUILD_SYSTEMS @@ -109,7 +110,7 @@ def main(): failed_builds = [] for build_info in builds_for_current_job: if not build_info.build: - logging.info('Skip build detected. Skipping...') + logging.info("Skip building app {}".format(build_info.app_dir)) continue logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) @@ -124,9 +125,9 @@ def main(): raise SystemExit(1) else: if not build_info.preserve: - logging.info('NOT preserve artifacts detected. Deleting...') + logging.info("Removing build directory {}".format(build_info.build_dir)) # we only remove binaries here, log files are still needed by check_build_warnings.py - shutil.rmtree(build_info.work_dir, ignore_errors=True) + shutil.rmtree(build_info.build_dir, ignore_errors=True) if failed_builds: logging.error("The following build have failed:") diff --git a/tools/ci/build_examples.sh b/tools/ci/build_examples.sh index 4a5915d10..df7e4745a 100755 --- a/tools/ci/build_examples.sh +++ b/tools/ci/build_examples.sh @@ -72,13 +72,9 @@ cd ${IDF_PATH} # If changing the work-dir or build-dir format, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. -${IDF_PATH}/tools/find_apps.py examples \ +${IDF_PATH}/tools/find_apps.py \ -vv \ --format json \ - --build-system ${EXAMPLE_TEST_BUILD_SYSTEM} \ - --target ${IDF_TARGET} \ - --recursive \ - --exclude examples/build_system/idf_as_lib \ --work-dir "${BUILD_PATH}/@f/@w/@t" \ --build-dir build \ --build-log "${LOG_PATH}/@f_@w.txt" \ @@ -86,7 +82,7 @@ ${IDF_PATH}/tools/find_apps.py examples \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ - --scan-tests-json ${SCAN_EXAMPLE_TEST_JSON} + --app-list ${SCAN_EXAMPLE_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index 627d1bb78..7909044f0 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -62,12 +62,9 @@ cd ${IDF_PATH} # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. -${IDF_PATH}/tools/find_apps.py tools/test_apps \ +${IDF_PATH}/tools/find_apps.py \ -vv \ --format json \ - --build-system cmake \ - --target ${IDF_TARGET} \ - --recursive \ --work-dir "${BUILD_PATH}/@f/@w/@t" \ --build-dir build \ --build-log "${LOG_PATH}/@f_@w.txt" \ @@ -75,7 +72,7 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ --config 'sdkconfig.ci=default' \ --config 'sdkconfig.ci.*=' \ --config '=default' \ - --scan-tests-json ${SCAN_CUSTOM_TEST_JSON} + --app-list ${SCAN_CUSTOM_TEST_JSON} # --config rules above explained: # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" diff --git a/tools/ci/build_unit_test.sh b/tools/ci/build_unit_test.sh index dd4402f1d..deeb0e8cd 100755 --- a/tools/ci/build_unit_test.sh +++ b/tools/ci/build_unit_test.sh @@ -63,7 +63,8 @@ cd ${IDF_PATH} # This part of the script produces the same result for all the unit test app build jobs. It may be moved to a separate stage # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. -${IDF_PATH}/tools/find_apps.py tools/unit-test-app \ +${IDF_PATH}/tools/find_apps.py \ + -p tools/unit-test-app \ -vv \ --format json \ --build-system cmake \ diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index bb01035ac..0d63869a3 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -6,11 +6,11 @@ # log files for every build. # Exits with a non-zero exit code if any warning is found. -import os -import sys import argparse import logging +import os import re +import sys try: from find_build_apps import BuildItem, setup_logging @@ -78,7 +78,7 @@ def main(): found_warnings = 0 for build_item in build_items: if not build_item.build: - logging.debug('Skipping build detected. Skipping checking...') + logging.debug("Skip checking build log for app {}".format(build_item.app_dir)) continue if not build_item.build_log_path: logging.debug("No log file for {}".format(build_item.work_dir)) diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 3d56725bf..d09977aef 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -176,7 +176,8 @@ build_examples_cmake_esp32s2: variables: LOG_PATH: "${CI_PROJECT_DIR}/log_test_apps" BUILD_PATH: "${CI_PROJECT_DIR}/build_test_apps" - SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}.json + CUSTOM_TEST_BUILD_SYSTEM: "cmake" + SCAN_CUSTOM_TEST_JSON: ${CI_PROJECT_DIR}/tools/test_apps/test_configs/scan_${IDF_TARGET}_${CUSTOM_TEST_BUILD_SYSTEM}.json only: variables: - $BOT_TRIGGER_WITH_LABEL == null diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index 7d2b80ec6..a300f4413 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -203,6 +203,7 @@ scan_tests: extends: .scan_build_tests only: variables: + - $BOT_TRIGGER_WITH_LABEL == null - $BOT_LABEL_EXAMPLE_TEST - $BOT_LABEL_CUSTOM_TEST artifacts: @@ -215,6 +216,6 @@ scan_tests: TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs script: - - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR --exclude examples/build_system/idf_as_lib -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index fbac2bb9c..88764d4f3 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -52,13 +52,21 @@ def main(): actions = parser.add_subparsers(dest='action') common = argparse.ArgumentParser(add_help=False) - common.add_argument('paths', type=str, nargs='+', + common.add_argument('paths', + nargs='+', help="One or more app paths") - common.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) - common.add_argument('-c', '--ci-config-file', type=str, required=True, + common.add_argument('-b', '--build-system', + choices=BUILD_SYSTEMS.keys(), + default=BUILD_SYSTEM_CMAKE) + common.add_argument('-c', '--ci-config-file', + required=True, help="gitlab ci config target-test file") - common.add_argument('-o', '--output-path', type=str, required=True, + common.add_argument('-o', '--output-path', + required=True, help="output path of the scan result") + common.add_argument("--exclude", + action="append", + help="Ignore specified directory. Can be used multiple times.") common.add_argument('--preserve', action="store_true", help='add this flag to preserve artifacts for all apps') common.add_argument('--build-all', action="store_true", @@ -99,7 +107,8 @@ def main(): ''' scan_info_dict = defaultdict(dict) # store the test cases dir, exclude these folders when scan for standalone apps - exclude_apps = [] + default_exclude = args.exclude if args.exclude else [] + exclude_apps = default_exclude build_system = args.build_system.lower() build_system_class = BUILD_SYSTEMS[build_system] @@ -112,7 +121,7 @@ def main(): app_target = case.case_info['target'] if app_target.lower() != target.lower(): continue - test_case_apps.update(find_apps(build_system_class, app_dir, True, [], target.lower())) + test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower())) exclude_apps.append(app_dir) for target in VALID_TARGETS: @@ -130,18 +139,21 @@ def main(): apps.append({ 'app_dir': app_dir, 'build': True, + 'build_system': args.build_system, + 'target': target, 'preserve': args.preserve or test_case_apps_preserve_default }) for app_dir in scan_info_dict[target]['standalone_apps']: apps.append({ 'app_dir': app_dir, 'build': build_all if build_system == 'cmake' else True, + 'build_system': args.build_system, + 'target': target, 'preserve': (args.preserve and build_all) if build_system == 'cmake' else False }) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) - if apps: - with open(output_path, 'w') as fw: - fw.writelines([json.dumps(app) + '\n' for app in apps]) + with open(output_path, 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps]) if __name__ == '__main__': diff --git a/tools/find_apps.py b/tools/find_apps.py index a7cfe0872..0fe856b48 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -5,12 +5,13 @@ # Produces the list of builds. The list can be consumed by build_apps.py, which performs the actual builds. import argparse -import json -import os -import sys -import re import glob +import json import logging +import os +import re +import sys + import typing from find_build_apps import ( @@ -197,7 +198,10 @@ def main(): action="store_true", help="Look for apps in the specified directories recursively.", ) - parser.add_argument("--build-system", choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) + parser.add_argument( + "--build-system", + choices=BUILD_SYSTEMS.keys() + ) parser.add_argument( "--work-dir", help="If set, the app is first copied into the specified directory, and then built." + @@ -241,60 +245,77 @@ def main(): help="Output the list of builds to the specified file", ) parser.add_argument( - '-s', - '--scan-tests-json', + "--app-list", default=None, - help="Scan tests result. Restrict the build/architect behavior to apps need to be built.\n" + help="Scan tests results. Restrict the build/artifacts preservation behavior to apps need to be built. " "If the file does not exist, will build all apps and upload all artifacts." ) - parser.add_argument("paths", nargs="+", help="One or more app paths.") + parser.add_argument( + "-p", "--paths", + nargs="+", + help="One or more app paths." + ) args = parser.parse_args() setup_logging(args) - build_system_class = BUILD_SYSTEMS[args.build_system] - - # If the build target is not set explicitly, get it from the environment or use the default one (esp32) - if not args.target: - env_target = os.environ.get("IDF_TARGET") - if env_target: - logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) - args.target = env_target - else: - logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) - args.target = DEFAULT_TARGET + # Arguments Validation + if args.app_list: + conflict_args = [args.recursive, args.build_system, args.target, args.exclude, args.paths] + if any(conflict_args): + raise ValueError('Conflict settings. "recursive", "build_system", "target", "exclude", "paths" should not ' + 'be specified with "app_list"') + if not os.path.exists(args.app_list): + raise OSError("File not found {}".format(args.app_list)) + else: + # If the build target is not set explicitly, get it from the environment or use the default one (esp32) + if not args.target: + env_target = os.environ.get("IDF_TARGET") + if env_target: + logging.info("--target argument not set, using IDF_TARGET={} from the environment".format(env_target)) + args.target = env_target + else: + logging.info("--target argument not set, using IDF_TARGET={} as the default".format(DEFAULT_TARGET)) + args.target = DEFAULT_TARGET + if not args.build_system: + logging.info("--build-system argument not set, using {} as the default".format(BUILD_SYSTEM_CMAKE)) + args.build_system = BUILD_SYSTEM_CMAKE + required_args = [args.build_system, args.target, args.paths] + if not all(required_args): + raise ValueError('If app_list not set, arguments "build_system", "target", "paths" are required.') # Prepare the list of app paths, try to read from the scan_tests result. - # If the file exists, then follow the file's app_dir and build/archifacts behavior, won't do find_apps() again. + # If the file exists, then follow the file's app_dir and build/artifacts behavior, won't do find_apps() again. # If the file not exists, will do find_apps() first, then build all apps and upload all artifacts. - if args.scan_tests_json and os.path.exists(args.scan_tests_json): - apps = [json.loads(line) for line in open(args.scan_tests_json)] + if args.app_list: + apps = [json.loads(line) for line in open(args.app_list)] else: app_dirs = [] + build_system_class = BUILD_SYSTEMS[args.build_system] for path in args.paths: app_dirs += find_apps(build_system_class, path, args.recursive, args.exclude or [], args.target) - apps = [{'app_dir': app_dir, 'build': True, 'preserve': True} for app_dir in app_dirs] + apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs] if not apps: - logging.critical("No {} apps found".format(build_system_class.NAME)) + logging.critical("No apps found") raise SystemExit(1) logging.info("Found {} apps".format(len(apps))) - apps.sort(key=lambda x: x['app_dir']) + apps.sort(key=lambda x: x["app_dir"]) # Find compatible configurations of each app, collect them as BuildItems build_items = [] # type: typing.List[BuildItem] config_rules = config_rules_from_str(args.config or []) for app in apps: build_items += find_builds_for_app( - app['app_dir'], + app["app_dir"], args.work_dir, args.build_dir, args.build_log, - args.target, - args.build_system, + args.target or app["target"], + args.build_system or app["build_system"], config_rules, - app['build'], - app['preserve'], + app["build"], + app["preserve"], ) logging.info("Found {} builds".format(len(build_items))) From 2c2f66aa2147d902bdc1655232058994a0a04648 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Sun, 28 Jun 2020 16:40:49 +0800 Subject: [PATCH 12/15] add supported target to ESP32-S2 Temperature Sensor Example --- examples/peripherals/temp_sensor_esp32s2/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/peripherals/temp_sensor_esp32s2/README.md b/examples/peripherals/temp_sensor_esp32s2/README.md index 161798695..e0e328f63 100644 --- a/examples/peripherals/temp_sensor_esp32s2/README.md +++ b/examples/peripherals/temp_sensor_esp32s2/README.md @@ -1,3 +1,6 @@ +| Supported Targets | ESP32-S2 | +| ----------------- | -------- | + # ESP32-S2 Temperature Sensor Example The ESP32-S2 has a built-in temperature sensor. The temperature sensor module contains an 8-bit Sigma-Delta ADC and a temperature offset DAC. From 408715230522e80ff4a98d7b04edcee506f1c02f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 29 Jun 2020 18:11:49 +0800 Subject: [PATCH 13/15] remove build_or_not attr. --- tools/build_apps.py | 9 +- tools/ci/check_build_warnings.py | 8 +- .../python_packages/ttfw_idf/CIScanTests.py | 163 ++++++++++-------- tools/find_apps.py | 14 +- tools/find_build_apps/common.py | 4 - 5 files changed, 98 insertions(+), 100 deletions(-) diff --git a/tools/build_apps.py b/tools/build_apps.py index 9d2969ca3..c18e259c9 100755 --- a/tools/build_apps.py +++ b/tools/build_apps.py @@ -77,10 +77,9 @@ def main(): setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] - if not build_items: - logging.error("Empty build list!") - raise SystemExit(1) + logging.warning("Empty build list") + SystemExit(0) num_builds = len(build_items) num_jobs = args.parallel_count @@ -109,10 +108,6 @@ def main(): failed_builds = [] for build_info in builds_for_current_job: - if not build_info.build: - logging.info("Skip building app {}".format(build_info.app_dir)) - continue - logging.info("Running build {}: {}".format(build_info.index, repr(build_info))) build_system_class = BUILD_SYSTEMS[build_info.build_system] try: diff --git a/tools/ci/check_build_warnings.py b/tools/ci/check_build_warnings.py index 0d63869a3..d8b359111 100755 --- a/tools/ci/check_build_warnings.py +++ b/tools/ci/check_build_warnings.py @@ -70,16 +70,12 @@ def main(): setup_logging(args) build_items = [BuildItem.from_json(line) for line in args.build_list] - if not build_items: - logging.error("Empty build list!") - raise SystemExit(1) + logging.warning("Empty build list") + SystemExit(0) found_warnings = 0 for build_item in build_items: - if not build_item.build: - logging.debug("Skip checking build log for app {}".format(build_item.app_dir)) - continue if not build_item.build_log_path: logging.debug("No log file for {}".format(build_item.work_dir)) continue diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index 88764d4f3..f78f3ede5 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -1,6 +1,7 @@ import argparse import errno import json +import logging import os import re from collections import defaultdict @@ -14,80 +15,73 @@ VALID_TARGETS = [ 'esp32s2', ] -SPECIAL_REFS = [ - 'master', - re.compile(r'^release/v'), - re.compile(r'^v\d+\.\d+'), +TEST_LABELS = { + 'example_test': 'BOT_LABEL_EXAMPLE_TEST', + 'test_apps': 'BOT_LABEL_CUSTOM_TEST', +} + +BUILD_ALL_LABELS = [ + 'BOT_LABEL_BUILD_ALL_APPS', + 'BOT_LABEL_REGULAR_TEST', ] -def _judge_build_all(args_build_all): - if args_build_all: - return True - if os.getenv('BUILD_ALL_APPS'): - return True - - ref = os.getenv('CI_COMMIT_REF_NAME') - pipeline_src = os.getenv('CI_PIPELINE_SOURCE') - if not ref or not pipeline_src: - return False - - # scheduled pipeline will build all - if pipeline_src == 'schedule': - return True - - # master, release/v..., v1.2.3..., and will build all - for special_ref in SPECIAL_REFS: - if isinstance(special_ref, re._pattern_type): - if special_ref.match(ref): - return True - else: - if ref == special_ref: - return True +def _has_build_all_label(): + for label in BUILD_ALL_LABELS: + if os.getenv(label): + return True return False +def _judge_build_or_not(action, build_all): # type: (str, bool) -> (bool, bool) + """ + :return: (build_or_not_for_test_related_apps, build_or_not_for_non_related_apps) + """ + if build_all or _has_build_all_label() or (not os.getenv('BOT_TRIGGER_WITH_LABEL')): + logging.info('Build all apps') + return True, True + + if os.getenv(TEST_LABELS[action]): + logging.info('Build test cases apps') + return True, False + else: + logging.info('Skip all') + return False, False + + +def output_json(apps_dict_list, target, build_system, output_dir): + output_path = os.path.join(output_dir, 'scan_{}_{}.json'.format(target.lower(), build_system)) + with open(output_path, 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps_dict_list]) + + def main(): parser = argparse.ArgumentParser(description='Scan the required build tests') - actions = parser.add_subparsers(dest='action') - - common = argparse.ArgumentParser(add_help=False) - common.add_argument('paths', + parser.add_argument('test_type', + choices=TEST_LABELS.keys(), + help='Scan test type') + parser.add_argument('paths', nargs='+', - help="One or more app paths") - common.add_argument('-b', '--build-system', + help='One or more app paths') + parser.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) - common.add_argument('-c', '--ci-config-file', + parser.add_argument('-c', '--ci-config-file', required=True, help="gitlab ci config target-test file") - common.add_argument('-o', '--output-path', + parser.add_argument('-o', '--output-path', required=True, help="output path of the scan result") - common.add_argument("--exclude", + parser.add_argument("--exclude", action="append", - help="Ignore specified directory. Can be used multiple times.") - common.add_argument('--preserve', action="store_true", + help='Ignore specified directory. Can be used multiple times.') + parser.add_argument('--preserve', action="store_true", help='add this flag to preserve artifacts for all apps') - common.add_argument('--build-all', action="store_true", + parser.add_argument('--build-all', action="store_true", help='add this flag to build all apps') - actions.add_parser('example_test', parents=[common]) - actions.add_parser('test_apps', parents=[common]) - args = parser.parse_args() - - test_cases = [] - for path in args.paths: - if args.action == 'example_test': - assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup) - elif args.action == 'test_apps': - CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') - assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup) - else: - raise SystemExit(1) # which is impossible - - test_cases.extend(assign.search_cases()) + build_test_case_apps, build_standalone_apps = _judge_build_or_not(args.test_type, args.build_all) if not os.path.exists(args.output_path): try: @@ -96,6 +90,23 @@ def main(): if e.errno != errno.EEXIST: raise e + if (not build_standalone_apps) and (not build_test_case_apps): + for target in VALID_TARGETS: + output_json([], target, args.build_system, args.output_path) + SystemExit(0) + + test_cases = [] + for path in set(args.paths): + if args.test_type == 'example_test': + assign = CIExampleAssignTest(path, args.ci_config_file, ExampleGroup) + elif args.test_type == 'test_apps': + CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r'^test_app_test_.+') + assign = CIExampleAssignTest(path, args.ci_config_file, TestAppsGroup) + else: + raise SystemExit(1) # which is impossible + + test_cases.extend(assign.search_cases()) + ''' { : { @@ -113,32 +124,37 @@ def main(): build_system = args.build_system.lower() build_system_class = BUILD_SYSTEMS[build_system] - for target in VALID_TARGETS: - target_dict = scan_info_dict[target] - test_case_apps = target_dict['test_case_apps'] = set() - for case in test_cases: - app_dir = case.case_info['app_dir'] - app_target = case.case_info['target'] - if app_target.lower() != target.lower(): - continue - test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower())) - exclude_apps.append(app_dir) + if build_test_case_apps: + for target in VALID_TARGETS: + target_dict = scan_info_dict[target] + test_case_apps = target_dict['test_case_apps'] = set() + for case in test_cases: + app_dir = case.case_info['app_dir'] + app_target = case.case_info['target'] + if app_target.lower() != target.lower(): + continue + test_case_apps.update(find_apps(build_system_class, app_dir, True, default_exclude, target.lower())) + exclude_apps.append(app_dir) + else: + for target in VALID_TARGETS: + scan_info_dict[target]['test_case_apps'] = set() - for target in VALID_TARGETS: - target_dict = scan_info_dict[target] - standalone_apps = target_dict['standalone_apps'] = set() - for path in args.paths: - standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower())) + if build_standalone_apps: + for target in VALID_TARGETS: + target_dict = scan_info_dict[target] + standalone_apps = target_dict['standalone_apps'] = set() + for path in args.paths: + standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower())) + else: + for target in VALID_TARGETS: + scan_info_dict[target]['standalone_apps'] = set() - build_all = _judge_build_all(args.build_all) test_case_apps_preserve_default = True if build_system == 'cmake' else False - for target in VALID_TARGETS: apps = [] for app_dir in scan_info_dict[target]['test_case_apps']: apps.append({ 'app_dir': app_dir, - 'build': True, 'build_system': args.build_system, 'target': target, 'preserve': args.preserve or test_case_apps_preserve_default @@ -146,10 +162,9 @@ def main(): for app_dir in scan_info_dict[target]['standalone_apps']: apps.append({ 'app_dir': app_dir, - 'build': build_all if build_system == 'cmake' else True, 'build_system': args.build_system, 'target': target, - 'preserve': (args.preserve and build_all) if build_system == 'cmake' else False + 'preserve': args.preserve }) output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) with open(output_path, 'w') as fw: diff --git a/tools/find_apps.py b/tools/find_apps.py index 0fe856b48..6fdaf178f 100755 --- a/tools/find_apps.py +++ b/tools/find_apps.py @@ -49,8 +49,8 @@ def dict_from_sdkconfig(path): def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, - build_system, config_rules, build_or_not=True, preserve_artifacts=True): - # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool, bool) -> typing.List[BuildItem] + build_system, config_rules, preserve_artifacts=True): + # type: (str, str, str, str, str, str, typing.List[ConfigRule], bool) -> typing.List[BuildItem] """ Find configurations (sdkconfig file fragments) for the given app, return them as BuildItem objects :param app_path: app directory (can be / usually will be a relative path) @@ -63,7 +63,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, a different CONFIG_IDF_TARGET value. :param build_system: name of the build system, index into BUILD_SYSTEMS dictionary :param config_rules: mapping of sdkconfig file name patterns to configuration names - :param build_or_not: determine if it will build in build_apps.py :param preserve_artifacts: determine if the built binary will be uploaded as artifacts. :return: list of BuildItems representing build configuration of the app """ @@ -109,7 +108,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, sdkconfig_path, config_name, build_system, - build_or_not, preserve_artifacts, )) @@ -125,7 +123,6 @@ def find_builds_for_app(app_path, work_dir, build_dir, build_log, target_arg, None, default_config_name, build_system, - build_or_not, preserve_artifacts, ) ] @@ -296,10 +293,10 @@ def main(): apps = [{"app_dir": app_dir, "build": True, "preserve": True} for app_dir in app_dirs] if not apps: - logging.critical("No apps found") - raise SystemExit(1) - logging.info("Found {} apps".format(len(apps))) + logging.warning("No apps found") + SystemExit(0) + logging.info("Found {} apps".format(len(apps))) apps.sort(key=lambda x: x["app_dir"]) # Find compatible configurations of each app, collect them as BuildItems @@ -314,7 +311,6 @@ def main(): args.target or app["target"], args.build_system or app["build_system"], config_rules, - app["build"], app["preserve"], ) logging.info("Found {} builds".format(len(build_items))) diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index 31bbad247..a068f5f5b 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -71,7 +71,6 @@ class BuildItem(object): sdkconfig_path, config_name, build_system, - build_or_not, preserve_artifacts, ): # These internal variables store the paths with environment variables and placeholders; @@ -86,7 +85,6 @@ class BuildItem(object): self.target = target self.build_system = build_system - self.build = build_or_not self.preserve = preserve_artifacts self._app_name = os.path.basename(os.path.normpath(app_path)) @@ -160,7 +158,6 @@ class BuildItem(object): "config": self.config_name, "target": self.target, "verbose": self.verbose, - "build": self.build, "preserve": self.preserve, }) @@ -179,7 +176,6 @@ class BuildItem(object): config_name=d["config"], target=d["target"], build_system=d["build_system"], - build_or_not=d["build"], preserve_artifacts=d["preserve"] ) result.verbose = d["verbose"] From 34f459c1e12f1fa61c07afc996914c47ab39f47f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 6 Jul 2020 19:05:39 +0800 Subject: [PATCH 14/15] remove this line. it will block pipelines without unit test --- tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index b04a7654f..e9c012e6d 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -171,7 +171,6 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): test_cases.extend(get_test_cases_from_yml(self.test_case_path)) else: print("Test case path is invalid. Should only happen when use @bot to skip unit test.") - raise SystemExit(1) # filter keys are lower case. Do map lower case keys with original keys. try: From 744ec42e9fa842ee9d82d483fb7f775ffa344701 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 7 Jul 2020 16:25:57 +0800 Subject: [PATCH 15/15] scan test when run regular_test --- tools/ci/config/pre_check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index a300f4413..97c01f4fe 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -204,6 +204,7 @@ scan_tests: only: variables: - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_REGULAR_TEST - $BOT_LABEL_EXAMPLE_TEST - $BOT_LABEL_CUSTOM_TEST artifacts: