From 010d7e90238388aba4a484fdf31f3d76e452b101 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Fri, 19 Jun 2020 18:58:03 +0800 Subject: [PATCH] apply to make build system --- tools/ci/config/build.yml | 5 +-- tools/ci/config/pre_check.yml | 3 +- .../python_packages/ttfw_idf/CIScanTests.py | 36 +++++++++++-------- tools/find_build_apps/cmake.py | 30 ++++++++++++++++ tools/find_build_apps/common.py | 29 ++------------- tools/find_build_apps/make.py | 6 +++- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 6b10837e0..200becc2c 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -96,9 +96,6 @@ build_esp_idf_tests_cmake_esp32s2: - mkdir ${BUILD_PATH} - mkdir -p ${LOG_PATH} - ${IDF_PATH}/tools/ci/build_examples.sh - # Check if the tests demand Make built binaries. If not, delete them - - if [ ${EXAMPLE_TEST_BUILD_SYSTEM} == "cmake" ]; then exit 0; fi - - rm -rf ${BUILD_PATH} build_examples_make: extends: .build_examples_template @@ -113,6 +110,7 @@ build_examples_make: BUILD_PATH: "${CI_PROJECT_DIR}/build_examples_make" EXAMPLE_TEST_BUILD_SYSTEM: "make" IDF_TARGET: "esp32" # currently we only support esp32 + SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json only: refs: - master @@ -146,7 +144,6 @@ build_examples_make: EXAMPLE_TEST_BUILD_SYSTEM: "cmake" SCAN_EXAMPLE_TEST_JSON: ${CI_PROJECT_DIR}/examples/test_configs/scan_${IDF_TARGET}.json - build_examples_cmake_esp32: extends: .build_examples_cmake variables: diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml index a7e971df7..7d2b80ec6 100644 --- a/tools/ci/config/pre_check.yml +++ b/tools/ci/config/pre_check.yml @@ -215,5 +215,6 @@ scan_tests: TEST_APPS_TEST_DIR: ${CI_PROJECT_DIR}/tools/test_apps TEST_APPS_OUTPUT_DIR: ${CI_PROJECT_DIR}/tools/test_apps/test_configs script: - - python $CI_SCAN_TESTS_PY example_test $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b make $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR + - python $CI_SCAN_TESTS_PY example_test -b cmake $EXAMPLE_TEST_DIR -c $TEST_CONFIG_FILE -o $EXAMPLE_TEST_OUTPUT_DIR - python $CI_SCAN_TESTS_PY test_apps $TEST_APPS_TEST_DIR -c $TEST_CONFIG_FILE -o $TEST_APPS_OUTPUT_DIR diff --git a/tools/ci/python_packages/ttfw_idf/CIScanTests.py b/tools/ci/python_packages/ttfw_idf/CIScanTests.py index a6d62a9b1..fbac2bb9c 100644 --- a/tools/ci/python_packages/ttfw_idf/CIScanTests.py +++ b/tools/ci/python_packages/ttfw_idf/CIScanTests.py @@ -6,7 +6,7 @@ import re from collections import defaultdict from find_apps import find_apps -from find_build_apps import CMakeBuildSystem +from find_build_apps import BUILD_SYSTEMS, BUILD_SYSTEM_CMAKE from ttfw_idf.CIAssignExampleTest import CIExampleAssignTest, TestAppsGroup, ExampleGroup VALID_TARGETS = [ @@ -54,13 +54,14 @@ def main(): common = argparse.ArgumentParser(add_help=False) common.add_argument('paths', type=str, nargs='+', help="One or more app paths") - common.add_argument('-c', '--ci_config_file', type=str, required=True, + common.add_argument('-b', '--build-system', choices=BUILD_SYSTEMS.keys(), default=BUILD_SYSTEM_CMAKE) + common.add_argument('-c', '--ci-config-file', type=str, required=True, help="gitlab ci config target-test file") - common.add_argument('-o', '--output_path', type=str, required=True, + common.add_argument('-o', '--output-path', type=str, required=True, help="output path of the scan result") - common.add_argument('-p', '--preserve-all', action="store_true", + common.add_argument('--preserve', action="store_true", help='add this flag to preserve artifacts for all apps') - common.add_argument('-b', '--build-all', action="store_true", + common.add_argument('--build-all', action="store_true", help='add this flag to build all apps') actions.add_parser('example_test', parents=[common]) @@ -90,8 +91,8 @@ def main(): ''' { : { - 'test_case_apps': [], - 'standalone_apps': [], + 'test_case_apps': [], # which is used in target tests + 'standalone_apps': [], # which is not }, ... } @@ -100,6 +101,9 @@ def main(): # store the test cases dir, exclude these folders when scan for standalone apps exclude_apps = [] + build_system = args.build_system.lower() + build_system_class = BUILD_SYSTEMS[build_system] + for target in VALID_TARGETS: target_dict = scan_info_dict[target] test_case_apps = target_dict['test_case_apps'] = set() @@ -108,32 +112,36 @@ def main(): app_target = case.case_info['target'] if app_target.lower() != target.lower(): continue - test_case_apps.update(find_apps(CMakeBuildSystem, app_dir, True, [], target.lower())) + test_case_apps.update(find_apps(build_system_class, app_dir, True, [], target.lower())) exclude_apps.append(app_dir) for target in VALID_TARGETS: target_dict = scan_info_dict[target] standalone_apps = target_dict['standalone_apps'] = set() for path in args.paths: - standalone_apps.update(find_apps(CMakeBuildSystem, path, True, exclude_apps, target.lower())) + standalone_apps.update(find_apps(build_system_class, path, True, exclude_apps, target.lower())) build_all = _judge_build_all(args.build_all) + test_case_apps_preserve_default = True if build_system == 'cmake' else False + for target in VALID_TARGETS: apps = [] for app_dir in scan_info_dict[target]['test_case_apps']: apps.append({ 'app_dir': app_dir, 'build': True, - 'preserve': True, + 'preserve': args.preserve or test_case_apps_preserve_default }) for app_dir in scan_info_dict[target]['standalone_apps']: apps.append({ 'app_dir': app_dir, - 'build': build_all, - 'preserve': args.preserve_all and build_all, # you can't preserve the artifacts if you don't build them right? + 'build': build_all if build_system == 'cmake' else True, + 'preserve': (args.preserve and build_all) if build_system == 'cmake' else False }) - with open(os.path.join(args.output_path, 'scan_{}.json'.format(target.lower())), 'w') as fw: - fw.writelines([json.dumps(app) + '\n' for app in apps]) + output_path = os.path.join(args.output_path, 'scan_{}_{}.json'.format(target.lower(), build_system)) + if apps: + with open(output_path, 'w') as fw: + fw.writelines([json.dumps(app) + '\n' for app in apps]) if __name__ == '__main__': diff --git a/tools/find_build_apps/cmake.py b/tools/find_build_apps/cmake.py index ab4a842ad..56b57c864 100644 --- a/tools/find_build_apps/cmake.py +++ b/tools/find_build_apps/cmake.py @@ -93,3 +93,33 @@ class CMakeBuildSystem(BuildSystem): if CMAKE_PROJECT_LINE not in cmakelists_file_content: return False return True + + @staticmethod + def supported_targets(app_path): + formal_to_usual = { + 'ESP32': 'esp32', + 'ESP32-S2': 'esp32s2', + } + + readme_file_content = BuildSystem._read_readme(app_path) + if not readme_file_content: + return None + match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) + if not match: + return None + if len(match) > 1: + raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) + support_str = match[0].strip() + + targets = [] + for part in support_str.split('|'): + for inner in part.split(' '): + inner = inner.strip() + if not inner: + continue + elif inner in formal_to_usual: + targets.append(formal_to_usual[inner]) + else: + raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" + .format(inner, app_path, ', '.join(formal_to_usual.keys()))) + return targets diff --git a/tools/find_build_apps/common.py b/tools/find_build_apps/common.py index dcb39432d..31bbad247 100644 --- a/tools/find_build_apps/common.py +++ b/tools/find_build_apps/common.py @@ -341,34 +341,9 @@ class BuildSystem(object): return readme_file.read() @staticmethod + @abstractmethod def supported_targets(app_path): - formal_to_usual = { - 'ESP32': 'esp32', - 'ESP32-S2': 'esp32s2', - } - - readme_file_content = BuildSystem._read_readme(app_path) - if not readme_file_content: - return None - match = re.findall(BuildSystem.SUPPORTED_TARGETS_REGEX, readme_file_content) - if not match: - return None - if len(match) > 1: - raise NotImplementedError("Can't determine the value of SUPPORTED_TARGETS in {}".format(app_path)) - support_str = match[0].strip() - - targets = [] - for part in support_str.split('|'): - for inner in part.split(' '): - inner = inner.strip() - if not inner: - continue - elif inner in formal_to_usual: - targets.append(formal_to_usual[inner]) - else: - raise NotImplementedError("Can't recognize value of target {} in {}, now we only support '{}'" - .format(inner, app_path, ', '.join(formal_to_usual.keys()))) - return targets + pass class BuildError(RuntimeError): diff --git a/tools/find_build_apps/make.py b/tools/find_build_apps/make.py index a8b153fd0..e7b885732 100644 --- a/tools/find_build_apps/make.py +++ b/tools/find_build_apps/make.py @@ -1,8 +1,8 @@ import logging import os +import shlex import subprocess import sys -import shlex from .common import BuildSystem, BuildError @@ -58,3 +58,7 @@ class MakeBuildSystem(BuildSystem): if MAKE_PROJECT_LINE not in makefile_content: return False return True + + @staticmethod + def supported_targets(app_path): + return ['esp32']