From e63764b46806d5ff5c78d68574d84e1b13693039 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Sun, 20 Oct 2019 20:55:11 +0200 Subject: [PATCH 1/3] CI: test-apps introduction Introducing feature of adding arbitrary projects which could be build or executed in the CI for the only purpose as testing Closes IDF-641 --- tools/ci/build_test_apps.sh | 96 +++++++++++++++++++ tools/ci/config/assign-test.yml | 7 +- tools/ci/config/build.yml | 44 +++++++++ tools/ci/config/target-test.yml | 30 ++++++ tools/ci/executable-list.txt | 1 + .../tiny_test_fw/Utility/CIAssignTest.py | 7 +- .../tiny_test_fw/Utility/SearchCases.py | 4 +- .../ttfw_idf/CIAssignExampleTest.py | 11 +++ tools/ci/python_packages/ttfw_idf/IDFApp.py | 30 ++++++ tools/ci/python_packages/ttfw_idf/__init__.py | 34 ++++++- tools/test_apps/startup/CMakeLists.txt | 6 ++ tools/test_apps/startup/README.txt | 4 + tools/test_apps/startup/main/CMakeLists.txt | 2 + .../startup/main/test_startup_main.c | 6 ++ tools/test_apps/startup/sdkconfig.ci.default | 0 .../startup/sdkconfig.ci.flash_80m_qio | 2 + tools/test_apps/startup/test.py | 35 +++++++ 17 files changed, 312 insertions(+), 7 deletions(-) create mode 100755 tools/ci/build_test_apps.sh create mode 100644 tools/test_apps/startup/CMakeLists.txt create mode 100644 tools/test_apps/startup/README.txt create mode 100644 tools/test_apps/startup/main/CMakeLists.txt create mode 100644 tools/test_apps/startup/main/test_startup_main.c create mode 100644 tools/test_apps/startup/sdkconfig.ci.default create mode 100644 tools/test_apps/startup/sdkconfig.ci.flash_80m_qio create mode 100644 tools/test_apps/startup/test.py diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh new file mode 100755 index 000000000..f014e6219 --- /dev/null +++ b/tools/ci/build_test_apps.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Build test apps +# +# Runs as part of CI process. +# + +# ----------------------------------------------------------------------------- +# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d). + +if [[ ! -z ${DEBUG_SHELL} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. + +export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH" + +# ----------------------------------------------------------------------------- + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${IDF_PATH} ] && die "IDF_PATH is not set" +[ -z ${LOG_PATH} ] && die "LOG_PATH is not set" +[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set" +[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set" +[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH} +[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH} + +if [ -z ${CI_NODE_TOTAL} ]; then + CI_NODE_TOTAL=1 + echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}" +fi +if [ -z ${CI_NODE_INDEX} ]; then + # Gitlab uses a 1-based index + CI_NODE_INDEX=1 + echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}" +fi + + +set -o nounset # Exit if variable not set. + +# Convert LOG_PATH to relative, to make the json file less verbose. +LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH}) +BUILD_PATH=$(realpath --relative-to ${IDF_PATH} ${BUILD_PATH}) + +ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json" +JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json" +mkdir -p "${BUILD_PATH}/example_builds" + +echo "build_examples running for target $IDF_TARGET" + +cd ${IDF_PATH} + +# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage +# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. + +# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. + +${IDF_PATH}/tools/find_apps.py tools/test_apps \ + -vv \ + --format json \ + --build-system cmake \ + --target ${IDF_TARGET} \ + --recursive \ + --build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \ + --build-log "${LOG_PATH}/@f.txt" \ + --output ${ALL_BUILD_LIST_JSON} \ + --config 'sdkconfig.ci=default' \ + --config 'sdkconfig.ci.*=' \ + --config '=default' \ + +# --config rules above explained: +# 1. If sdkconfig.ci exists, use it build the example with configuration name "default" +# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration +# 3. If none of the above exist, build the default configuration under the name "default" + +# The part below is where the actual builds happen + +${IDF_PATH}/tools/build_apps.py \ + -vv \ + --format json \ + --keep-going \ + --parallel-count ${CI_NODE_TOTAL} \ + --parallel-index ${CI_NODE_INDEX} \ + --output-build-list ${JOB_BUILD_LIST_JSON} \ + ${ALL_BUILD_LIST_JSON}\ + + +# Check for build warnings +${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON} diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 5a7020b94..8978f7d87 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -5,19 +5,22 @@ assign_test: image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG stage: assign_test # gitlab ci do not support match job with RegEx or wildcard now in dependencies. - # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage. + # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage. dependencies: - build_ssc_esp32 - build_esp_idf_tests_cmake + - build_test_apps_esp32 variables: SUBMODULES_TO_FETCH: "components/esptool_py/esptool" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" + TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" artifacts: paths: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite - $EXAMPLE_CONFIG_OUTPUT_PATH + - $TEST_APP_CONFIG_OUTPUT_PATH - build_examples/artifact_index.json expire_in: 1 week only: @@ -29,6 +32,8 @@ assign_test: script: # assign example tests - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH + # assign test apps + - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH # assign unit test cases - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 2edfa6ec8..f0da827cf 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -184,6 +184,50 @@ build_examples_cmake_esp32s2: variables: IDF_TARGET: esp32s2 +.build_test_apps: &build_test_apps + extends: .build_template + parallel: 2 + stage: pre_build + artifacts: + when: always + paths: + - build_test_apps/list.json + - build_test_apps/list_job_*.json + - build_test_apps/*/*/*/build/*.bin + - build_test_apps/*/*/*/sdkconfig + - build_test_apps/*/*/*/build/*.elf + - build_test_apps/*/*/*/build/*.map + - build_test_apps/*/*/*/build/flasher_args.json + - build_test_apps/*/*/*/build/bootloader/*.bin + - build_test_apps/*/*/*/build/partition_table/*.bin + - $LOG_PATH + expire_in: 3 days + variables: + LOG_PATH: "$CI_PROJECT_DIR/log_test_apps" + BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps" + only: + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_BUILD + - $BOT_LABEL_INTEGRATION_TEST + - $BOT_LABEL_REGULAR_TEST + - $BOT_LABEL_WEEKEND_TEST + script: + - mkdir -p ${BUILD_PATH} + - mkdir -p ${LOG_PATH} + - ${IDF_PATH}/tools/ci/build_test_apps.sh + +build_test_apps_esp32: + extends: .build_test_apps + variables: + IDF_TARGET: esp32 + +build_test_apps_esp32s2: + extends: .build_test_apps + variables: + IDF_TARGET: esp32s2beta + + # If you want to add new build example jobs, please add it into dependencies of `.example_test_template` build_docs: diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 290165667..672c63399 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -83,6 +83,30 @@ # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE +.test_app_template: + extends: .example_test_template + stage: pre_target_test + dependencies: + - assign_test + - build_test_apps_esp32 + variables: + TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" + TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" + CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" + LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" + ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" + script: + - *define_config_file_name + # first test if config file exists, if not exist, exit 0 + - test -e $CONFIG_FILE || exit 0 + # clone test env configs + - git clone $TEST_ENV_CONFIG_REPOSITORY + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs + - cd $TEST_FW_PATH + # run test + - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE + + .unit_test_template: extends: .example_test_template stage: target_test @@ -279,6 +303,12 @@ example_test_010: - ESP32 - Example_ExtFlash +test_app_test_001: + extends: .test_app_template + tags: + - ESP32 + - test_jtag_arm + example_test_011: extends: .example_debug_template tags: diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 391d13eec..4826250f0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -33,6 +33,7 @@ tools/check_python_dependencies.py tools/ci/apply_bot_filter.py tools/ci/build_examples.sh tools/ci/build_examples_cmake.sh +tools/ci/build_test_apps.sh tools/ci/check-executable.sh tools/ci/check-line-endings.sh tools/ci/check_build_warnings.py diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py index 056d75f1c..284432b9a 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py @@ -148,6 +148,7 @@ class AssignTest(object): def __init__(self, test_case_path, ci_config_file, case_group=Group): self.test_case_path = test_case_path + self.test_case_file_pattern = None self.test_cases = [] self.jobs = self._parse_gitlab_ci_config(ci_config_file) self.case_group = case_group @@ -177,7 +178,7 @@ class AssignTest(object): job_list.sort(key=lambda x: x["name"]) return job_list - def _search_cases(self, test_case_path, case_filter=None): + def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None): """ :param test_case_path: path contains test case folder :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param. @@ -186,7 +187,7 @@ class AssignTest(object): _case_filter = self.DEFAULT_FILTER.copy() if case_filter: _case_filter.update(case_filter) - test_methods = SearchCases.Search.search_test_cases(test_case_path) + test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern) return CaseConfig.filter_test_cases(test_methods, _case_filter) def _group_cases(self): @@ -276,7 +277,7 @@ class AssignTest(object): failed_to_assign = [] assigned_groups = [] case_filter = self._apply_bot_filter() - self.test_cases = self._search_cases(self.test_case_path, case_filter) + self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern) self._apply_bot_test_count() test_groups = self._group_cases() diff --git a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py index b5e076283..984083a14 100644 --- a/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py +++ b/tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py @@ -93,14 +93,14 @@ class Search(object): return replicated_cases @classmethod - def search_test_cases(cls, test_case): + def search_test_cases(cls, test_case, test_case_file_pattern=None): """ search all test cases from a folder or file, and then do case replicate. :param test_case: test case file(s) path :return: a list of replicated test methods """ - test_case_files = cls._search_test_case_files(test_case, cls.TEST_CASE_FILE_PATTERN) + test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN) test_cases = [] for test_case_file in test_case_files: test_cases += cls._search_cases_from_file(test_case_file) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index ed27c58eb..0949c7e86 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -82,8 +82,19 @@ if __name__ == '__main__': help="output path of config files") parser.add_argument("--pipeline_id", "-p", type=int, default=None, help="pipeline_id") + parser.add_argument("--job-prefix", + help="prefix of the test job name in CI yml file") + parser.add_argument("--test-case-file-pattern", + help="file name pattern used to find Python test case files") args = parser.parse_args() + if args.job_prefix: + CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix)) + + assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) + if args.test_case_file_pattern: + assign_test.test_case_file_pattern = args.test_case_file_pattern + assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) assign_test.assign_cases() assign_test.output_configs(args.output_path) diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index 6838e75ed..92376740e 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -402,6 +402,36 @@ class UT(IDFApp): raise OSError("Failed to get unit-test-app binary path") +class TestApp(IDFApp): + def _get_sdkconfig_paths(self): + """ + overrides the parent method to provide exact path of sdkconfig for example tests + """ + return [os.path.join(self.binary_path, "..", "sdkconfig")] + + def get_binary_path(self, app_path, config_name=None): + # local build folder + path = os.path.join(self.idf_path, app_path, "build") + if os.path.exists(path): + return path + + if not config_name: + config_name = "default" + + # Search for CI build folders. + # Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target + # (see tools/ci/build_test_apps.sh) + # For example: $IDF_PATH/build_test_apps/startup/default/esp32 + app_path_underscored = app_path.replace(os.path.sep, "_") + build_root = os.path.join(self.idf_path, "build_test_apps") + for dirpath in os.listdir(build_root): + if os.path.basename(dirpath) == app_path_underscored: + path = os.path.join(build_root, dirpath, config_name, self.target, "build") + return path + + raise OSError("Failed to find test app binary") + + class SSC(IDFApp): def get_binary_path(self, app_path, config_name=None, target=None): # TODO: to implement SSC get binary path diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index f03cc71f9..46328c1c5 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -15,7 +15,7 @@ import os import re from tiny_test_fw import TinyFW, Utility -from .IDFApp import IDFApp, Example, LoadableElfExample, UT # noqa: export all Apps for users +from .IDFApp import IDFApp, Example, LoadableElfExample, UT, TestApp # noqa: export all Apps for users from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users @@ -88,6 +88,38 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio return test +def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, + level="integration", erase_nvs=True, **kwargs): + """ + decorator for testing idf unit tests (with default values for some keyword args). + + :param app: test application class + :param dut: dut class + :param chip: chip supported, string or tuple + :param module: module, string + :param execution_time: execution time in minutes, int + :param level: test level, could be used to filter test cases, string + :param erase_nvs: if need to erase_nvs in DUT.start_app() + :param kwargs: other keyword args + :return: test method + """ + try: + # try to config the default behavior of erase nvs + dut.ERASE_NVS = erase_nvs + except AttributeError: + pass + + original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module, + execution_time=execution_time, level=level, **kwargs) + + def test(func): + test_func = original_method(func) + test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"]) + return test_func + + return test + + def log_performance(item, value): """ do print performance with pre-defined format to console diff --git a/tools/test_apps/startup/CMakeLists.txt b/tools/test_apps/startup/CMakeLists.txt new file mode 100644 index 000000000..2e1c4fe17 --- /dev/null +++ b/tools/test_apps/startup/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(test_startup) diff --git a/tools/test_apps/startup/README.txt b/tools/test_apps/startup/README.txt new file mode 100644 index 000000000..a10fd3a4a --- /dev/null +++ b/tools/test_apps/startup/README.txt @@ -0,0 +1,4 @@ +This project tests if the app can start up in a certain configuration. +To add new configuration, create one more sdkconfig.ci.NAME file in this directory. + +If you need to test for anything other than app starting up, create another test project. diff --git a/tools/test_apps/startup/main/CMakeLists.txt b/tools/test_apps/startup/main/CMakeLists.txt new file mode 100644 index 000000000..53aa18c09 --- /dev/null +++ b/tools/test_apps/startup/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRCS "test_startup_main.c" + INCLUDE_DIRS ".") diff --git a/tools/test_apps/startup/main/test_startup_main.c b/tools/test_apps/startup/main/test_startup_main.c new file mode 100644 index 000000000..38dc6c993 --- /dev/null +++ b/tools/test_apps/startup/main/test_startup_main.c @@ -0,0 +1,6 @@ +#include + +void app_main(void) +{ + printf("app_main running\n"); +} diff --git a/tools/test_apps/startup/sdkconfig.ci.default b/tools/test_apps/startup/sdkconfig.ci.default new file mode 100644 index 000000000..e69de29bb diff --git a/tools/test_apps/startup/sdkconfig.ci.flash_80m_qio b/tools/test_apps/startup/sdkconfig.ci.flash_80m_qio new file mode 100644 index 000000000..447c18988 --- /dev/null +++ b/tools/test_apps/startup/sdkconfig.ci.flash_80m_qio @@ -0,0 +1,2 @@ +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y diff --git a/tools/test_apps/startup/test.py b/tools/test_apps/startup/test.py new file mode 100644 index 000000000..75d067d6b --- /dev/null +++ b/tools/test_apps/startup/test.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import re +import os +import sys +import glob + +try: + import IDF +except ImportError: + # This environment variable is expected on the host machine + test_fw_path = os.getenv("TEST_FW_PATH") + if test_fw_path and test_fw_path not in sys.path: + sys.path.insert(0, test_fw_path) + + import IDF + +import Utility + + +@IDF.idf_test_app_test(env_tag="test_jtag_arm") +def test_startup(env, extra_data): + config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*")) + config_names = [s.replace("sdkconfig.ci.", "") for s in config_files] + for name in config_names: + Utility.console_log("Checking config \"{}\"... ".format(name), end="") + dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name) + dut.start_app() + dut.expect("app_main running") + env.close_dut(dut.name) + Utility.console_log("done") + + +if __name__ == '__main__': + test_startup() From 692deac5aefeeb068d9c80baa0e08dd92d211495 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Mon, 27 Jan 2020 12:12:49 +0100 Subject: [PATCH 2/3] CI: assign and target-test stages updated to run test-apps in the ci --- tools/ci/build_test_apps.sh | 10 +++- tools/ci/config/assign-test.yml | 5 +- tools/ci/config/build.yml | 5 +- tools/ci/config/target-test.yml | 25 +++++----- .../ttfw_idf/CIAssignExampleTest.py | 47 ++++++++++------- .../ttfw_idf/CIAssignUnitTest.py | 2 +- tools/ci/python_packages/ttfw_idf/IDFApp.py | 50 +++++++------------ tools/ci/python_packages/ttfw_idf/__init__.py | 9 ++-- .../startup/{test.py => app_test.py} | 20 ++------ 9 files changed, 85 insertions(+), 88 deletions(-) rename tools/test_apps/startup/{test.py => app_test.py} (56%) diff --git a/tools/ci/build_test_apps.sh b/tools/ci/build_test_apps.sh index f014e6219..d8498fe63 100755 --- a/tools/ci/build_test_apps.sh +++ b/tools/ci/build_test_apps.sh @@ -57,7 +57,7 @@ echo "build_examples running for target $IDF_TARGET" cd ${IDF_PATH} -# This part of the script produces the same result for all the example build jobs. It may be moved to a separate stage +# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage # (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact. # If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py. @@ -68,7 +68,8 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ --build-system cmake \ --target ${IDF_TARGET} \ --recursive \ - --build-dir "\${IDF_PATH}/${BUILD_PATH}/@f/@w/@t/build" \ + --work-dir "${BUILD_PATH}/@f/@w/@t" \ + --build-dir build \ --build-log "${LOG_PATH}/@f.txt" \ --output ${ALL_BUILD_LIST_JSON} \ --config 'sdkconfig.ci=default' \ @@ -79,6 +80,11 @@ ${IDF_PATH}/tools/find_apps.py tools/test_apps \ # 1. If sdkconfig.ci exists, use it build the example with configuration name "default" # 2. If sdkconfig.ci.* exists, use it to build the "*" configuration # 3. If none of the above exist, build the default configuration under the name "default" +# --work-dir and --build-log above uses "placeholders" @x: +# - @f: full path to the test with slashes replaced with underscores +# - @w: wildcard used as config name +# - @t: target name +# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32 # The part below is where the actual builds happen diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 8978f7d87..4666bd9b6 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -9,7 +9,6 @@ assign_test: dependencies: - build_ssc_esp32 - build_esp_idf_tests_cmake - - build_test_apps_esp32 variables: SUBMODULES_TO_FETCH: "components/esptool_py/esptool" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" @@ -22,6 +21,7 @@ assign_test: - $EXAMPLE_CONFIG_OUTPUT_PATH - $TEST_APP_CONFIG_OUTPUT_PATH - build_examples/artifact_index.json + - build_test_apps/artifact_index.json expire_in: 1 week only: variables: @@ -29,11 +29,12 @@ assign_test: - $BOT_LABEL_UNIT_TEST - $BOT_LABEL_INTEGRATION_TEST - $BOT_LABEL_EXAMPLE_TEST + - $BOT_LABEL_CUSTOM_TEST script: # assign example tests - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH # assign test apps - - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH + - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH # assign unit test cases - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index f0da827cf..e5f9c48b1 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -186,8 +186,7 @@ build_examples_cmake_esp32s2: .build_test_apps: &build_test_apps extends: .build_template - parallel: 2 - stage: pre_build + stage: build artifacts: when: always paths: @@ -225,7 +224,7 @@ build_test_apps_esp32: build_test_apps_esp32s2: extends: .build_test_apps variables: - IDF_TARGET: esp32s2beta + IDF_TARGET: esp32s2 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template` diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 672c63399..3384d3cd0 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -85,27 +85,26 @@ .test_app_template: extends: .example_test_template - stage: pre_target_test + stage: target_test dependencies: - assign_test - - build_test_apps_esp32 + only: + refs: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + - triggers + - schedules + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_CUSTOM_TEST + - $BOT_LABEL_EXAMPLE_TEST variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps" CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" - script: - - *define_config_file_name - # first test if config file exists, if not exist, exit 0 - - test -e $CONFIG_FILE || exit 0 - # clone test env configs - - git clone $TEST_ENV_CONFIG_REPOSITORY - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - - cd $TEST_FW_PATH - # run test - - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE - .unit_test_template: extends: .example_test_template diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py index 0949c7e86..d887ae1d0 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py @@ -25,25 +25,34 @@ import json import gitlab_api from tiny_test_fw.Utility import CIAssignTest - -EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] IDF_PATH_FROM_ENV = os.getenv("IDF_PATH") -if IDF_PATH_FROM_ENV: - ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV, - "build_examples", "artifact_index.json") -else: - ARTIFACT_INDEX_FILE = "artifact_index.json" class ExampleGroup(CIAssignTest.Group): SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] + BUILD_LOCAL_DIR = "build_examples" + BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"] + + +class TestAppsGroup(ExampleGroup): + BUILD_LOCAL_DIR = "build_test_apps" + BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"] class CIExampleAssignTest(CIAssignTest.AssignTest): CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+") -def create_artifact_index_file(project_id=None, pipeline_id=None): +def get_artifact_index_file(case_group=ExampleGroup): + if IDF_PATH_FROM_ENV: + artifact_index_file = os.path.join(IDF_PATH_FROM_ENV, + case_group.BUILD_LOCAL_DIR, "artifact_index.json") + else: + artifact_index_file = "artifact_index.json" + return artifact_index_file + + +def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup): if project_id is None: project_id = os.getenv("CI_PROJECT_ID") if pipeline_id is None: @@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None): artifact_index_list = [] def format_build_log_path(): - return "build_examples/list_job_{}.json".format(job_info["parallel_num"]) + parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job + return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1) - for build_job_name in EXAMPLE_BUILD_JOB_NAMES: + for build_job_name in case_group.BUILD_JOB_NAMES: job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id) for job_info in job_info_list: raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0] @@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None): for build_info in build_info_list: build_info["ci_job_id"] = job_info["id"] artifact_index_list.append(build_info) + artifact_index_file = get_artifact_index_file(case_group=case_group) try: - os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE)) + os.makedirs(os.path.dirname(artifact_index_file)) except OSError: # already created pass - with open(ARTIFACT_INDEX_FILE, "w") as f: + with open(artifact_index_file, "w") as f: json.dump(artifact_index_list, f) @@ -86,16 +97,18 @@ if __name__ == '__main__': help="prefix of the test job name in CI yml file") parser.add_argument("--test-case-file-pattern", help="file name pattern used to find Python test case files") + parser.add_argument('--custom-group', + help='select custom-group for the test cases, if other than ExampleTest', + choices=['example','test-apps'], default='example') + args = parser.parse_args() if args.job_prefix: CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix)) - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) - if args.test_case_file_pattern: - assign_test.test_case_file_pattern = args.test_case_file_pattern + case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) + assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group) assign_test.assign_cases() assign_test.output_configs(args.output_path) - create_artifact_index_file() + create_artifact_index_file(case_group=case_group) diff --git a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py index 7b7fcb1b5..fc1de8e4c 100644 --- a/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py +++ b/tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py @@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): def __init__(self, test_case_path, ci_config_file): CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group) - def _search_cases(self, test_case_path, case_filter=None): + def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None): """ For unit test case, we don't search for test functions. The unit test cases is stored in a yaml file which is created in job build-idf-test. diff --git a/tools/ci/python_packages/ttfw_idf/IDFApp.py b/tools/ci/python_packages/ttfw_idf/IDFApp.py index 92376740e..897fc0e39 100644 --- a/tools/ci/python_packages/ttfw_idf/IDFApp.py +++ b/tools/ci/python_packages/ttfw_idf/IDFApp.py @@ -310,7 +310,7 @@ class Example(IDFApp): """ return [os.path.join(self.binary_path, "..", "sdkconfig")] - def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None): + def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"): # build folder of example path path = os.path.join(self.idf_path, app_path, "build") if os.path.exists(path): @@ -327,7 +327,7 @@ class Example(IDFApp): # (see tools/ci/build_examples_cmake.sh) # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32 app_path_underscored = app_path.replace(os.path.sep, "_") - example_path = os.path.join(self.idf_path, "build_examples") + example_path = os.path.join(self.idf_path, local_build_dir) for dirpath in os.listdir(example_path): if os.path.basename(dirpath) == app_path_underscored: path = os.path.join(example_path, dirpath, config_name, target, "build") @@ -341,7 +341,8 @@ class Example(IDFApp): if path: return path else: - artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup), app_path, config_name, target) path = artifacts.download_artifacts() if path: @@ -369,7 +370,8 @@ class LoadableElfExample(Example): if path: return path else: - artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE, + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup), app_path, config_name, target) path = artifacts.download_artifact_files(self.app_files) if path: @@ -402,34 +404,20 @@ class UT(IDFApp): raise OSError("Failed to get unit-test-app binary path") -class TestApp(IDFApp): - def _get_sdkconfig_paths(self): - """ - overrides the parent method to provide exact path of sdkconfig for example tests - """ - return [os.path.join(self.binary_path, "..", "sdkconfig")] - - def get_binary_path(self, app_path, config_name=None): - # local build folder - path = os.path.join(self.idf_path, app_path, "build") - if os.path.exists(path): +class TestApp(Example): + def get_binary_path(self, app_path, config_name=None, target=None): + path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps") + if path: return path - - if not config_name: - config_name = "default" - - # Search for CI build folders. - # Path format: $IDF_PATH/build_test_apps/app_path_with_underscores/config/target - # (see tools/ci/build_test_apps.sh) - # For example: $IDF_PATH/build_test_apps/startup/default/esp32 - app_path_underscored = app_path.replace(os.path.sep, "_") - build_root = os.path.join(self.idf_path, "build_test_apps") - for dirpath in os.listdir(build_root): - if os.path.basename(dirpath) == app_path_underscored: - path = os.path.join(build_root, dirpath, config_name, self.target, "build") - return path - - raise OSError("Failed to find test app binary") + else: + artifacts = Artifacts(self.idf_path, + CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup), + app_path, config_name, target) + path = artifacts.download_artifacts() + if path: + return os.path.join(self.idf_path, path) + else: + raise OSError("Failed to find example binary") class SSC(IDFApp): diff --git a/tools/ci/python_packages/ttfw_idf/__init__.py b/tools/ci/python_packages/ttfw_idf/__init__.py index 46328c1c5..66e12ee48 100644 --- a/tools/ci/python_packages/ttfw_idf/__init__.py +++ b/tools/ci/python_packages/ttfw_idf/__init__.py @@ -88,10 +88,11 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio return test -def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, - level="integration", erase_nvs=True, **kwargs): +def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1, + level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs): + """ - decorator for testing idf unit tests (with default values for some keyword args). + decorator for idf custom tests (with default values for some keyword args). :param app: test application class :param dut: dut class @@ -100,6 +101,8 @@ def idf_test_app_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", exec :param execution_time: execution time in minutes, int :param level: test level, could be used to filter test cases, string :param erase_nvs: if need to erase_nvs in DUT.start_app() + :param config_name: if specified, name of the app configuration + :param group: identifier to group custom tests (unused for now, defaults to "test-apps") :param kwargs: other keyword args :return: test method """ diff --git a/tools/test_apps/startup/test.py b/tools/test_apps/startup/app_test.py similarity index 56% rename from tools/test_apps/startup/test.py rename to tools/test_apps/startup/app_test.py index 75d067d6b..f6c1a779b 100644 --- a/tools/test_apps/startup/test.py +++ b/tools/test_apps/startup/app_test.py @@ -1,27 +1,15 @@ #!/usr/bin/env python -import re import os -import sys import glob - -try: - import IDF -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF - -import Utility +import ttfw_idf +from tiny_test_fw import Utility -@IDF.idf_test_app_test(env_tag="test_jtag_arm") +@ttfw_idf.idf_test_app_test(env_tag="test_jtag_arm") def test_startup(env, extra_data): config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*")) - config_names = [s.replace("sdkconfig.ci.", "") for s in config_files] + config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files] for name in config_names: Utility.console_log("Checking config \"{}\"... ".format(name), end="") dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name) From 30c6e27bcc7448cc14212c54a8007b8f32d8f8c6 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 29 Jan 2020 10:20:46 +0100 Subject: [PATCH 3/3] CI: document test-apps feature, updated test-apps structure --- tools/test_apps/README.md | 32 +++++++++++++++++++ .../{ => system}/startup/CMakeLists.txt | 0 .../test_apps/{ => system}/startup/README.txt | 0 .../{ => system}/startup/app_test.py | 4 +-- .../{ => system}/startup/main/CMakeLists.txt | 0 .../startup/main/test_startup_main.c | 0 .../{ => system}/startup/sdkconfig.ci.default | 0 .../startup/sdkconfig.ci.flash_80m_qio | 0 8 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tools/test_apps/README.md rename tools/test_apps/{ => system}/startup/CMakeLists.txt (100%) rename tools/test_apps/{ => system}/startup/README.txt (100%) rename tools/test_apps/{ => system}/startup/app_test.py (78%) rename tools/test_apps/{ => system}/startup/main/CMakeLists.txt (100%) rename tools/test_apps/{ => system}/startup/main/test_startup_main.c (100%) rename tools/test_apps/{ => system}/startup/sdkconfig.ci.default (100%) rename tools/test_apps/{ => system}/startup/sdkconfig.ci.flash_80m_qio (100%) diff --git a/tools/test_apps/README.md b/tools/test_apps/README.md new file mode 100644 index 000000000..3bb1e7e12 --- /dev/null +++ b/tools/test_apps/README.md @@ -0,0 +1,32 @@ +# Test Apps + +This directory contains a set of ESP-IDF projects to be used as tests only, which aim to exercise various +configuration of components to check completely arbitrary functionality should it be building only, executing under +various conditions or combination with other components, including custom test frameworks. + +The test apps are not intended to demonstrate the ESP-IDF functionality in any way. + +# Test Apps projects + +Test applications are treated the same way as ESP-IDF examples, so each project directory shall contain +* Build recipe in cmake and the main component with app sources +* Configuration files + - `sdkconfig.ci` - Default configuration for the project + - `sdkconfig.ci.` - Other configurations, where `` indicates name of the configuration +* Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only) + - test file in the project dir must end with `_test.py`, by should be named `app_test.py` + - test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")` + +# Test Apps layout + +The test apps should be grouped into subdirectories by category. Categories are: +* `protocols` contains test of protocol interactions. +* `network` contains system network tests +* `system` contains tests on the internal chip features, debugging and development tools. + +# Test Apps local execution + +* Append relevant `sdkconfig.ci.` to the sdkconfig for the configuration under test +* Run `idf.py menuconfig` to configure local project attributes +* Run `idf.py build` to build the test app +* Run `python app_test.py` to run the test locally diff --git a/tools/test_apps/startup/CMakeLists.txt b/tools/test_apps/system/startup/CMakeLists.txt similarity index 100% rename from tools/test_apps/startup/CMakeLists.txt rename to tools/test_apps/system/startup/CMakeLists.txt diff --git a/tools/test_apps/startup/README.txt b/tools/test_apps/system/startup/README.txt similarity index 100% rename from tools/test_apps/startup/README.txt rename to tools/test_apps/system/startup/README.txt diff --git a/tools/test_apps/startup/app_test.py b/tools/test_apps/system/startup/app_test.py similarity index 78% rename from tools/test_apps/startup/app_test.py rename to tools/test_apps/system/startup/app_test.py index f6c1a779b..ebd4cf330 100644 --- a/tools/test_apps/startup/app_test.py +++ b/tools/test_apps/system/startup/app_test.py @@ -6,13 +6,13 @@ import ttfw_idf from tiny_test_fw import Utility -@ttfw_idf.idf_test_app_test(env_tag="test_jtag_arm") +@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps") def test_startup(env, extra_data): config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*")) config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files] for name in config_names: Utility.console_log("Checking config \"{}\"... ".format(name), end="") - dut = env.get_dut("startup", "tools/test_apps/startup", app_config_name=name) + dut = env.get_dut("startup", "tools/test_apps/system/startup", app_config_name=name) dut.start_app() dut.expect("app_main running") env.close_dut(dut.name) diff --git a/tools/test_apps/startup/main/CMakeLists.txt b/tools/test_apps/system/startup/main/CMakeLists.txt similarity index 100% rename from tools/test_apps/startup/main/CMakeLists.txt rename to tools/test_apps/system/startup/main/CMakeLists.txt diff --git a/tools/test_apps/startup/main/test_startup_main.c b/tools/test_apps/system/startup/main/test_startup_main.c similarity index 100% rename from tools/test_apps/startup/main/test_startup_main.c rename to tools/test_apps/system/startup/main/test_startup_main.c diff --git a/tools/test_apps/startup/sdkconfig.ci.default b/tools/test_apps/system/startup/sdkconfig.ci.default similarity index 100% rename from tools/test_apps/startup/sdkconfig.ci.default rename to tools/test_apps/system/startup/sdkconfig.ci.default diff --git a/tools/test_apps/startup/sdkconfig.ci.flash_80m_qio b/tools/test_apps/system/startup/sdkconfig.ci.flash_80m_qio similarity index 100% rename from tools/test_apps/startup/sdkconfig.ci.flash_80m_qio rename to tools/test_apps/system/startup/sdkconfig.ci.flash_80m_qio