Merge branch 'feature/test_apps_v2' into 'master'

test apps (2nd attempt)

Closes IDF-641

See merge request espressif/esp-idf!7084

tools/ci/build_test_apps.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+# Build test apps
+#
+# Runs as part of CI process.
+#
+
+# -----------------------------------------------------------------------------
+# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
+
+if [[ ! -z ${DEBUG_SHELL} ]]
+then
+  set -x # Activate the expand mode if DEBUG is anything but empty.
+fi
+
+set -o errexit # Exit if command failed.
+set -o pipefail # Exit if pipe failed.
+
+export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
+
+# -----------------------------------------------------------------------------
+
+die() {
+    echo "${1:-"Unknown Error"}" 1>&2
+    exit 1
+}
+
+[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
+[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
+[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
+[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
+[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
+[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
+
+if [ -z ${CI_NODE_TOTAL} ]; then
+    CI_NODE_TOTAL=1
+    echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
+fi
+if [ -z ${CI_NODE_INDEX} ]; then
+    # Gitlab uses a 1-based index
+    CI_NODE_INDEX=1
+    echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
+fi
+
+
+set -o nounset # Exit if variable not set.
+
+# Convert LOG_PATH to relative, to make the json file less verbose.
+LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH})
+BUILD_PATH=$(realpath --relative-to ${IDF_PATH} ${BUILD_PATH})
+
+ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
+JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
+mkdir -p "${BUILD_PATH}/example_builds"
+
+echo "build_examples running for target $IDF_TARGET"
+
+cd ${IDF_PATH}
+
+# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage
+# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
+
+# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
+
+${IDF_PATH}/tools/find_apps.py tools/test_apps \
+    -vv \
+    --format json \
+    --build-system cmake \
+    --target ${IDF_TARGET} \
+    --recursive \
+    --work-dir "${BUILD_PATH}/@f/@w/@t" \
+    --build-dir build \
+    --build-log "${LOG_PATH}/@f.txt" \
+    --output ${ALL_BUILD_LIST_JSON} \
+    --config 'sdkconfig.ci=default' \
+    --config 'sdkconfig.ci.*=' \
+    --config '=default' \
+
+# --config rules above explained:
+# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
+# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
+# 3. If none of the above exist, build the default configuration under the name "default"
+# --work-dir and --build-log above uses "placeholders" @x:
+# - @f: full path to the test with slashes replaced with underscores
+# - @w: wildcard used as config name
+# - @t: target name
+# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
+
+# The part below is where the actual builds happen
+
+${IDF_PATH}/tools/build_apps.py \
+    -vv \
+    --format json \
+    --keep-going \
+    --parallel-count ${CI_NODE_TOTAL} \
+    --parallel-index ${CI_NODE_INDEX} \
+    --output-build-list ${JOB_BUILD_LIST_JSON} \
+    ${ALL_BUILD_LIST_JSON}\
+
+
+# Check for build warnings
+${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}

tools/ci/config/assign-test.yml

@@ -5,20 +5,23 @@ assign_test:
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   stage: assign_test
   # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
+  # we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
   dependencies:
     - build_ssc_esp32
     - build_esp_idf_tests_cmake
   variables:
     SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
+    TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
   artifacts:
     paths:
       - components/idf_test/*/CIConfigs
       - components/idf_test/*/TC.sqlite
       - $EXAMPLE_CONFIG_OUTPUT_PATH
+      - $TEST_APP_CONFIG_OUTPUT_PATH
       - build_examples/artifact_index.json
+      - build_test_apps/artifact_index.json
     expire_in: 1 week
   only:
     variables:
@@ -26,9 +29,12 @@ assign_test:
       - $BOT_LABEL_UNIT_TEST
       - $BOT_LABEL_INTEGRATION_TEST
       - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_CUSTOM_TEST
   script:
     # assign example tests
     - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    # assign test apps
+    - python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
     # assign unit test cases
     - python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
     # clone test script to assign tests

tools/ci/config/build.yml

@@ -184,6 +184,49 @@ build_examples_cmake_esp32s2:
   variables:
     IDF_TARGET: esp32s2
 
+.build_test_apps: &build_test_apps
+  extends: .build_template
+  stage: build
+  artifacts:
+    when: always
+    paths:
+      - build_test_apps/list.json
+      - build_test_apps/list_job_*.json
+      - build_test_apps/*/*/*/build/*.bin
+      - build_test_apps/*/*/*/sdkconfig
+      - build_test_apps/*/*/*/build/*.elf
+      - build_test_apps/*/*/*/build/*.map
+      - build_test_apps/*/*/*/build/flasher_args.json
+      - build_test_apps/*/*/*/build/bootloader/*.bin
+      - build_test_apps/*/*/*/build/partition_table/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_test_apps"
+    BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    - mkdir -p ${BUILD_PATH}
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_test_apps.sh
+
+build_test_apps_esp32:
+  extends: .build_test_apps
+  variables:
+    IDF_TARGET: esp32
+
+build_test_apps_esp32s2:
+  extends: .build_test_apps
+  variables:
+    IDF_TARGET: esp32s2
+
+
 # If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
 .build_docs_template: &build_docs_template

tools/ci/config/target-test.yml

@@ -83,6 +83,29 @@
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
 
+.test_app_template:
+  extends: .example_test_template
+  stage: target_test
+  dependencies:
+    - assign_test
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_CUSTOM_TEST
+      - $BOT_LABEL_EXAMPLE_TEST
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+
 .unit_test_template:
   extends: .example_test_template
   stage: target_test
@@ -279,6 +302,12 @@ example_test_010:
     - ESP32
     - Example_ExtFlash
 
+test_app_test_001:
+  extends: .test_app_template
+  tags:
+    - ESP32
+    - test_jtag_arm
+
 example_test_011:
   extends: .example_debug_template
   tags:

tools/ci/executable-list.txt

@@ -32,6 +32,7 @@ tools/check_python_dependencies.py
 tools/ci/apply_bot_filter.py
 tools/ci/build_examples.sh
 tools/ci/build_examples_cmake.sh
+tools/ci/build_test_apps.sh
 tools/ci/check-executable.sh
 tools/ci/check-line-endings.sh
 tools/ci/check_build_warnings.py

tools/ci/python_packages/tiny_test_fw/Utility/CIAssignTest.py

@@ -148,6 +148,7 @@ class AssignTest(object):
 
     def __init__(self, test_case_path, ci_config_file, case_group=Group):
         self.test_case_path = test_case_path
+        self.test_case_file_pattern = None
         self.test_cases = []
         self.jobs = self._parse_gitlab_ci_config(ci_config_file)
         self.case_group = case_group
@@ -177,7 +178,7 @@ class AssignTest(object):
         job_list.sort(key=lambda x: x["name"])
         return job_list
 
-    def _search_cases(self, test_case_path, case_filter=None):
+    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
         """
         :param test_case_path: path contains test case folder
         :param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
@@ -186,7 +187,7 @@ class AssignTest(object):
         _case_filter = self.DEFAULT_FILTER.copy()
         if case_filter:
             _case_filter.update(case_filter)
-        test_methods = SearchCases.Search.search_test_cases(test_case_path)
+        test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern)
         return CaseConfig.filter_test_cases(test_methods, _case_filter)
 
     def _group_cases(self):
@@ -276,7 +277,7 @@ class AssignTest(object):
         failed_to_assign = []
         assigned_groups = []
         case_filter = self._apply_bot_filter()
-        self.test_cases = self._search_cases(self.test_case_path, case_filter)
+        self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern)
         self._apply_bot_test_count()
         test_groups = self._group_cases()
 

tools/ci/python_packages/tiny_test_fw/Utility/SearchCases.py

@@ -93,14 +93,14 @@ class Search(object):
         return replicated_cases
 
     @classmethod
-    def search_test_cases(cls, test_case):
+    def search_test_cases(cls, test_case, test_case_file_pattern=None):
         """
         search all test cases from a folder or file, and then do case replicate.
 
         :param test_case: test case file(s) path
         :return: a list of replicated test methods
         """
-        test_case_files = cls._search_test_case_files(test_case, cls.TEST_CASE_FILE_PATTERN)
+        test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN)
         test_cases = []
         for test_case_file in test_case_files:
             test_cases += cls._search_cases_from_file(test_case_file)

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -25,25 +25,34 @@ import json
 import gitlab_api
 from tiny_test_fw.Utility import CIAssignTest
 
-
-EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
 IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
-if IDF_PATH_FROM_ENV:
-    ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
-                                       "build_examples", "artifact_index.json")
-else:
-    ARTIFACT_INDEX_FILE = "artifact_index.json"
 
 
 class ExampleGroup(CIAssignTest.Group):
     SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
+    BUILD_LOCAL_DIR = "build_examples"
+    BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
+
+
+class TestAppsGroup(ExampleGroup):
+    BUILD_LOCAL_DIR = "build_test_apps"
+    BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
 
 
 class CIExampleAssignTest(CIAssignTest.AssignTest):
     CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
 
 
-def create_artifact_index_file(project_id=None, pipeline_id=None):
+def get_artifact_index_file(case_group=ExampleGroup):
+    if IDF_PATH_FROM_ENV:
+        artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
+                                           case_group.BUILD_LOCAL_DIR, "artifact_index.json")
+    else:
+        artifact_index_file = "artifact_index.json"
+    return artifact_index_file
+
+
+def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
     if project_id is None:
         project_id = os.getenv("CI_PROJECT_ID")
     if pipeline_id is None:
@@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
     artifact_index_list = []
 
     def format_build_log_path():
-        return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
+        parallel = job_info["parallel_num"]    # Could be None if "parallel_num" not defined for the job
+        return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
 
-    for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
+    for build_job_name in case_group.BUILD_JOB_NAMES:
         job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
         for job_info in job_info_list:
             raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
@@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
             for build_info in build_info_list:
                 build_info["ci_job_id"] = job_info["id"]
                 artifact_index_list.append(build_info)
+    artifact_index_file = get_artifact_index_file(case_group=case_group)
     try:
-        os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
+        os.makedirs(os.path.dirname(artifact_index_file))
     except OSError:
         # already created
         pass
 
-    with open(ARTIFACT_INDEX_FILE, "w") as f:
+    with open(artifact_index_file, "w") as f:
         json.dump(artifact_index_list, f)
 
 
@@ -82,9 +93,22 @@ if __name__ == '__main__':
                         help="output path of config files")
     parser.add_argument("--pipeline_id", "-p", type=int, default=None,
                         help="pipeline_id")
+    parser.add_argument("--job-prefix",
+                        help="prefix of the test job name in CI yml file")
+    parser.add_argument("--test-case-file-pattern",
+                        help="file name pattern used to find Python test case files")
+    parser.add_argument('--custom-group',
+                        help='select custom-group for the test cases, if other than ExampleTest',
+                        choices=['example','test-apps'], default='example')
+
     args = parser.parse_args()
 
-    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
+    if args.job_prefix:
+        CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
+
+    case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
+
+    assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
     assign_test.assign_cases()
     assign_test.output_configs(args.output_path)
-    create_artifact_index_file()
+    create_artifact_index_file(case_group=case_group)

tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py

@@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
     def __init__(self, test_case_path, ci_config_file):
         CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
 
-    def _search_cases(self, test_case_path, case_filter=None):
+    def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
         """
         For unit test case, we don't search for test functions.
         The unit test cases is stored in a yaml file which is created in job build-idf-test.

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -310,7 +310,7 @@ class Example(IDFApp):
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
+    def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
         # build folder of example path
         path = os.path.join(self.idf_path, app_path, "build")
         if os.path.exists(path):
@@ -327,7 +327,7 @@ class Example(IDFApp):
         # (see tools/ci/build_examples_cmake.sh)
         # For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
         app_path_underscored = app_path.replace(os.path.sep, "_")
-        example_path = os.path.join(self.idf_path, "build_examples")
+        example_path = os.path.join(self.idf_path, local_build_dir)
         for dirpath in os.listdir(example_path):
             if os.path.basename(dirpath) == app_path_underscored:
                 path = os.path.join(example_path, dirpath, config_name, target, "build")
@@ -341,7 +341,8 @@ class Example(IDFApp):
         if path:
             return path
         else:
-            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+            artifacts = Artifacts(self.idf_path,
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
                                   app_path, config_name, target)
             path = artifacts.download_artifacts()
             if path:
@@ -369,7 +370,8 @@ class LoadableElfExample(Example):
         if path:
             return path
         else:
-            artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+            artifacts = Artifacts(self.idf_path,
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
                                   app_path, config_name, target)
             path = artifacts.download_artifact_files(self.app_files)
             if path:
@@ -402,6 +404,22 @@ class UT(IDFApp):
         raise OSError("Failed to get unit-test-app binary path")
 
 
+class TestApp(Example):
+    def get_binary_path(self, app_path, config_name=None, target=None):
+        path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
+        if path:
+            return path
+        else:
+            artifacts = Artifacts(self.idf_path,
+                                  CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
+                                  app_path, config_name, target)
+            path = artifacts.download_artifacts()
+            if path:
+                return os.path.join(self.idf_path, path)
+            else:
+                raise OSError("Failed to find example binary")
+
+
 class SSC(IDFApp):
     def get_binary_path(self, app_path, config_name=None, target=None):
         # TODO: to implement SSC get binary path

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -15,7 +15,7 @@ import os
 import re
 
 from tiny_test_fw import TinyFW, Utility
-from .IDFApp import IDFApp, Example, LoadableElfExample, UT  # noqa: export all Apps for users
+from .IDFApp import IDFApp, Example, LoadableElfExample, UT, TestApp  # noqa: export all Apps for users
 from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT  # noqa: export DUTs for users
 
 
@@ -88,6 +88,41 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
     return test
 
 
+def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
+                    level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
+
+    """
+    decorator for idf custom tests (with default values for some keyword args).
+
+    :param app: test application class
+    :param dut: dut class
+    :param chip: chip supported, string or tuple
+    :param module: module, string
+    :param execution_time: execution time in minutes, int
+    :param level: test level, could be used to filter test cases, string
+    :param erase_nvs: if need to erase_nvs in DUT.start_app()
+    :param config_name: if specified, name of the app configuration
+    :param group: identifier to group custom tests (unused for now, defaults to "test-apps")
+    :param kwargs: other keyword args
+    :return: test method
+    """
+    try:
+        # try to config the default behavior of erase nvs
+        dut.ERASE_NVS = erase_nvs
+    except AttributeError:
+        pass
+
+    original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
+                                         execution_time=execution_time, level=level, **kwargs)
+
+    def test(func):
+        test_func = original_method(func)
+        test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
+        return test_func
+
+    return test
+
+
 def log_performance(item, value):
     """
     do print performance with pre-defined format to console

tools/test_apps/README.md

@@ -0,0 +1,32 @@
+# Test Apps
+
+This directory contains a set of ESP-IDF projects to be used as tests only, which aim to exercise various
+configuration of components to check completely arbitrary functionality should it be building only, executing under
+various conditions or combination with other components, including custom test frameworks.
+
+The test apps are not intended to demonstrate the ESP-IDF functionality in any way.
+
+# Test Apps projects
+
+Test applications are treated the same way as ESP-IDF examples, so each project directory shall contain
+* Build recipe in cmake and the main component with app sources
+* Configuration files
+  - `sdkconfig.ci` - Default configuration for the project
+  - `sdkconfig.ci.<CONFIG>` - Other configurations, where `<CONFIG>` indicates name of the configuration
+* Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only)
+  - test file in the project dir must end with `_test.py`, by should be named  `app_test.py`
+  - test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")`
+
+# Test Apps layout
+
+The test apps should be grouped into subdirectories by category. Categories are:
+* `protocols` contains test of protocol interactions.
+* `network` contains system network tests
+* `system` contains tests on the internal chip features, debugging and development tools.
+
+# Test Apps local execution
+
+* Append relevant `sdkconfig.ci.<CONFIG>` to the sdkconfig for the configuration under test
+* Run `idf.py menuconfig` to configure local project attributes
+* Run `idf.py build` to build the test app
+* Run `python app_test.py` to run the test locally

tools/test_apps/system/startup/CMakeLists.txt

@@ -0,0 +1,6 @@
+# The following lines of boilerplate have to be in your project's
+# CMakeLists in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.5)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(test_startup)

tools/test_apps/system/startup/README.txt

@@ -0,0 +1,4 @@
+This project tests if the app can start up in a certain configuration.
+To add new configuration, create one more sdkconfig.ci.NAME file in this directory.
+
+If you need to test for anything other than app starting up, create another test project.

tools/test_apps/system/startup/app_test.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import os
+import glob
+import ttfw_idf
+from tiny_test_fw import Utility
+
+
+@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps")
+def test_startup(env, extra_data):
+    config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
+    config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]
+    for name in config_names:
+        Utility.console_log("Checking config \"{}\"... ".format(name), end="")
+        dut = env.get_dut("startup", "tools/test_apps/system/startup", app_config_name=name)
+        dut.start_app()
+        dut.expect("app_main running")
+        env.close_dut(dut.name)
+        Utility.console_log("done")
+
+
+if __name__ == '__main__':
+    test_startup()

tools/test_apps/system/startup/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "test_startup_main.c"
+                       INCLUDE_DIRS ".")

tools/test_apps/system/startup/main/test_startup_main.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+void app_main(void)
+{
+    printf("app_main running\n");
+}

tools/test_apps/system/startup/sdkconfig.ci.flash_80m_qio

@@ -0,0 +1,2 @@
+CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
+CONFIG_ESPTOOLPY_FLASHMODE_QIO=y