Merge branch 'feature/test_apps_v2' into 'master'
test apps (2nd attempt) Closes IDF-641 See merge request espressif/esp-idf!7084
This commit is contained in:
commit
3017bfb8e3
19 changed files with 360 additions and 26 deletions
102
tools/ci/build_test_apps.sh
Executable file
102
tools/ci/build_test_apps.sh
Executable file
|
@ -0,0 +1,102 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Build test apps
|
||||||
|
#
|
||||||
|
# Runs as part of CI process.
|
||||||
|
#
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Safety settings (see https://gist.github.com/ilg-ul/383869cbb01f61a51c4d).
|
||||||
|
|
||||||
|
if [[ ! -z ${DEBUG_SHELL} ]]
|
||||||
|
then
|
||||||
|
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -o errexit # Exit if command failed.
|
||||||
|
set -o pipefail # Exit if pipe failed.
|
||||||
|
|
||||||
|
export PATH="$IDF_PATH/tools/ci:$IDF_PATH/tools:$PATH"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "${1:-"Unknown Error"}" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ -z ${IDF_PATH} ] && die "IDF_PATH is not set"
|
||||||
|
[ -z ${LOG_PATH} ] && die "LOG_PATH is not set"
|
||||||
|
[ -z ${BUILD_PATH} ] && die "BUILD_PATH is not set"
|
||||||
|
[ -z ${IDF_TARGET} ] && die "IDF_TARGET is not set"
|
||||||
|
[ -d ${LOG_PATH} ] || mkdir -p ${LOG_PATH}
|
||||||
|
[ -d ${BUILD_PATH} ] || mkdir -p ${BUILD_PATH}
|
||||||
|
|
||||||
|
if [ -z ${CI_NODE_TOTAL} ]; then
|
||||||
|
CI_NODE_TOTAL=1
|
||||||
|
echo "Assuming CI_NODE_TOTAL=${CI_NODE_TOTAL}"
|
||||||
|
fi
|
||||||
|
if [ -z ${CI_NODE_INDEX} ]; then
|
||||||
|
# Gitlab uses a 1-based index
|
||||||
|
CI_NODE_INDEX=1
|
||||||
|
echo "Assuming CI_NODE_INDEX=${CI_NODE_INDEX}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
set -o nounset # Exit if variable not set.
|
||||||
|
|
||||||
|
# Convert LOG_PATH to relative, to make the json file less verbose.
|
||||||
|
LOG_PATH=$(realpath --relative-to ${IDF_PATH} ${LOG_PATH})
|
||||||
|
BUILD_PATH=$(realpath --relative-to ${IDF_PATH} ${BUILD_PATH})
|
||||||
|
|
||||||
|
ALL_BUILD_LIST_JSON="${BUILD_PATH}/list.json"
|
||||||
|
JOB_BUILD_LIST_JSON="${BUILD_PATH}/list_job_${CI_NODE_INDEX}.json"
|
||||||
|
mkdir -p "${BUILD_PATH}/example_builds"
|
||||||
|
|
||||||
|
echo "build_examples running for target $IDF_TARGET"
|
||||||
|
|
||||||
|
cd ${IDF_PATH}
|
||||||
|
|
||||||
|
# This part of the script produces the same result for all the test app build jobs. It may be moved to a separate stage
|
||||||
|
# (pre-build) later, then the build jobs will receive ${BUILD_LIST_JSON} file as an artifact.
|
||||||
|
|
||||||
|
# If changing the work-dir or build-dir, remember to update the "artifacts" in gitlab-ci configs, and IDFApp.py.
|
||||||
|
|
||||||
|
${IDF_PATH}/tools/find_apps.py tools/test_apps \
|
||||||
|
-vv \
|
||||||
|
--format json \
|
||||||
|
--build-system cmake \
|
||||||
|
--target ${IDF_TARGET} \
|
||||||
|
--recursive \
|
||||||
|
--work-dir "${BUILD_PATH}/@f/@w/@t" \
|
||||||
|
--build-dir build \
|
||||||
|
--build-log "${LOG_PATH}/@f.txt" \
|
||||||
|
--output ${ALL_BUILD_LIST_JSON} \
|
||||||
|
--config 'sdkconfig.ci=default' \
|
||||||
|
--config 'sdkconfig.ci.*=' \
|
||||||
|
--config '=default' \
|
||||||
|
|
||||||
|
# --config rules above explained:
|
||||||
|
# 1. If sdkconfig.ci exists, use it build the example with configuration name "default"
|
||||||
|
# 2. If sdkconfig.ci.* exists, use it to build the "*" configuration
|
||||||
|
# 3. If none of the above exist, build the default configuration under the name "default"
|
||||||
|
# --work-dir and --build-log above uses "placeholders" @x:
|
||||||
|
# - @f: full path to the test with slashes replaced with underscores
|
||||||
|
# - @w: wildcard used as config name
|
||||||
|
# - @t: target name
|
||||||
|
# so the workdir .../@f/@w/@t would expand to e.g. tools_test_apps_system_startup/default/esp32
|
||||||
|
|
||||||
|
# The part below is where the actual builds happen
|
||||||
|
|
||||||
|
${IDF_PATH}/tools/build_apps.py \
|
||||||
|
-vv \
|
||||||
|
--format json \
|
||||||
|
--keep-going \
|
||||||
|
--parallel-count ${CI_NODE_TOTAL} \
|
||||||
|
--parallel-index ${CI_NODE_INDEX} \
|
||||||
|
--output-build-list ${JOB_BUILD_LIST_JSON} \
|
||||||
|
${ALL_BUILD_LIST_JSON}\
|
||||||
|
|
||||||
|
|
||||||
|
# Check for build warnings
|
||||||
|
${IDF_PATH}/tools/ci/check_build_warnings.py -vv ${JOB_BUILD_LIST_JSON}
|
|
@ -5,20 +5,23 @@ assign_test:
|
||||||
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
|
||||||
stage: assign_test
|
stage: assign_test
|
||||||
# gitlab ci do not support match job with RegEx or wildcard now in dependencies.
|
# gitlab ci do not support match job with RegEx or wildcard now in dependencies.
|
||||||
# we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
|
# we have a lot build example jobs. now we don't use dependencies, just download all artifacts of build stage.
|
||||||
dependencies:
|
dependencies:
|
||||||
- build_ssc_esp32
|
- build_ssc_esp32
|
||||||
- build_esp_idf_tests_cmake
|
- build_esp_idf_tests_cmake
|
||||||
variables:
|
variables:
|
||||||
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
|
||||||
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
|
||||||
|
TEST_APP_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/tools/test_apps/test_configs"
|
||||||
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- components/idf_test/*/CIConfigs
|
- components/idf_test/*/CIConfigs
|
||||||
- components/idf_test/*/TC.sqlite
|
- components/idf_test/*/TC.sqlite
|
||||||
- $EXAMPLE_CONFIG_OUTPUT_PATH
|
- $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||||
|
- $TEST_APP_CONFIG_OUTPUT_PATH
|
||||||
- build_examples/artifact_index.json
|
- build_examples/artifact_index.json
|
||||||
|
- build_test_apps/artifact_index.json
|
||||||
expire_in: 1 week
|
expire_in: 1 week
|
||||||
only:
|
only:
|
||||||
variables:
|
variables:
|
||||||
|
@ -26,9 +29,12 @@ assign_test:
|
||||||
- $BOT_LABEL_UNIT_TEST
|
- $BOT_LABEL_UNIT_TEST
|
||||||
- $BOT_LABEL_INTEGRATION_TEST
|
- $BOT_LABEL_INTEGRATION_TEST
|
||||||
- $BOT_LABEL_EXAMPLE_TEST
|
- $BOT_LABEL_EXAMPLE_TEST
|
||||||
|
- $BOT_LABEL_CUSTOM_TEST
|
||||||
script:
|
script:
|
||||||
# assign example tests
|
# assign example tests
|
||||||
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
|
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
|
||||||
|
# assign test apps
|
||||||
|
- python tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py --custom-group test-apps --job-prefix test_app_test_ $IDF_PATH/tools/test_apps $CI_TARGET_TEST_CONFIG_FILE $TEST_APP_CONFIG_OUTPUT_PATH
|
||||||
# assign unit test cases
|
# assign unit test cases
|
||||||
- python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
|
- python tools/ci/python_packages/ttfw_idf/CIAssignUnitTest.py $UNIT_TEST_CASE_FILE $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
|
||||||
# clone test script to assign tests
|
# clone test script to assign tests
|
||||||
|
|
|
@ -184,6 +184,49 @@ build_examples_cmake_esp32s2:
|
||||||
variables:
|
variables:
|
||||||
IDF_TARGET: esp32s2
|
IDF_TARGET: esp32s2
|
||||||
|
|
||||||
|
.build_test_apps: &build_test_apps
|
||||||
|
extends: .build_template
|
||||||
|
stage: build
|
||||||
|
artifacts:
|
||||||
|
when: always
|
||||||
|
paths:
|
||||||
|
- build_test_apps/list.json
|
||||||
|
- build_test_apps/list_job_*.json
|
||||||
|
- build_test_apps/*/*/*/build/*.bin
|
||||||
|
- build_test_apps/*/*/*/sdkconfig
|
||||||
|
- build_test_apps/*/*/*/build/*.elf
|
||||||
|
- build_test_apps/*/*/*/build/*.map
|
||||||
|
- build_test_apps/*/*/*/build/flasher_args.json
|
||||||
|
- build_test_apps/*/*/*/build/bootloader/*.bin
|
||||||
|
- build_test_apps/*/*/*/build/partition_table/*.bin
|
||||||
|
- $LOG_PATH
|
||||||
|
expire_in: 3 days
|
||||||
|
variables:
|
||||||
|
LOG_PATH: "$CI_PROJECT_DIR/log_test_apps"
|
||||||
|
BUILD_PATH: "$CI_PROJECT_DIR/build_test_apps"
|
||||||
|
only:
|
||||||
|
variables:
|
||||||
|
- $BOT_TRIGGER_WITH_LABEL == null
|
||||||
|
- $BOT_LABEL_BUILD
|
||||||
|
- $BOT_LABEL_INTEGRATION_TEST
|
||||||
|
- $BOT_LABEL_REGULAR_TEST
|
||||||
|
- $BOT_LABEL_WEEKEND_TEST
|
||||||
|
script:
|
||||||
|
- mkdir -p ${BUILD_PATH}
|
||||||
|
- mkdir -p ${LOG_PATH}
|
||||||
|
- ${IDF_PATH}/tools/ci/build_test_apps.sh
|
||||||
|
|
||||||
|
build_test_apps_esp32:
|
||||||
|
extends: .build_test_apps
|
||||||
|
variables:
|
||||||
|
IDF_TARGET: esp32
|
||||||
|
|
||||||
|
build_test_apps_esp32s2:
|
||||||
|
extends: .build_test_apps
|
||||||
|
variables:
|
||||||
|
IDF_TARGET: esp32s2
|
||||||
|
|
||||||
|
|
||||||
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
|
# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
|
||||||
|
|
||||||
.build_docs_template: &build_docs_template
|
.build_docs_template: &build_docs_template
|
||||||
|
|
|
@ -83,6 +83,29 @@
|
||||||
# run test
|
# run test
|
||||||
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
- python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
|
||||||
|
|
||||||
|
.test_app_template:
|
||||||
|
extends: .example_test_template
|
||||||
|
stage: target_test
|
||||||
|
dependencies:
|
||||||
|
- assign_test
|
||||||
|
only:
|
||||||
|
refs:
|
||||||
|
- master
|
||||||
|
- /^release\/v/
|
||||||
|
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||||
|
- triggers
|
||||||
|
- schedules
|
||||||
|
variables:
|
||||||
|
- $BOT_TRIGGER_WITH_LABEL == null
|
||||||
|
- $BOT_LABEL_CUSTOM_TEST
|
||||||
|
- $BOT_LABEL_EXAMPLE_TEST
|
||||||
|
variables:
|
||||||
|
TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
|
||||||
|
TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/test_apps"
|
||||||
|
CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/tools/test_apps/test_configs"
|
||||||
|
LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
|
||||||
|
ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
|
||||||
|
|
||||||
.unit_test_template:
|
.unit_test_template:
|
||||||
extends: .example_test_template
|
extends: .example_test_template
|
||||||
stage: target_test
|
stage: target_test
|
||||||
|
@ -279,6 +302,12 @@ example_test_010:
|
||||||
- ESP32
|
- ESP32
|
||||||
- Example_ExtFlash
|
- Example_ExtFlash
|
||||||
|
|
||||||
|
test_app_test_001:
|
||||||
|
extends: .test_app_template
|
||||||
|
tags:
|
||||||
|
- ESP32
|
||||||
|
- test_jtag_arm
|
||||||
|
|
||||||
example_test_011:
|
example_test_011:
|
||||||
extends: .example_debug_template
|
extends: .example_debug_template
|
||||||
tags:
|
tags:
|
||||||
|
|
|
@ -32,6 +32,7 @@ tools/check_python_dependencies.py
|
||||||
tools/ci/apply_bot_filter.py
|
tools/ci/apply_bot_filter.py
|
||||||
tools/ci/build_examples.sh
|
tools/ci/build_examples.sh
|
||||||
tools/ci/build_examples_cmake.sh
|
tools/ci/build_examples_cmake.sh
|
||||||
|
tools/ci/build_test_apps.sh
|
||||||
tools/ci/check-executable.sh
|
tools/ci/check-executable.sh
|
||||||
tools/ci/check-line-endings.sh
|
tools/ci/check-line-endings.sh
|
||||||
tools/ci/check_build_warnings.py
|
tools/ci/check_build_warnings.py
|
||||||
|
|
|
@ -148,6 +148,7 @@ class AssignTest(object):
|
||||||
|
|
||||||
def __init__(self, test_case_path, ci_config_file, case_group=Group):
|
def __init__(self, test_case_path, ci_config_file, case_group=Group):
|
||||||
self.test_case_path = test_case_path
|
self.test_case_path = test_case_path
|
||||||
|
self.test_case_file_pattern = None
|
||||||
self.test_cases = []
|
self.test_cases = []
|
||||||
self.jobs = self._parse_gitlab_ci_config(ci_config_file)
|
self.jobs = self._parse_gitlab_ci_config(ci_config_file)
|
||||||
self.case_group = case_group
|
self.case_group = case_group
|
||||||
|
@ -177,7 +178,7 @@ class AssignTest(object):
|
||||||
job_list.sort(key=lambda x: x["name"])
|
job_list.sort(key=lambda x: x["name"])
|
||||||
return job_list
|
return job_list
|
||||||
|
|
||||||
def _search_cases(self, test_case_path, case_filter=None):
|
def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
|
||||||
"""
|
"""
|
||||||
:param test_case_path: path contains test case folder
|
:param test_case_path: path contains test case folder
|
||||||
:param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
|
:param case_filter: filter for test cases. the filter to use is default filter updated with case_filter param.
|
||||||
|
@ -186,7 +187,7 @@ class AssignTest(object):
|
||||||
_case_filter = self.DEFAULT_FILTER.copy()
|
_case_filter = self.DEFAULT_FILTER.copy()
|
||||||
if case_filter:
|
if case_filter:
|
||||||
_case_filter.update(case_filter)
|
_case_filter.update(case_filter)
|
||||||
test_methods = SearchCases.Search.search_test_cases(test_case_path)
|
test_methods = SearchCases.Search.search_test_cases(test_case_path, test_case_file_pattern)
|
||||||
return CaseConfig.filter_test_cases(test_methods, _case_filter)
|
return CaseConfig.filter_test_cases(test_methods, _case_filter)
|
||||||
|
|
||||||
def _group_cases(self):
|
def _group_cases(self):
|
||||||
|
@ -276,7 +277,7 @@ class AssignTest(object):
|
||||||
failed_to_assign = []
|
failed_to_assign = []
|
||||||
assigned_groups = []
|
assigned_groups = []
|
||||||
case_filter = self._apply_bot_filter()
|
case_filter = self._apply_bot_filter()
|
||||||
self.test_cases = self._search_cases(self.test_case_path, case_filter)
|
self.test_cases = self._search_cases(self.test_case_path, case_filter, self.test_case_file_pattern)
|
||||||
self._apply_bot_test_count()
|
self._apply_bot_test_count()
|
||||||
test_groups = self._group_cases()
|
test_groups = self._group_cases()
|
||||||
|
|
||||||
|
|
|
@ -93,14 +93,14 @@ class Search(object):
|
||||||
return replicated_cases
|
return replicated_cases
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def search_test_cases(cls, test_case):
|
def search_test_cases(cls, test_case, test_case_file_pattern=None):
|
||||||
"""
|
"""
|
||||||
search all test cases from a folder or file, and then do case replicate.
|
search all test cases from a folder or file, and then do case replicate.
|
||||||
|
|
||||||
:param test_case: test case file(s) path
|
:param test_case: test case file(s) path
|
||||||
:return: a list of replicated test methods
|
:return: a list of replicated test methods
|
||||||
"""
|
"""
|
||||||
test_case_files = cls._search_test_case_files(test_case, cls.TEST_CASE_FILE_PATTERN)
|
test_case_files = cls._search_test_case_files(test_case, test_case_file_pattern or cls.TEST_CASE_FILE_PATTERN)
|
||||||
test_cases = []
|
test_cases = []
|
||||||
for test_case_file in test_case_files:
|
for test_case_file in test_case_files:
|
||||||
test_cases += cls._search_cases_from_file(test_case_file)
|
test_cases += cls._search_cases_from_file(test_case_file)
|
||||||
|
|
|
@ -25,25 +25,34 @@ import json
|
||||||
import gitlab_api
|
import gitlab_api
|
||||||
from tiny_test_fw.Utility import CIAssignTest
|
from tiny_test_fw.Utility import CIAssignTest
|
||||||
|
|
||||||
|
|
||||||
EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
|
||||||
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
|
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
|
||||||
if IDF_PATH_FROM_ENV:
|
|
||||||
ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
|
|
||||||
"build_examples", "artifact_index.json")
|
|
||||||
else:
|
|
||||||
ARTIFACT_INDEX_FILE = "artifact_index.json"
|
|
||||||
|
|
||||||
|
|
||||||
class ExampleGroup(CIAssignTest.Group):
|
class ExampleGroup(CIAssignTest.Group):
|
||||||
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
|
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
|
||||||
|
BUILD_LOCAL_DIR = "build_examples"
|
||||||
|
BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppsGroup(ExampleGroup):
|
||||||
|
BUILD_LOCAL_DIR = "build_test_apps"
|
||||||
|
BUILD_JOB_NAMES = ["build_test_apps_esp32", "build_test_apps_esp32s2"]
|
||||||
|
|
||||||
|
|
||||||
class CIExampleAssignTest(CIAssignTest.AssignTest):
|
class CIExampleAssignTest(CIAssignTest.AssignTest):
|
||||||
CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
|
CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
|
||||||
|
|
||||||
|
|
||||||
def create_artifact_index_file(project_id=None, pipeline_id=None):
|
def get_artifact_index_file(case_group=ExampleGroup):
|
||||||
|
if IDF_PATH_FROM_ENV:
|
||||||
|
artifact_index_file = os.path.join(IDF_PATH_FROM_ENV,
|
||||||
|
case_group.BUILD_LOCAL_DIR, "artifact_index.json")
|
||||||
|
else:
|
||||||
|
artifact_index_file = "artifact_index.json"
|
||||||
|
return artifact_index_file
|
||||||
|
|
||||||
|
|
||||||
|
def create_artifact_index_file(project_id=None, pipeline_id=None, case_group=ExampleGroup):
|
||||||
if project_id is None:
|
if project_id is None:
|
||||||
project_id = os.getenv("CI_PROJECT_ID")
|
project_id = os.getenv("CI_PROJECT_ID")
|
||||||
if pipeline_id is None:
|
if pipeline_id is None:
|
||||||
|
@ -52,9 +61,10 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
||||||
artifact_index_list = []
|
artifact_index_list = []
|
||||||
|
|
||||||
def format_build_log_path():
|
def format_build_log_path():
|
||||||
return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
|
parallel = job_info["parallel_num"] # Could be None if "parallel_num" not defined for the job
|
||||||
|
return "{}/list_job_{}.json".format(case_group.BUILD_LOCAL_DIR, parallel or 1)
|
||||||
|
|
||||||
for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
|
for build_job_name in case_group.BUILD_JOB_NAMES:
|
||||||
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
||||||
for job_info in job_info_list:
|
for job_info in job_info_list:
|
||||||
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
|
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
|
||||||
|
@ -62,13 +72,14 @@ def create_artifact_index_file(project_id=None, pipeline_id=None):
|
||||||
for build_info in build_info_list:
|
for build_info in build_info_list:
|
||||||
build_info["ci_job_id"] = job_info["id"]
|
build_info["ci_job_id"] = job_info["id"]
|
||||||
artifact_index_list.append(build_info)
|
artifact_index_list.append(build_info)
|
||||||
|
artifact_index_file = get_artifact_index_file(case_group=case_group)
|
||||||
try:
|
try:
|
||||||
os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
|
os.makedirs(os.path.dirname(artifact_index_file))
|
||||||
except OSError:
|
except OSError:
|
||||||
# already created
|
# already created
|
||||||
pass
|
pass
|
||||||
|
|
||||||
with open(ARTIFACT_INDEX_FILE, "w") as f:
|
with open(artifact_index_file, "w") as f:
|
||||||
json.dump(artifact_index_list, f)
|
json.dump(artifact_index_list, f)
|
||||||
|
|
||||||
|
|
||||||
|
@ -82,9 +93,22 @@ if __name__ == '__main__':
|
||||||
help="output path of config files")
|
help="output path of config files")
|
||||||
parser.add_argument("--pipeline_id", "-p", type=int, default=None,
|
parser.add_argument("--pipeline_id", "-p", type=int, default=None,
|
||||||
help="pipeline_id")
|
help="pipeline_id")
|
||||||
|
parser.add_argument("--job-prefix",
|
||||||
|
help="prefix of the test job name in CI yml file")
|
||||||
|
parser.add_argument("--test-case-file-pattern",
|
||||||
|
help="file name pattern used to find Python test case files")
|
||||||
|
parser.add_argument('--custom-group',
|
||||||
|
help='select custom-group for the test cases, if other than ExampleTest',
|
||||||
|
choices=['example','test-apps'], default='example')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
|
if args.job_prefix:
|
||||||
|
CIExampleAssignTest.CI_TEST_JOB_PATTERN = re.compile(r"^{}.+".format(args.job_prefix))
|
||||||
|
|
||||||
|
case_group = ExampleGroup if args.custom_group == 'example' else TestAppsGroup
|
||||||
|
|
||||||
|
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=case_group)
|
||||||
assign_test.assign_cases()
|
assign_test.assign_cases()
|
||||||
assign_test.output_configs(args.output_path)
|
assign_test.output_configs(args.output_path)
|
||||||
create_artifact_index_file()
|
create_artifact_index_file(case_group=case_group)
|
||||||
|
|
|
@ -139,7 +139,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
|
||||||
def __init__(self, test_case_path, ci_config_file):
|
def __init__(self, test_case_path, ci_config_file):
|
||||||
CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
|
CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group)
|
||||||
|
|
||||||
def _search_cases(self, test_case_path, case_filter=None):
|
def _search_cases(self, test_case_path, case_filter=None, test_case_file_pattern=None):
|
||||||
"""
|
"""
|
||||||
For unit test case, we don't search for test functions.
|
For unit test case, we don't search for test functions.
|
||||||
The unit test cases is stored in a yaml file which is created in job build-idf-test.
|
The unit test cases is stored in a yaml file which is created in job build-idf-test.
|
||||||
|
|
|
@ -310,7 +310,7 @@ class Example(IDFApp):
|
||||||
"""
|
"""
|
||||||
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
||||||
|
|
||||||
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None):
|
def _try_get_binary_from_local_fs(self, app_path, config_name=None, target=None, local_build_dir="build_examples"):
|
||||||
# build folder of example path
|
# build folder of example path
|
||||||
path = os.path.join(self.idf_path, app_path, "build")
|
path = os.path.join(self.idf_path, app_path, "build")
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
|
@ -327,7 +327,7 @@ class Example(IDFApp):
|
||||||
# (see tools/ci/build_examples_cmake.sh)
|
# (see tools/ci/build_examples_cmake.sh)
|
||||||
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
|
# For example: $IDF_PATH/build_examples/examples_get-started_blink/default/esp32
|
||||||
app_path_underscored = app_path.replace(os.path.sep, "_")
|
app_path_underscored = app_path.replace(os.path.sep, "_")
|
||||||
example_path = os.path.join(self.idf_path, "build_examples")
|
example_path = os.path.join(self.idf_path, local_build_dir)
|
||||||
for dirpath in os.listdir(example_path):
|
for dirpath in os.listdir(example_path):
|
||||||
if os.path.basename(dirpath) == app_path_underscored:
|
if os.path.basename(dirpath) == app_path_underscored:
|
||||||
path = os.path.join(example_path, dirpath, config_name, target, "build")
|
path = os.path.join(example_path, dirpath, config_name, target, "build")
|
||||||
|
@ -341,7 +341,8 @@ class Example(IDFApp):
|
||||||
if path:
|
if path:
|
||||||
return path
|
return path
|
||||||
else:
|
else:
|
||||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
artifacts = Artifacts(self.idf_path,
|
||||||
|
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||||
app_path, config_name, target)
|
app_path, config_name, target)
|
||||||
path = artifacts.download_artifacts()
|
path = artifacts.download_artifacts()
|
||||||
if path:
|
if path:
|
||||||
|
@ -369,7 +370,8 @@ class LoadableElfExample(Example):
|
||||||
if path:
|
if path:
|
||||||
return path
|
return path
|
||||||
else:
|
else:
|
||||||
artifacts = Artifacts(self.idf_path, CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
artifacts = Artifacts(self.idf_path,
|
||||||
|
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.ExampleGroup),
|
||||||
app_path, config_name, target)
|
app_path, config_name, target)
|
||||||
path = artifacts.download_artifact_files(self.app_files)
|
path = artifacts.download_artifact_files(self.app_files)
|
||||||
if path:
|
if path:
|
||||||
|
@ -402,6 +404,22 @@ class UT(IDFApp):
|
||||||
raise OSError("Failed to get unit-test-app binary path")
|
raise OSError("Failed to get unit-test-app binary path")
|
||||||
|
|
||||||
|
|
||||||
|
class TestApp(Example):
|
||||||
|
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||||
|
path = self._try_get_binary_from_local_fs(app_path, config_name, target, local_build_dir="build_test_apps")
|
||||||
|
if path:
|
||||||
|
return path
|
||||||
|
else:
|
||||||
|
artifacts = Artifacts(self.idf_path,
|
||||||
|
CIAssignExampleTest.get_artifact_index_file(case_group=CIAssignExampleTest.TestAppsGroup),
|
||||||
|
app_path, config_name, target)
|
||||||
|
path = artifacts.download_artifacts()
|
||||||
|
if path:
|
||||||
|
return os.path.join(self.idf_path, path)
|
||||||
|
else:
|
||||||
|
raise OSError("Failed to find example binary")
|
||||||
|
|
||||||
|
|
||||||
class SSC(IDFApp):
|
class SSC(IDFApp):
|
||||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||||
# TODO: to implement SSC get binary path
|
# TODO: to implement SSC get binary path
|
||||||
|
|
|
@ -15,7 +15,7 @@ import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from tiny_test_fw import TinyFW, Utility
|
from tiny_test_fw import TinyFW, Utility
|
||||||
from .IDFApp import IDFApp, Example, LoadableElfExample, UT # noqa: export all Apps for users
|
from .IDFApp import IDFApp, Example, LoadableElfExample, UT, TestApp # noqa: export all Apps for users
|
||||||
from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users
|
from .IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT, ESP32QEMUDUT # noqa: export DUTs for users
|
||||||
|
|
||||||
|
|
||||||
|
@ -88,6 +88,41 @@ def idf_unit_test(app=UT, dut=IDFDUT, chip="ESP32", module="unit-test", executio
|
||||||
return test
|
return test
|
||||||
|
|
||||||
|
|
||||||
|
def idf_custom_test(app=TestApp, dut=IDFDUT, chip="ESP32", module="misc", execution_time=1,
|
||||||
|
level="integration", erase_nvs=True, config_name=None, group="test-apps", **kwargs):
|
||||||
|
|
||||||
|
"""
|
||||||
|
decorator for idf custom tests (with default values for some keyword args).
|
||||||
|
|
||||||
|
:param app: test application class
|
||||||
|
:param dut: dut class
|
||||||
|
:param chip: chip supported, string or tuple
|
||||||
|
:param module: module, string
|
||||||
|
:param execution_time: execution time in minutes, int
|
||||||
|
:param level: test level, could be used to filter test cases, string
|
||||||
|
:param erase_nvs: if need to erase_nvs in DUT.start_app()
|
||||||
|
:param config_name: if specified, name of the app configuration
|
||||||
|
:param group: identifier to group custom tests (unused for now, defaults to "test-apps")
|
||||||
|
:param kwargs: other keyword args
|
||||||
|
:return: test method
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# try to config the default behavior of erase nvs
|
||||||
|
dut.ERASE_NVS = erase_nvs
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
original_method = TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
|
||||||
|
execution_time=execution_time, level=level, **kwargs)
|
||||||
|
|
||||||
|
def test(func):
|
||||||
|
test_func = original_method(func)
|
||||||
|
test_func.case_info["ID"] = format_case_id(chip, test_func.case_info["name"])
|
||||||
|
return test_func
|
||||||
|
|
||||||
|
return test
|
||||||
|
|
||||||
|
|
||||||
def log_performance(item, value):
|
def log_performance(item, value):
|
||||||
"""
|
"""
|
||||||
do print performance with pre-defined format to console
|
do print performance with pre-defined format to console
|
||||||
|
|
32
tools/test_apps/README.md
Normal file
32
tools/test_apps/README.md
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
# Test Apps
|
||||||
|
|
||||||
|
This directory contains a set of ESP-IDF projects to be used as tests only, which aim to exercise various
|
||||||
|
configuration of components to check completely arbitrary functionality should it be building only, executing under
|
||||||
|
various conditions or combination with other components, including custom test frameworks.
|
||||||
|
|
||||||
|
The test apps are not intended to demonstrate the ESP-IDF functionality in any way.
|
||||||
|
|
||||||
|
# Test Apps projects
|
||||||
|
|
||||||
|
Test applications are treated the same way as ESP-IDF examples, so each project directory shall contain
|
||||||
|
* Build recipe in cmake and the main component with app sources
|
||||||
|
* Configuration files
|
||||||
|
- `sdkconfig.ci` - Default configuration for the project
|
||||||
|
- `sdkconfig.ci.<CONFIG>` - Other configurations, where `<CONFIG>` indicates name of the configuration
|
||||||
|
* Test executor in `ttfw_idf` format if the project is intended to also run tests (otherwise the example is build only)
|
||||||
|
- test file in the project dir must end with `_test.py`, by should be named `app_test.py`
|
||||||
|
- test cases shall be decorated with `@ttfw_idf.idf_custom_test(env_tag="...")`
|
||||||
|
|
||||||
|
# Test Apps layout
|
||||||
|
|
||||||
|
The test apps should be grouped into subdirectories by category. Categories are:
|
||||||
|
* `protocols` contains test of protocol interactions.
|
||||||
|
* `network` contains system network tests
|
||||||
|
* `system` contains tests on the internal chip features, debugging and development tools.
|
||||||
|
|
||||||
|
# Test Apps local execution
|
||||||
|
|
||||||
|
* Append relevant `sdkconfig.ci.<CONFIG>` to the sdkconfig for the configuration under test
|
||||||
|
* Run `idf.py menuconfig` to configure local project attributes
|
||||||
|
* Run `idf.py build` to build the test app
|
||||||
|
* Run `python app_test.py` to run the test locally
|
6
tools/test_apps/system/startup/CMakeLists.txt
Normal file
6
tools/test_apps/system/startup/CMakeLists.txt
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
# The following lines of boilerplate have to be in your project's
|
||||||
|
# CMakeLists in this exact order for cmake to work correctly
|
||||||
|
cmake_minimum_required(VERSION 3.5)
|
||||||
|
|
||||||
|
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||||
|
project(test_startup)
|
4
tools/test_apps/system/startup/README.txt
Normal file
4
tools/test_apps/system/startup/README.txt
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
This project tests if the app can start up in a certain configuration.
|
||||||
|
To add new configuration, create one more sdkconfig.ci.NAME file in this directory.
|
||||||
|
|
||||||
|
If you need to test for anything other than app starting up, create another test project.
|
23
tools/test_apps/system/startup/app_test.py
Normal file
23
tools/test_apps/system/startup/app_test.py
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import os
|
||||||
|
import glob
|
||||||
|
import ttfw_idf
|
||||||
|
from tiny_test_fw import Utility
|
||||||
|
|
||||||
|
|
||||||
|
@ttfw_idf.idf_custom_test(env_tag="test_jtag_arm", group="test-apps")
|
||||||
|
def test_startup(env, extra_data):
|
||||||
|
config_files = glob.glob(os.path.join(os.path.dirname(__file__), "sdkconfig.ci.*"))
|
||||||
|
config_names = [os.path.basename(s).replace("sdkconfig.ci.", "") for s in config_files]
|
||||||
|
for name in config_names:
|
||||||
|
Utility.console_log("Checking config \"{}\"... ".format(name), end="")
|
||||||
|
dut = env.get_dut("startup", "tools/test_apps/system/startup", app_config_name=name)
|
||||||
|
dut.start_app()
|
||||||
|
dut.expect("app_main running")
|
||||||
|
env.close_dut(dut.name)
|
||||||
|
Utility.console_log("done")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
test_startup()
|
2
tools/test_apps/system/startup/main/CMakeLists.txt
Normal file
2
tools/test_apps/system/startup/main/CMakeLists.txt
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
idf_component_register(SRCS "test_startup_main.c"
|
||||||
|
INCLUDE_DIRS ".")
|
6
tools/test_apps/system/startup/main/test_startup_main.c
Normal file
6
tools/test_apps/system/startup/main/test_startup_main.c
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
void app_main(void)
|
||||||
|
{
|
||||||
|
printf("app_main running\n");
|
||||||
|
}
|
0
tools/test_apps/system/startup/sdkconfig.ci.default
Normal file
0
tools/test_apps/system/startup/sdkconfig.ci.default
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
CONFIG_ESPTOOLPY_FLASHFREQ_80M=y
|
||||||
|
CONFIG_ESPTOOLPY_FLASHMODE_QIO=y
|
Loading…
Reference in a new issue