CI: use include to split large CI config file

tools/ci/config/assign-test.yml

@@ -0,0 +1,74 @@
+
+assign_test:
+  tags:
+    - assign_test
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  stage: assign_test
+  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
+  # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
+  dependencies:
+    - build_ssc
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
+  artifacts:
+    paths:
+      - components/idf_test/*/CIConfigs
+      - components/idf_test/*/TC.sqlite
+      - $EXAMPLE_CONFIG_OUTPUT_PATH
+    expire_in: 1 week
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_UNIT_TEST
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_EXAMPLE_TEST
+  script:
+    # assign example tests
+    - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $CI_TARGET_TEST_CONFIG_FILE $EXAMPLE_CONFIG_OUTPUT_PATH
+    # assign unit test cases
+    - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $CI_TARGET_TEST_CONFIG_FILE $IDF_PATH/components/idf_test/unit_test/CIConfigs
+    # clone test script to assign tests
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # assgin integration test cases
+    - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $CI_TARGET_TEST_CONFIG_FILE -b $IDF_PATH/SSC/ssc_bin
+
+update_test_cases:
+  stage: assign_test
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env
+  tags:
+    - deploy_test
+  only:
+    refs:
+      - master
+      - schedules
+  dependencies:
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  artifacts:
+    when: always
+    paths:
+      - ${CI_PROJECT_DIR}/test-management/*.log
+    expire_in: 1 week
+  variables:
+    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+    PYTHON_VER: 3
+  script:
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - git clone $TEST_MANAGEMENT_REPO
+    - python $CHECKOUT_REF_SCRIPT test-management test-management
+    - cd test-management
+    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # update unit test cases
+    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILE -r $GIT_SHA
+    # update example test cases
+    - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT tiny_test_fw -d ${CI_PROJECT_DIR}/examples -r $GIT_SHA
+    # organize test cases
+    - python OrganizeTestCases.py $JIRA_TEST_MANAGEMENT_PROJECT

tools/ci/config/build.yml

@@ -0,0 +1,247 @@
+
+.build_template:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+    V: "0"
+
+.build_esp_idf_unit_test_template:
+  extends: .build_template
+  artifacts:
+    paths:
+      - tools/unit-test-app/output
+      - components/idf_test/unit_test/TestCaseAll.yml
+    expire_in: 3 days
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_UNIT_TEST
+      - $BOT_LABEL_REGULAR_TEST
+
+build_template_app:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  variables:
+    BATCH_BUILD: "1"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    # Set the variable for 'esp-idf-template' testing
+    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
+    - git clone ${ESP_IDF_TEMPLATE_GIT}
+    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
+    - cd esp-idf-template
+    # Try to use the same branch name for esp-idf-template that we're
+    # using on esp-idf. If it doesn't exist then just stick to the default
+    # branch
+    - make defconfig
+    # Test debug build (default)
+    - make all V=1
+    # Now test release build
+    - make clean
+    - sed -i.bak -e's/CONFIG_OPTIMIZATION_LEVEL_DEBUG\=y/CONFIG_OPTIMIZATION_LEVEL_RELEASE=y/' sdkconfig
+    - make all V=1
+    # Check if there are any stray printf/ets_printf references in WiFi libs
+    - cd ../components/esp_wifi/lib_esp32
+    - test $(xtensa-esp32-elf-nm *.a | grep -w printf | wc -l) -eq 0
+    - test $(xtensa-esp32-elf-nm *.a | grep -w ets_printf | wc -l) -eq 0
+
+
+build_ssc:
+  extends: .build_template
+  parallel: 3
+  artifacts:
+    paths:
+      - SSC/ssc_bin
+    expire_in: 1 week
+  variables:
+    SSC_CONFIG_FOLDER: "$CI_PROJECT_DIR/SSC/configs/ESP32_IDF"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_INTEGRATION_TEST
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    - git clone $SSC_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT SSC SSC
+    - cd SSC
+    - MAKEFLAGS= ./ci_build_ssc.sh
+
+build_esp_idf_tests_make:
+  extends: .build_esp_idf_unit_test_template
+  script:
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+    - cd $CI_PROJECT_DIR/tools/unit-test-app
+    - MAKEFLAGS= make help # make sure kconfig tools are built in single process
+    - make ut-clean-all-configs
+    - make ut-build-all-configs
+    - python tools/UnitTestParser.py
+    # Check if the tests demand Make built binaries. If not, delete them
+    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
+    - rm -rf builds output sdkconfig
+    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
+
+build_esp_idf_tests_cmake:
+  extends: .build_esp_idf_unit_test_template
+  script:
+    - export PATH="$IDF_PATH/tools:$PATH"
+    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
+    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
+    - cd $CI_PROJECT_DIR/tools/unit-test-app
+    - idf.py ut-clean-all-configs
+    - idf.py ut-build-all-configs
+    - python tools/UnitTestParser.py
+    # Check if the tests demand CMake built binaries. If not, delete them
+    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi
+    - rm -rf builds output sdkconfig
+    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml
+
+build_examples_make:
+  extends: .build_template
+  parallel: 8
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
+  retry: 1
+  artifacts:
+    when: always
+    paths:
+      - build_examples/*/*/*/build/*.bin
+      - build_examples/*/*/*/sdkconfig
+      - build_examples/*/*/*/build/*.elf
+      - build_examples/*/*/*/build/*.map
+      - build_examples/*/*/*/build/download.config
+      - build_examples/*/*/*/build/bootloader/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples
+    - mkdir build_examples
+    - cd build_examples
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples.sh
+
+# same as above, but for CMake
+build_examples_cmake:
+  extends: .build_template
+  parallel: 5
+  artifacts:
+    when: always
+    paths:
+      - build_examples_cmake/*/*/*/build/*.bin
+      - build_examples_cmake/*/*/*/sdkconfig
+      - build_examples_cmake/*/*/*/build/*.elf
+      - build_examples_cmake/*/*/*/build/*.map
+      - build_examples_cmake/*/*/*/build/flasher_args.json
+      - build_examples_cmake/*/*/*/build/bootloader/*.bin
+      - $LOG_PATH
+    expire_in: 3 days
+  variables:
+    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_EXAMPLE_TEST
+      - $BOT_LABEL_REGULAR_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    # it's not possible to build 100% out-of-tree and have the "artifacts"
+    # mechanism work, but this is the next best thing
+    - rm -rf build_examples_cmake
+    - mkdir build_examples_cmake
+    - cd build_examples_cmake
+    # build some of examples
+    - mkdir -p ${LOG_PATH}
+    - ${IDF_PATH}/tools/ci/build_examples_cmake.sh
+
+# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
+
+build_docs:
+  stage: build
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build_docs
+  artifacts:
+    when: always
+    paths:
+      # English version of documentation
+      - docs/en/doxygen-warning-log.txt
+      - docs/en/sphinx-warning-log.txt
+      - docs/en/sphinx-warning-log-sanitized.txt
+      - docs/en/_build/html
+      - docs/sphinx-err-*
+      # Chinese version of documentation
+      - docs/zh_CN/doxygen-warning-log.txt
+      - docs/zh_CN/sphinx-warning-log.txt
+      - docs/zh_CN/sphinx-warning-log-sanitized.txt
+      - docs/zh_CN/_build/html
+    expire_in: 3 days
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_BUILD_DOCS
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    - cd docs
+    - ./check_lang_folder_sync.sh
+    - cd en
+    - make gh-linkcheck
+    - make html
+    - ../check_doc_warnings.sh
+    - cd ../zh_CN
+    - make gh-linkcheck
+    - make html
+    - ../check_doc_warnings.sh
+
+verify_cmake_style:
+  extends: .check_job_template
+  stage: build
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD
+      - $BOT_LABEL_REGULAR_TEST
+  script:
+    tools/cmake/run_cmake_lint.sh
+
+test_build_system:
+  extends: .build_template
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/test_build_system.sh
+
+test_build_system_cmake:
+  extends: .build_template
+  script:
+    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
+    - rm -rf test_build_system
+    - mkdir test_build_system
+    - cd test_build_system
+    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh

tools/ci/config/check.yml

@@ -0,0 +1,122 @@
+
+# copy from .gitlab-ci.yml as anchor is not global
+.show_submodule_urls: &show_submodule_urls |
+  git config --get-regexp '^submodule\..*\.url$' || true
+
+check_line_endings:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-line-endings.sh ${IDF_PATH}
+
+check_commit_msg:
+  extends: .check_job_template
+  script:
+    - git status
+    - git log -n10 --oneline
+    # commit start with "WIP: " need to be squashed before merge
+    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
+
+check_permissions:
+  extends: .check_job_template
+  script:
+    - tools/ci/check-executable.sh
+
+check_version:
+  extends: .check_job_template
+  # Don't run this for feature/bugfix branches, so that it is possible to modify
+  # esp_idf_version.h in a branch before tagging the next version.
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - export IDF_PATH=$PWD
+    - tools/ci/check_idf_version.sh
+
+check_examples_cmake_make:
+  extends: .check_job_template_with_filter
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  script:
+    - tools/ci/check_examples_cmake_make.sh
+
+check_python_style:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - flake8_output.txt
+    expire_in: 1 week
+  script:
+    # run it only under Python 3 (it is very slow under Python 2)
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+
+check_kconfigs:
+  extends: .check_job_template_with_filter
+  artifacts:
+    when: on_failure
+    paths:
+      - components/*/Kconfig*.new
+      - examples/*/*/*/Kconfig*.new
+      - examples/*/*/*/*/Kconfig*.new
+      - tools/*/Kconfig*.new
+      - tools/*/*/Kconfig*.new
+      - tools/*/*/*/Kconfig*.new
+    expire_in: 1 week
+  script:
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
+    - ${IDF_PATH}/tools/check_kconfigs.py
+
+check_ut_cmake_make:
+  extends: .check_job_template_with_filter
+  stage: check
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - build
+  except:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  dependencies: []
+  script:
+    - tools/ci/check_ut_cmake_make.sh
+
+check_submodule_sync:
+  extends: .check_job_template
+  tags:
+    - github_sync
+  retry: 2
+  variables:
+    GIT_STRATEGY: clone
+    GIT_SUBMODULE_STRATEGY: none
+    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  before_script: []
+  after_script: []
+  script:
+    - git submodule deinit --force .
+    # setting the default remote URL to the public one, to resolve relative location URLs
+    - git config remote.origin.url ${PUBLIC_IDF_URL}
+    # check if all submodules are correctly synced to public repostory
+    - git submodule init
+    - *show_submodule_urls
+    - git submodule update --recursive
+    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
+
+check_artifacts_expire_time:
+  extends: .check_job_template
+  script:
+    # check if we have set expire time for all artifacts
+    - python tools/ci/check_artifacts_expire_time.py
+
+check_pipeline_triggered_by_label:
+  extends: .check_job_template
+  stage: post_check
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL
+  script:
+    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
+    # We want to make sure some jobs are always executed to detect regression.
+    - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; }

tools/ci/config/deploy.yml

@@ -0,0 +1,153 @@
+
+.clang_tidy_deploy_template:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  tags:
+    - deploy
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - export GIT_VER=$(git describe --always)
+    - cd $IDF_PATH/examples/get-started/hello_world/tidybuild
+    - mv report $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - export STATIC_REPORT_PATH="web/static_analysis/esp-idf/"
+    - ssh $DOCS_SERVER -x "mkdir -p $STATIC_REPORT_PATH/clang-tidy"
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$STATIC_REPORT_PATH/clang-tidy
+    - ssh $DOCS_SERVER -x "cd $STATIC_REPORT_PATH/clang-tidy && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    # add link to view the report
+    - echo "[static analysis][clang tidy] $CI_DOCKER_REGISTRY/static_analysis/esp-idf/clang-tidy/${GIT_VER}/index.html"
+    - test ! -e ${GIT_VER}/FAILED_RULES || { echo 'Failed static analysis rules!'; cat ${GIT_VER}/FAILED_RULES; exit 1; }
+
+clang_tidy_deploy:
+  extends: .clang_tidy_deploy_template
+  dependencies:
+    - clang_tidy_check
+    - clang_tidy_check_all
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+
+clang_tidy_deploy_regular:
+  extends: .clang_tidy_deploy_template
+  dependencies:
+    - clang_tidy_check_regular
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_LABEL_STATIC_ANALYSIS
+      - $BOT_LABEL_STATIC_ANALYSIS_ALL
+
+push_to_github:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+  only:
+    - master
+    - /^release\/v/
+    - /^v\d+\.\d+(\.\d+)?($|-)/
+  when: on_success
+  dependencies: []
+  extends: .before_script_lesser
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+    - git remote remove github &>/dev/null || true
+    - git remote add github git@github.com:espressif/esp-idf.git
+    - tools/ci/push_to_github.sh
+
+deploy_docs:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - deploy
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  dependencies:
+    - build_docs
+  extends: .before_script_lesser
+  script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
+    - export GIT_VER=$(git describe --always)
+    - cd docs/en/_build/
+    - mv html $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
+    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    - cd ../../zh_CN/_build/
+    - mv html $GIT_VER
+    - tar czvf $GIT_VER.tar.gz $GIT_VER
+    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
+    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
+    # add link to preview doc
+    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
+    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"
+
+deploy_test_result:
+  stage: deploy
+  image: $CI_DOCKER_REGISTRY/bot-env
+  tags:
+    - deploy_test
+  when: always
+  only:
+    refs:
+      - master
+      - schedules
+  artifacts:
+    when: always
+    paths:
+      - ${CI_PROJECT_DIR}/test-management/*.log
+      # save all test logs as artifacts, make it easier to track errors
+      - ${CI_PROJECT_DIR}/TEST_LOGS
+      - $CI_PROJECT_DIR/$CI_COMMIT_SHA
+    expire_in: 1 mos
+  variables:
+    UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
+    BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"
+  before_script:
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
+    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+  script:
+    - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8)
+    - export REV_COUNT=$(git rev-list --count HEAD)
+    - export SUMMARY="IDF CI test result for $GIT_SHA (r${REV_COUNT})"
+    # artifacts of job update_test_cases creates test-management folder
+    # we need to remove it so we can clone test-management folder again
+    - rm -r test-management
+    - git clone $TEST_MANAGEMENT_REPO
+    - python3 $CHECKOUT_REF_SCRIPT test-management test-management
+    - cd test-management
+    - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE}
+    # update test results
+    - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL}

tools/ci/config/host-test.yml

@@ -0,0 +1,286 @@
+
+.host_test_template:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - host_test
+  dependencies: []
+  only:
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_HOST_TEST
+      - $BOT_LABEL_REGULAR_TEST
+
+.host_fuzzer_test_template:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  tags:
+    - host_test
+  dependencies: []
+  artifacts:
+    when: always
+    paths:
+      - ${FUZZER_TEST_DIR}/out/crashes
+      - ${FUZZER_TEST_DIR}/fuzz_output.log
+    expire_in: 1 week
+  only:
+    variables:
+      - $BOT_LABEL_FUZZER_TEST
+      - $BOT_LABEL_WEEKEND_TEST
+  script:
+    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
+    - cd ${FUZZER_TEST_DIR}
+    # run AFL fuzzer for one hour
+    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
+    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
+    # check no crashes found
+    - test -z "$(ls out/crashes/)" || exit 1
+
+.clang_tidy_check_template:
+  stage: host_test
+  image: ${CI_DOCKER_REGISTRY}/clang-static-analysis
+  tags:
+    - host_test
+  dependencies: []
+  artifacts:
+    reports:
+      junit: $IDF_PATH/output.xml
+    when: always
+    paths:
+      - $IDF_PATH/examples/get-started/hello_world/tidybuild/report/*
+    expire_in: 1 day
+  script:
+    - git clone $IDF_ANALYSIS_UTILS static_analysis_utils && cd static_analysis_utils
+    # Setup parameters of triggered/regular job
+    - export TRIGGERED_RELATIVE=${BOT_LABEL_STATIC_ANALYSIS-} && export TRIGGERED_ABSOLUTE=${BOT_LABEL_STATIC_ANALYSIS_ALL-} && export TARGET_BRANCH=${BOT_CUSTOMIZED_REVISION-}
+    - ./analyze.sh $IDF_PATH/examples/get-started/hello_world/ $IDF_PATH/tools/ci/static-analysis-rules.yml $IDF_PATH/output.xml
+
+test_nvs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make test
+
+test_nvs_coverage:
+  extends: .host_test_template
+  artifacts:
+    paths:
+      - components/nvs_flash/test_nvs_host/coverage_report
+    expire_in: 1 week
+  only:
+    refs:
+      - triggers
+    variables:
+      - $BOT_LABEL_NVS_COVERAGE
+  script:
+    - cd components/nvs_flash/test_nvs_host
+    - make coverage_report
+
+test_partition_table_on_host:
+  extends: .host_test_template
+  tags:
+    - build
+  script:
+    - cd components/partition_table/test_gen_esp32part_host
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py
+
+test_wl_on_host:
+  extends: .host_test_template
+  artifacts:
+    paths:
+      - components/wear_levelling/test_wl_host/coverage_report.zip
+    expire_in: 1 week
+  script:
+    - cd components/wear_levelling/test_wl_host
+    - make test
+
+test_fatfs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/fatfs/test_fatfs_host/
+    - make test
+
+test_ldgen_on_host:
+  extends: .host_test_template
+  script:
+    - cd tools/ldgen/test
+    - ./test_fragments.py
+    - ./test_generation.py
+
+clang_tidy_check:
+  extends: .clang_tidy_check_template
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+    BOT_LABEL_STATIC_ANALYSIS: 1
+
+clang_tidy_check_regular:
+  extends: .clang_tidy_check_template
+
+clang_tidy_check_all:
+  extends: .clang_tidy_check_template
+  variables:
+    BOT_NEEDS_TRIGGER_BY_NAME: 1
+    BOT_LABEL_STATIC_ANALYSIS_ALL: 1
+
+test_mdns_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host
+
+test_lwip_dns_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dns
+
+test_lwip_dhcp_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_client
+
+test_lwip_dhcps_fuzzer_on_host:
+  extends: .host_fuzzer_test_template
+  variables:
+    FUZZER_TEST_DIR: components/lwip/test_afl_host
+    FUZZER_PARAMS: MODE=dhcp_server
+
+test_spiffs_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/spiffs/test_spiffs_host/
+    - make test
+
+test_multi_heap_on_host:
+  extends: .host_test_template
+  script:
+    - cd components/heap/test_multi_heap_host
+    - ./test_all_configs.sh
+
+test_confserver:
+  extends: .host_test_template
+  script:
+    - cd tools/kconfig_new/test
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py
+
+test_idf_monitor:
+  extends: .host_test_template
+  artifacts:
+    # save artifacts always in order to access results which were retried without consequent failure
+    when: always
+    paths:
+      - tools/test_idf_monitor/outputs/*
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_monitor
+    - ./run_test_idf_monitor.py
+
+test_idf_size:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/test_idf_size/output
+      - tools/test_idf_size/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/test_idf_size
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+test_idf_tools:
+  extends: .host_test_template
+  script:
+    # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
+    - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif${CUSTOM_TOOLCHAIN_PATH:+\|${CUSTOM_TOOLCHAIN_PATH}}" | tr "\n" ":"); echo ${p%:})
+    - cd ${IDF_PATH}/tools/test_idf_tools
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+
+test_esp_err_to_name_on_host:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - components/esp32/esp_err_to_name.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
+    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
+
+test_esp_efuse_table_on_host:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - components/efuse/esp32/esp_efuse_table.c
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/components/efuse/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - git diff --exit-code -- ../components/esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
+    - cd ${IDF_PATH}/components/efuse/test_efuse_host
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+
+test_espcoredump:
+  extends: .host_test_template
+  artifacts:
+    when: always
+    paths:
+      - components/espcoredump/test/.coverage
+      - components/espcoredump/test/output
+    expire_in: 1 week
+  script:
+    - cd components/espcoredump/test/
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh
+
+test_logtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/logtrace/output
+      - tools/esp_app_trace/test/logtrace/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+test_sysviewtrace_proc:
+  extends: .host_test_template
+  artifacts:
+    when: on_failure
+    paths:
+      - tools/esp_app_trace/test/sysview/output
+      - tools/esp_app_trace/test/sysview/.coverage
+    expire_in: 1 week
+  script:
+    - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
+    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+
+check_doc_links:
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - check_doc_links
+  only:
+    refs:
+      # can only be triggered
+      - triggers
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_BUILD_DOCS
+  artifacts:
+    paths:
+      - docs/_build/linkcheck
+    expire_in: 1 week
+  script:
+    # must be triggered with CHECK_LINKS=Yes, otherwise exit without test
+    - test "$CHECK_LINKS" = "Yes" || exit 0
+    # can only run on master branch (otherwise the commit is not on Github yet)
+    - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0
+    - cd docs
+    - make linkcheck

tools/ci/config/target-test.yml

@@ -0,0 +1,572 @@
+
+# for parallel jobs, CI_JOB_NAME will be "job_name index/total" (for example, "IT_001 1/2")
+# we need to convert to pattern "job_name_index.yml"
+.define_config_file_name: &define_config_file_name |
+    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
+    JOG_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
+    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOG_FULL_NAME}.yml"
+
+.example_test_template:
+  stage: target_test
+  when: on_success
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_EXAMPLE_TEST
+  dependencies:
+    - assign_test
+    - build_examples_make
+    - build_examples_cmake
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+    expire_in: 1 week
+    reports:
+        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    - cd $TEST_FW_PATH
+    # run test
+    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
+
+.unit_test_template:
+  extends: .example_test_template
+  stage: target_test
+  dependencies:
+    - assign_test
+    - build_esp_idf_tests_make
+    - build_esp_idf_tests_cmake
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_UNIT_TEST
+  variables:
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
+
+.test_template:
+  stage: target_test
+  when: on_success
+  only:
+    refs:
+      - master
+      - /^release\/v/
+      - /^v\d+\.\d+(\.\d+)?($|-)/
+      - triggers
+      - schedules
+    variables:
+      - $BOT_TRIGGER_WITH_LABEL == null
+      - $BOT_LABEL_INTEGRATION_TEST
+  dependencies:
+    - assign_test
+    - build_ssc
+  artifacts:
+    when: always
+    reports:
+        junit: $LOG_PATH/*/XUNIT_RESULT.xml
+    paths:
+      - $LOG_PATH
+    expire_in: 1 week
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
+    LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
+    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
+    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
+    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone local test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    # clone test bench
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # run test
+    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
+
+test_weekend_mqtt:
+  extends: .example_test_template
+  stage: target_test
+  tags:
+    - ESP32
+    - Example_WIFI
+  only:
+    variables:
+      - $BOT_LABEL_WEEKEND_TEST
+  variables:
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/mqtt/weekend_test"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/env.yml"
+    CONFIG_FILE: "$CI_PROJECT_DIR/components/mqtt/weekend_test/config.yml"
+
+test_weekend_network:
+  extends: .example_test_template
+  stage: target_test
+  image: $CI_DOCKER_REGISTRY/rpi-net-suite$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - ESP32
+    - Example_WIFI
+  only:
+    variables:
+      - $BOT_LABEL_WEEKEND_TEST
+  variables:
+    TEST_CASE_PATH: "$CI_PROJECT_DIR/components/lwip/weekend_test"
+    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
+    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
+    ENV_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/env.yml"
+    CONFIG_FILE: "$CI_PROJECT_DIR/components/lwip/weekend_test/config.yml"
+
+example_test_001:
+  extends: .example_test_template
+  parallel: 2
+  tags:
+    - ESP32
+    - Example_WIFI
+
+example_test_002:
+  extends: .example_test_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  tags:
+    - ESP32
+    - Example_ShieldBox_Basic
+
+.example_test_003:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_SDIO
+
+example_test_004:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_CAN
+
+example_test_005:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_WIFI_BT
+
+example_test_006:
+  extends: .example_test_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  only:
+    variables:
+      - $BOT_LABEL_IPERF_STRESS_TEST
+  tags:
+    - ESP32
+    - Example_ShieldBox
+
+example_test_007:
+  extends: .example_test_template
+  tags:
+    - ESP32
+    - Example_I2C_CCS811_SENSOR
+
+UT_001:
+  extends: .unit_test_template
+  parallel: 50
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+UT_002:
+  extends: .unit_test_template
+  parallel: 18
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - psram
+
+UT_003:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+
+UT_004:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+
+UT_005:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SDMODE
+    - psram
+
+UT_006:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_SPIMODE
+    - psram
+
+UT_007:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_GPIO
+
+UT_008:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_GPIO
+    - psram
+
+UT_009:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_PCNT
+
+UT_010:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_PCNT
+    - psram
+
+UT_011:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_LEDC
+
+UT_012:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_LEDC
+    - psram
+
+UT_013:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T2_RS485
+
+UT_014:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_RS485
+    - psram
+
+UT_015:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_RMT
+
+UT_016:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_RMT
+    - psram
+
+UT_017:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - EMMC
+
+UT_018:
+  extends: .unit_test_template
+  parallel: 5
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+    - 8Mpsram
+
+UT_019:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - Example_SPI_Multi_device
+
+UT_020:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - Example_SPI_Multi_device
+    - psram
+
+UT_021:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T2_I2C
+
+UT_022:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_I2C
+    - psram
+
+UT_023:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_MCPWM
+
+UT_024:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_MCPWM
+    - psram
+
+UT_025:
+  extends: .unit_test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - UT_T1_I2S
+
+UT_026:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_I2S
+    - psram
+
+UT_027:
+  extends: .unit_test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+
+UT_028:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - psram
+
+UT_029:
+  extends: .unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T2_1
+    - 8Mpsram
+
+# Gitlab parallel max value is 50. We need to create another UT job if parallel is larger than 50.
+UT_030:
+  extends: .unit_test_template
+  parallel: 5
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+nvs_compatible_test:
+  extends: .test_template
+  artifacts:
+    when: always
+    paths:
+      - $LOG_PATH
+      - nvs_wifi.bin
+    expire_in: 1 mos
+  tags:
+    - ESP32_IDF
+    - NVS_Compatible
+  script:
+    - *define_config_file_name
+    # first test if config file exists, if not exist, exit 0
+    - test -e $CONFIG_FILE || exit 0
+    # clone local test env configs
+    - git clone $TEST_ENV_CONFIG_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
+    # clone test bench
+    - git clone $TEST_SCRIPT_REPOSITORY
+    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
+    - cd auto_test_script
+    # prepare nvs bins
+    - ./Tools/prepare_nvs_bin.sh
+    # run test
+    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH
+
+IT_001:
+  extends: .test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - SSC_T1_4
+
+IT_002:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_2
+
+IT_003:
+  extends: .test_template
+  parallel: 13
+  tags:
+    - ESP32_IDF
+    - SSC_T2_5
+
+IT_004:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_APC
+
+IT_005:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T1_5
+
+IT_006:
+  extends: .test_template
+  parallel: 8
+  tags:
+    - ESP32_IDF
+    - SSC_T1_6
+
+IT_007:
+  extends: .test_template
+  parallel: 3
+  tags:
+    - ESP32_IDF
+    - SSC_T1_7
+
+IT_008:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_8
+
+IT_009:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_3
+
+IT_010:
+  extends: .test_template
+  parallel: 4
+  tags:
+    - ESP32_IDF
+    - SSC_T5_1
+
+IT_011:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_MESH1
+
+IT_012:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T2_MESH1
+
+IT_013:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T3_MESH1
+
+IT_014:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T6_MESH1
+
+IT_015:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T12_MESH1
+
+IT_016:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T50_MESH1
+
+IT_017:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T1_MESH2
+
+IT_018:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T1_9
+
+IT_019:
+  extends: .test_template
+  parallel: 2
+  tags:
+    - ESP32_IDF
+    - SSC_T2_2
+
+IT_020:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T2_3
+
+IT_021:
+  extends: .test_template
+  tags:
+    - ESP32_IDF
+    - SSC_T2_4