OVMS3-idf/.gitlab-ci.yml

stages:
  - build
  - assign_test
  - host_test
  - target_test
  - check
  - deploy
  - post_check

variables:
# System environment

  # Common parameters for the 'make' during CI tests
  MAKEFLAGS: "-j5 --no-keep-going"

# GitLab-CI environment

  # more attempts for more robust
  GET_SOURCES_ATTEMPTS: "10"
  ARTIFACT_DOWNLOAD_ATTEMPTS: "10"

  # We use get-full-sources.sh script to fetch the submodules and/or re-fetch the repo
  # if it was corrupted (if submodule update fails this can happen)
  GIT_STRATEGY: fetch
  GIT_SUBMODULE_STRATEGY: none

  UNIT_TEST_BUILD_SYSTEM: make
# IDF environment

  IDF_PATH: "$CI_PROJECT_DIR"
  BATCH_BUILD: "1"
  V: "0"
  APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py"
  CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"

# Docker images
  BOT_DOCKER_IMAGE_TAG: ":latest"

# When 'fetch' strategy is used, Gitlab removes untracked files before checking out
# new revision. However if the new revision doesn't include some of the submodules
# which were present in the old revision, such submodule directories would not be
# removed by the checkout. This extra step ensures that these stale submodules
# are removed.
.git_clean_stale_submodules:  &git_clean_stale_submodules >
  find . -name '.git' -not -path './.git' -printf '%P\n'
  | sed 's|/.git||'
  | xargs -I {} sh -c '
  grep -q {} .gitmodules
  ||  (echo "Removing {}, has .git directory but not in .gitmodules file"
  && rm -rf {});'

# before each job, we need to check if this job is filtered by bot stage/job filter
.apply_bot_filter: &apply_bot_filter
  python $APPLY_BOT_FILTER_SCRIPT || exit 0

.setup_tools_unless_target_test: &setup_tools_unless_target_test |
  if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
  tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
  fi

before_script:
  - source tools/ci/setup_python.sh
  - *git_clean_stale_submodules
  # apply bot filter in before script
  - *apply_bot_filter
  # add gitlab ssh key
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
  - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
  - chmod 600 ~/.ssh/id_rsa
  - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
  # Download and install tools, if needed
  - *setup_tools_unless_target_test

  # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
  #
  # (the same regular expressions are used to set these are used in 'only:' sections below
  - source tools/ci/configure_ci_environment.sh

  # fetch the submodules (& if necessary re-fetch repo) from gitlab
  - time ./tools/ci/get-full-sources.sh

# used for check scripts which we want to run unconditionally
.do_nothing_before_no_filter:
  before_script: &do_nothing_before_no_filter
    - source tools/ci/setup_python.sh
    - *git_clean_stale_submodules

# used for everything else where we want to do no prep, except for bot filter
.do_nothing_before:
  before_script: &do_nothing_before
    - source tools/ci/setup_python.sh
    - *git_clean_stale_submodules
    # apply bot filter in before script
    - *apply_bot_filter
    - echo "Not setting up GitLab key, not fetching submodules"
    - source tools/ci/configure_ci_environment.sh

.add_gitlab_key_before:
  before_script: &add_gitlab_key_before
    - source tools/ci/setup_python.sh
    - *git_clean_stale_submodules
    # apply bot filter in before script
    - *apply_bot_filter
    - echo "Not fetching submodules"
    - source tools/ci/configure_ci_environment.sh
    # add gitlab ssh key
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config

build_template_app:
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - build
  variables:
    BATCH_BUILD: "1"
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
  script:
    # Set the variable for 'esp-idf-template' testing
    - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"}
    - git clone ${ESP_IDF_TEMPLATE_GIT}
    # Try to use the same branch name for esp-idf-template that we're
    # using on esp-idf. If it doesn't exist then just stick to the default
    # branch
    - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template
    - cd esp-idf-template
    - make defconfig
    # Test debug build (default)
    - make all V=1
    # Now test release build
    - make clean
    - sed -i.bak -e's/CONFIG_OPTIMIZATION_LEVEL_DEBUG\=y/CONFIG_OPTIMIZATION_LEVEL_RELEASE=y/' sdkconfig
    - make all V=1
    # Check if there are any stray printf/ets_printf references in WiFi libs
    - cd ../components/esp32/lib
    - test $(xtensa-esp32-elf-nm *.a | grep -w printf | wc -l) -eq 0
    - test $(xtensa-esp32-elf-nm *.a | grep -w ets_printf | wc -l) -eq 0


.build_template: &build_template
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - build
  variables:
    BATCH_BUILD: "1"
    V: "0"

build_ssc:
  <<: *build_template
  parallel: 3
  artifacts:
    paths:
      - SSC/ssc_bin
    expire_in: 1 week
  variables:
    SSC_CONFIG_FOLDER: "$CI_PROJECT_DIR/SSC/configs/ESP32_IDF"
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_INTEGRATION_TEST
      - $BOT_LABEL_REGULAR_TEST
  script:
    - git clone $SSC_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT SSC SSC
    - cd SSC
    - MAKEFLAGS= ./ci_build_ssc.sh

.build_esp_idf_unit_test_template: &build_esp_idf_unit_test_template
  <<: *build_template
  artifacts:
    paths:
      - tools/unit-test-app/output
      - components/idf_test/unit_test/TestCaseAll.yml
    expire_in: 4 days
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_UNIT_TEST
      - $BOT_LABEL_REGULAR_TEST

build_esp_idf_tests_make:
  <<: *build_esp_idf_unit_test_template
  script:
    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
    - cd $CI_PROJECT_DIR/tools/unit-test-app
    - MAKEFLAGS= make help # make sure kconfig tools are built in single process
    - make ut-clean-all-configs
    - make ut-build-all-configs
    - python tools/UnitTestParser.py
    # Check if the tests demand Make built binaries. If not, delete them
    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "make" ]; then exit 0; fi
    - rm -rf builds output sdkconfig
    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml

build_esp_idf_tests_cmake:
  <<: *build_esp_idf_unit_test_template
  script:
    - export PATH="$IDF_PATH/tools:$PATH"
    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
    - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS}
    - cd $CI_PROJECT_DIR/tools/unit-test-app
    - idf.py ut-clean-all-configs
    - idf.py ut-build-all-configs
    - python tools/UnitTestParser.py
    # Check if the tests demand CMake built binaries. If not, delete them
    - if [ "$UNIT_TEST_BUILD_SYSTEM" == "cmake" ]; then exit 0; fi
    - rm -rf builds output sdkconfig
    - rm $CI_PROJECT_DIR/components/idf_test/unit_test/TestCaseAll.yml

build_examples_make:
  <<: *build_template
  parallel: 8
  # This is a workaround for a rarely encountered issue with building examples in CI.
  # Probably related to building of Kconfig in 'make clean' stage
  retry: 1
  artifacts:
    when: always
    paths:
      - build_examples/*/*/*/build/*.bin
      - build_examples/*/*/*/sdkconfig
      - build_examples/*/*/*/build/*.elf
      - build_examples/*/*/*/build/*.map
      - build_examples/*/*/*/build/download.config
      - build_examples/*/*/*/build/bootloader/*.bin
      - $LOG_PATH
    expire_in: 4 days
  variables:
    LOG_PATH: "$CI_PROJECT_DIR/log_examples_make"
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_EXAMPLE_TEST
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_WEEKEND_TEST
  script:
    # it's not possible to build 100% out-of-tree and have the "artifacts"
    # mechanism work, but this is the next best thing
    - rm -rf build_examples
    - mkdir build_examples
    - cd build_examples
    # build some of examples
    - mkdir -p ${LOG_PATH}
    - ${IDF_PATH}/tools/ci/build_examples.sh

# same as above, but for CMake
build_examples_cmake:
  <<: *build_template
  parallel: 5
  artifacts:
    when: always
    paths:
      - build_examples_cmake/*/*/*/build/*.bin
      - build_examples_cmake/*/*/*/sdkconfig
      - build_examples_cmake/*/*/*/build/*.elf
      - build_examples_cmake/*/*/*/build/*.map
      - build_examples_cmake/*/*/*/build/flasher_args.json
      - build_examples_cmake/*/*/*/build/bootloader/*.bin
      - $LOG_PATH
    expire_in: 4 days
  variables:
    LOG_PATH: "$CI_PROJECT_DIR/log_examples_cmake"
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_EXAMPLE_TEST
      - $BOT_LABEL_REGULAR_TEST
      - $BOT_LABEL_WEEKEND_TEST
  script:
    # it's not possible to build 100% out-of-tree and have the "artifacts"
    # mechanism work, but this is the next best thing
    - rm -rf build_examples_cmake
    - mkdir build_examples_cmake
    - cd build_examples_cmake
    # build some of examples
    - mkdir -p ${LOG_PATH}
    - ${IDF_PATH}/tools/ci/build_examples_cmake.sh

build_docs:
  stage: build
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - build_docs
  artifacts:
    when: always
    paths:
      # English version of documentation
      - docs/en/doxygen-warning-log.txt
      - docs/en/sphinx-warning-log.txt
      - docs/en/sphinx-warning-log-sanitized.txt
      - docs/en/_build/html
      - docs/sphinx-err-*
      # Chinese version of documentation
      - docs/zh_CN/doxygen-warning-log.txt
      - docs/zh_CN/sphinx-warning-log.txt
      - docs/zh_CN/sphinx-warning-log-sanitized.txt
      - docs/zh_CN/_build/html
    expire_in: 4 days
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_BUILD_DOCS
      - $BOT_LABEL_REGULAR_TEST
  script:
    - cd docs
    - ./check_lang_folder_sync.sh
    - cd en
    - make gh-linkcheck
    - make html
    - ../check_doc_warnings.sh
    - cd ../zh_CN
    - make gh-linkcheck
    - make html
    - ../check_doc_warnings.sh

.check_job_template: &check_job_template
  stage: check
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - host_test
  dependencies: []
  before_script: *do_nothing_before_no_filter

verify_cmake_style:
  <<: *check_job_template
  stage: build
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD
      - $BOT_LABEL_REGULAR_TEST
  script:
    tools/cmake/run_cmake_lint.sh

build_docker:
  stage: build
  image: espressif/docker-builder:1
  tags:
    - build_docker_amd64_brno
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - schedules
  variables:
    DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
  before_script: []
  script:
    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}"
    # Build
    - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
    # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
    # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
    # Therefore, build a copy of the example located inside the container.
    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build

.host_test_template: &host_test_template
  stage: host_test
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - host_test
  dependencies: []
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_HOST_TEST
      - $BOT_LABEL_REGULAR_TEST

test_nvs_on_host:
  <<: *host_test_template
  script:
    - cd components/nvs_flash/test_nvs_host
    - make test

test_nvs_coverage:
  <<: *host_test_template
  artifacts:
    paths:
      - components/nvs_flash/test_nvs_host/coverage_report
    expire_in: 1 week
  only:
    refs:
      - triggers
    variables:
      - $BOT_LABEL_NVS_COVERAGE
  script:
    - cd components/nvs_flash/test_nvs_host
    - make coverage_report

test_partition_table_on_host:
  <<: *host_test_template
  tags:
    - build
  script:
    - cd components/partition_table/test_gen_esp32part_host
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py

test_wl_on_host:
  <<: *host_test_template
  artifacts:
    paths:
      - components/wear_levelling/test_wl_host/coverage_report.zip
    expire_in: 1 week
  script:
    - cd components/wear_levelling/test_wl_host
    - make test

test_fatfs_on_host:
  <<: *host_test_template
  script:
    - cd components/fatfs/test_fatfs_host/
    - make test

test_ldgen_on_host:
  <<: *host_test_template
  script:
    - cd tools/ldgen/test
    - ./test_fragments.py
    - ./test_generation.py

.host_fuzzer_test_template: &host_fuzzer_test_template
  stage: host_test
  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
  tags:
    - host_test
  dependencies: []
  artifacts:
    when: always
    paths:
      - ${FUZZER_TEST_DIR}/out/crashes
      - ${FUZZER_TEST_DIR}/fuzz_output.log
    expire_in: 1 week
  only:
    refs:
      # can only be triggered
      - triggers
    variables:
      - $BOT_LABEL_FUZZER_TEST
  script:
    - export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 && export AFL_SKIP_CPUFREQ=1
    - cd ${FUZZER_TEST_DIR}
    # run AFL fuzzer for one hour
    - ( ( make ${FUZZER_PARAMS} fuzz | tee fuzz_output.log | grep -v '\(Fuzzing test case\|Entering queue cycle\)' ) || pkill sleep ) &
    - ( sleep 3600 || mkdir -p out/crashes/env_failed ) && pkill afl-fuz
    # check no crashes found
    - test -z "$(ls out/crashes/)" || exit 1

test_mdns_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/mdns/test_afl_fuzz_host

test_lwip_dns_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dns

test_lwip_dhcp_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dhcp_client

test_lwip_dhcps_fuzzer_on_host:
  <<: *host_fuzzer_test_template
  variables:
    BOT_NEEDS_TRIGGER_BY_NAME: 1
    FUZZER_TEST_DIR: components/lwip/test_afl_host
    FUZZER_PARAMS: MODE=dhcp_server

test_spiffs_on_host:
  <<: *host_test_template
  script:
    - cd components/spiffs/test_spiffs_host/
    - make test

test_multi_heap_on_host:
  <<: *host_test_template
  script:
    - cd components/heap/test_multi_heap_host
    - ./test_all_configs.sh

test_confserver:
  <<: *host_test_template
  script:
    - cd tools/kconfig_new/test
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py

test_build_system:
  <<: *host_test_template
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system.sh

test_build_system_cmake:
  <<: *host_test_template
  script:
    - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
    - rm -rf test_build_system
    - mkdir test_build_system
    - cd test_build_system
    - ${IDF_PATH}/tools/ci/test_build_system_cmake.sh

test_idf_monitor:
  <<: *host_test_template
  artifacts:
    # save artifacts always in order to access results which were retried without consequent failure
    when: always
    paths:
      - tools/test_idf_monitor/outputs/*
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/test_idf_monitor
    - ./run_test_idf_monitor.py

test_idf_size:
  <<: *host_test_template
  artifacts:
    when: on_failure
    paths:
      - tools/test_idf_size/output
      - tools/test_idf_size/.coverage
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/test_idf_size
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh

test_esp_err_to_name_on_host:
  <<: *host_test_template
  artifacts:
    when: on_failure
    paths:
      - components/esp32/esp_err_to_name.c
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/tools/
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
    - git diff --exit-code -- ../components/esp32/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }

test_esp_efuse_table_on_host:
  <<: *host_test_template
  artifacts:
    when: on_failure
    paths:
      - components/efuse/esp32/esp_efuse_table.c
    expire_in: 1 week
  script:
    - cd ${IDF_PATH}/components/efuse/
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
    - cd ${IDF_PATH}/components/efuse/test_efuse_host
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py

test_espcoredump:
  <<: *host_test_template
  artifacts:
    when: always
    paths:
      - components/espcoredump/test/.coverage
      - components/espcoredump/test/output
    expire_in: 1 week
  script:
    - cd components/espcoredump/test/
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh

push_to_github:
  stage: deploy
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - deploy
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
  when: on_success
  dependencies: []
  before_script: *do_nothing_before
  script:
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64
    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
    - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
    - git remote remove github &>/dev/null || true
    - git remote add github git@github.com:espressif/esp-idf.git
    - tools/ci/push_to_github.sh

deploy_docs:
  stage: deploy
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - deploy
    - shiny
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD_DOCS
  dependencies:
    - build_docs
  before_script: *do_nothing_before
  script:
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64
    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
    - chmod 600 ~/.ssh/id_rsa
    - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config
    - export GIT_VER=$(git describe --always)
    - cd docs/en/_build/
    - mv html $GIT_VER
    - tar czvf $GIT_VER.tar.gz $GIT_VER
    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/en
    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/en && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
    - cd ../../zh_CN/_build/
    - mv html $GIT_VER
    - tar czvf $GIT_VER.tar.gz $GIT_VER
    - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH/zh_CN
    - ssh $DOCS_SERVER -x "cd $DOCS_PATH/zh_CN && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
    # add link to preview doc
    - echo "[document preview][en] $CI_DOCKER_REGISTRY/docs/esp-idf/en/${GIT_VER}/index.html"
    - echo "[document preview][zh_CN] $CI_DOCKER_REGISTRY/docs/esp-idf/zh_CN/${GIT_VER}/index.html"

check_doc_links:
  stage: host_test
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - check_doc_links
  only:
    refs:
      # can only be triggered
      - triggers
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_BUILD_DOCS
  artifacts:
    paths:
      - docs/_build/linkcheck
    expire_in: 1 week
  script:
    # must be triggered with CHECK_LINKS=Yes, otherwise exit without test
    - test "$CHECK_LINKS" = "Yes" || exit 0
    # can only run on master branch (otherwise the commit is not on Github yet)
    - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0
    - cd docs
    - make linkcheck

check_line_endings:
  <<: *check_job_template
  script:
    - tools/ci/check-line-endings.sh ${IDF_PATH}

check_commit_msg:
  <<: *check_job_template
  script:
    - git status
    - git log -n10 --oneline
    # commit start with "WIP: " need to be squashed before merge
    - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'

check_permissions:
  <<: *check_job_template
  script:
    - tools/ci/check-executable.sh

check_version:
  <<: *check_job_template
  # Don't run this for feature/bugfix branches, so that it is possible to modify
  # esp_idf_version.h in a branch before tagging the next version.
  only:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
  script:
    - export IDF_PATH=$PWD
    - tools/ci/check_idf_version.sh

check_examples_cmake_make:
  <<: *check_job_template
  except:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
  before_script: *do_nothing_before
  script:
    - tools/ci/check_examples_cmake_make.sh

check_python_style:
  <<: *check_job_template
  artifacts:
    when: on_failure
    paths:
      - flake8_output.txt
    expire_in: 1 week
  before_script: *do_nothing_before
  script:
    # run it only under Python 3 (it is very slow under Python 2)
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH

check_kconfigs:
  <<: *check_job_template
  before_script: *do_nothing_before
  artifacts:
    when: on_failure
    paths:
      - components/*/Kconfig*.new
      - examples/*/*/*/Kconfig*.new
      - examples/*/*/*/*/Kconfig*.new
      - tools/*/Kconfig*.new
      - tools/*/*/Kconfig*.new
      - tools/*/*/*/Kconfig*.new
    expire_in: 1 week
  script:
    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
    - ${IDF_PATH}/tools/check_kconfigs.py

check_ut_cmake_make:
  stage: check
  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - build
  except:
    - master
    - /^release\/v/
    - /^v\d+\.\d+(\.\d+)?($|-)/
  dependencies: []
  before_script: *do_nothing_before
  script:
    - tools/ci/check_ut_cmake_make.sh

check_submodule_sync:
  <<: *check_job_template
  tags:
    - github_sync
  variables:
    GIT_STRATEGY: clone
  retry: 2
  script:
    # check if all submodules are correctly synced to public repostory
    - git submodule update --init --recursive

check_artifacts_expire_time:
  <<: *check_job_template
  script:
    # check if we have set expire time for all artifacts
    - python tools/ci/check_artifacts_expire_time.py

check_pipeline_triggered_by_label:
  <<: *check_job_template
  stage: post_check
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL
  script:
    # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added.
    # We want to make sure some jobs are always executed to detect regression.
    - test "$BOT_LABEL_REGULAR_TEST" = "true" || exit -1

assign_test:
  tags:
    - assign_test
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  stage: assign_test
  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
  # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage.
  dependencies:
    - build_ssc
    - build_esp_idf_tests_make
    - build_esp_idf_tests_cmake
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
  artifacts:
    paths:
      - components/idf_test/*/CIConfigs
      - components/idf_test/*/TC.sqlite
      - $EXAMPLE_CONFIG_OUTPUT_PATH
    expire_in: 1 week
  only:
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_UNIT_TEST
      - $BOT_LABEL_INTEGRATION_TEST
      - $BOT_LABEL_EXAMPLE_TEST
  script:
    # assign example tests
    - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $IDF_PATH/.gitlab-ci.yml $EXAMPLE_CONFIG_OUTPUT_PATH
    # assign unit test cases
    - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $IDF_PATH/.gitlab-ci.yml $IDF_PATH/components/idf_test/unit_test/CIConfigs
    # clone test script to assign tests
    - git clone $TEST_SCRIPT_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
    # assgin integration test cases
    - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin

.define_config_file_name: &define_config_file_name |
    JOB_NAME_PREFIX=$(echo ${CI_JOB_NAME} | awk '{print $1}')
    JOG_FULL_NAME="${JOB_NAME_PREFIX}_${CI_NODE_INDEX}"
    CONFIG_FILE="${CONFIG_FILE_PATH}/${JOG_FULL_NAME}.yml"

.example_test_template: &example_test_template
  stage: target_test
  when: on_success
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_EXAMPLE_TEST
  dependencies:
    - assign_test
    - build_examples_make
    - build_examples_cmake
  artifacts:
    when: always
    paths:
      - $LOG_PATH
    expire_in: 1 week
    reports:
        junit: $LOG_PATH/*/XUNIT_RESULT.xml
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone test env configs
    - git clone $TEST_ENV_CONFIG_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    - cd $TEST_FW_PATH
    # run test
    - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE

.unit_test_template: &unit_test_template
  <<: *example_test_template
  stage: target_test
  dependencies:
    - assign_test
    - build_esp_idf_tests_make
    - build_esp_idf_tests_cmake
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_UNIT_TEST
  variables:
    TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
    TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/unit_test/CIConfigs"
    LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
    ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"

.test_template: &test_template
  stage: target_test
  when: on_success
  only:
    refs:
      - master
      - /^release\/v/
      - /^v\d+\.\d+(\.\d+)?($|-)/
      - triggers
      - schedules
    variables:
      - $BOT_TRIGGER_WITH_LABEL == null
      - $BOT_LABEL_INTEGRATION_TEST
  dependencies:
    - assign_test
    - build_ssc
  artifacts:
    when: always
    paths:
      - $LOG_PATH
    expire_in: 1 week
  variables:
    LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
    LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
    CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/components/idf_test/integration_test/CIConfigs"
  before_script: *add_gitlab_key_before
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - git clone $TEST_ENV_CONFIG_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    - git clone $TEST_SCRIPT_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
    # run test
    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

nvs_compatible_test:
  <<: *test_template
  artifacts:
    when: always
    paths:
      - $LOG_PATH
      - nvs_wifi.bin
    expire_in: 1 mos
  tags:
    - ESP32_IDF
    - NVS_Compatible
  script:
    - *define_config_file_name
    # first test if config file exists, if not exist, exit 0
    - test -e $CONFIG_FILE || exit 0
    # clone local test env configs
    - git clone $TEST_ENV_CONFIG_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
    # clone test bench
    - git clone $TEST_SCRIPT_REPOSITORY
    - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script
    - cd auto_test_script
    # prepare nvs bins
    - ./Tools/prepare_nvs_bin.sh
    # run test
    - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH

example_test_001:
  <<: *example_test_template
  parallel: 3
  tags:
    - ESP32
    - Example_WIFI

example_test_002:
  <<: *example_test_template
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  tags:
    - ESP32
    - Example_ShieldBox_Basic

.example_test_003:
  <<: *example_test_template
  tags:
    - ESP32
    - Example_SDIO

example_test_004A:
  <<: *example_test_template
  tags:
    - ESP32
    - Example_CAN1

example_test_004B:
  <<: *example_test_template
  tags:
    - ESP32
    - Example_CAN2

example_test_005:
  <<: *example_test_template
  tags:
    - ESP32
    - Example_WIFI_BT

example_test_006:
  <<: *example_test_template
  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
  only:
    variables:
      - $BOT_LABEL_IPERF_STRESS_TEST
  tags:
    - ESP32
    - Example_ShieldBox

example_test_007:
  <<: *example_test_template
  tags:
    - ESP32
    - Example_I2C_CCS811_SENSOR

UT_001:
  <<: *unit_test_template
  parallel: 50
  tags:
    - ESP32_IDF
    - UT_T1_1

# Max. allowed value of 'parallel' is 50.

UT_002:
  <<: *unit_test_template
  parallel: 16
  tags:
    - ESP32_IDF
    - UT_T1_1
    - psram

UT_003:
  <<: *unit_test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE

UT_004:
  <<: *unit_test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE

UT_005:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SDMODE
    - psram

UT_006:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_SPIMODE
    - psram

UT_007:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_GPIO

UT_008:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_GPIO
    - psram

UT_009:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_PCNT

UT_010:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_PCNT
    - psram

UT_011:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_LEDC

UT_012:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_LEDC
    - psram

UT_013:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T2_RS485

UT_014:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_RS485
    - psram

UT_015:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_RMT

UT_016:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_RMT
    - psram

UT_017:
  <<: *unit_test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - EMMC

UT_018:
  <<: *unit_test_template
  parallel: 5
  tags:
    - ESP32_IDF
    - UT_T1_1
    - 8Mpsram

UT_019:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - Example_SPI_Multi_device

UT_020:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - Example_SPI_Multi_device
    - psram

UT_021:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T2_I2C

UT_022:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_I2C
    - psram

UT_023:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_MCPWM

UT_024:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_MCPWM
    - psram

UT_025:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T1_I2S

UT_026:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_I2S
    - psram

UT_027:
  <<: *unit_test_template
  parallel: 4
  tags:
    - ESP32_IDF
    - UT_T2_1

UT_028:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_1
    - psram

UT_029:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T2_1
    - 8Mpsram

UT_030:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_1

UT_031:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_1

UT_032:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_1
    - psram

UT_033:
  extends: .unit_test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - UT_T1_PSRAMV0
    - psram

UT_034:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_no32kXTAL

UT_035:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_no32kXTAL

UT_036:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_no32kXTAL

UT_037:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_no32kXTAL
    - psram

UT_038:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_32kXTAL

UT_039:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_32kXTAL

UT_040:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_32kXTAL

UT_041:
  <<: *unit_test_template
  tags:
    - ESP32_IDF
    - UT_T1_32kXTAL
    - psram

IT_001:
  <<: *test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - SSC_T1_4

IT_002:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_2

IT_003:
  <<: *test_template
  parallel: 13
  tags:
    - ESP32_IDF
    - SSC_T2_5

IT_004:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_APC

IT_005:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_5

IT_006:
  <<: *test_template
  parallel: 7
  tags:
    - ESP32_IDF
    - SSC_T1_6

IT_007:
  <<: *test_template
  parallel: 3
  tags:
    - ESP32_IDF
    - SSC_T1_7

IT_008:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_8

IT_009:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_3

IT_011:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH1

IT_012:
  <<: *test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_MESH1

IT_011_03:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T2_MESH1

IT_013:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T3_MESH1

IT_014:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T6_MESH1

IT_015:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T12_MESH1

IT_016:
  <<: *test_template
  allow_failure: true
  tags:
    - ESP32_IDF
    - SSC_T50_MESH1

IT_017:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_MESH2

IT_018:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T1_9

IT_019:
  <<: *test_template
  parallel: 2
  tags:
    - ESP32_IDF
    - SSC_T2_2

IT_020:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T2_3

IT_021:
  <<: *test_template
  tags:
    - ESP32_IDF
    - SSC_T2_4