stages: - build - assign_test - host_test - unit_test - integration_test - deploy variables: # System environment # Common parameters for the 'make' during CI tests MAKEFLAGS: "-j5 --no-keep-going" # GitLab-CI environment # more attempts for more robust GET_SOURCES_ATTEMPTS: "10" ARTIFACT_DOWNLOAD_ATTEMPTS: "10" # We use get-full-sources.sh script to fetch the submodules and/or re-fetch the repo # if it was corrupted (if submodule update fails this can happen) GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: none # IDF environment IDF_PATH: "$CI_PROJECT_DIR" APPLY_BOT_FILTER_SCRIPT: "$CI_PROJECT_DIR/tools/ci/apply_bot_filter.py" CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py" # When 'fetch' strategy is used, Gitlab removes untracked files before checking out # new revision. However if the new revision doesn't include some of the submodules # which were present in the old revision, such submodule directories would not be # removed by the checkout. This extra step ensures that these stale submodules # are removed. .git_clean_stale_submodules: &git_clean_stale_submodules > find . -name '.git' -not -path './.git' -printf '%P\n' | sed 's|/.git||' | xargs -I {} sh -c ' grep -q {} .gitmodules || (echo "Removing {}, has .git directory but not in .gitmodules file" && rm -rf {});' # before each job, we need to check if this job is filtered by bot stage/job filter .apply_bot_filter: &apply_bot_filter python $APPLY_BOT_FILTER_SCRIPT || exit 0 before_script: - source tools/ci/setup_python.sh - *git_clean_stale_submodules # apply bot filter in before script - *apply_bot_filter # add gitlab ssh key - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not # # (the same regular expressions are used to set these are used in 'only:' sections below - source tools/ci/configure_ci_environment.sh # fetch the submodules (& if necessary re-fetch repo) from gitlab - time ./tools/ci/get-full-sources.sh .do_nothing_before: before_script: &do_nothing_before - source tools/ci/setup_python.sh - *git_clean_stale_submodules # apply bot filter in before script - *apply_bot_filter - echo "Not setting up GitLab key, not fetching submodules" - source tools/ci/configure_ci_environment.sh .add_gitlab_key_before: before_script: &add_gitlab_key_before - source tools/ci/setup_python.sh - *git_clean_stale_submodules # apply bot filter in before script - *apply_bot_filter - echo "Not fetching submodules" - source tools/ci/configure_ci_environment.sh # add gitlab ssh key - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config build_template_app: stage: build image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build variables: BATCH_BUILD: "1" IDF_CI_BUILD: "1" script: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - git clone ${ESP_IDF_TEMPLATE_GIT} - cd esp-idf-template # Try to use the same branch name for esp-idf-template that we're # using on esp-idf. If it doesn't exist then just stick to the default # branch - python $CHECKOUT_REF_SCRIPT esp-idf-template - make defconfig # Test debug build (default) - make all V=1 # Now test release build - make clean - sed -i.bak -e's/CONFIG_OPTIMIZATION_LEVEL_DEBUG\=y/CONFIG_OPTIMIZATION_LEVEL_RELEASE=y/' sdkconfig - make all V=1 # Check if there are any stray printf/ets_printf references in WiFi libs - cd ../components/esp32/lib - test $(xtensa-esp32-elf-nm *.a | grep -w printf | wc -l) -eq 0 - test $(xtensa-esp32-elf-nm *.a | grep -w ets_printf | wc -l) -eq 0 .build_template: &build_template stage: build image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build variables: BATCH_BUILD: "1" V: "0" .build_ssc_template: &build_ssc_template <<: *build_template artifacts: paths: - SSC/ssc_bin expire_in: 1 mos variables: SSC_CONFIG_FOLDER: "$CI_PROJECT_DIR/SSC/configs/ESP32_IDF" script: - git clone $SSC_REPOSITORY - cd SSC - python $CHECKOUT_REF_SCRIPT SSC - MAKEFLAGS= ./ci_build_ssc.sh "${CI_JOB_NAME}" "${IDF_PATH}/.gitlab-ci.yml" # don't forget to add to dependency to test_template when adding new build_ssc jobs build_ssc_00: <<: *build_ssc_template build_ssc_01: <<: *build_ssc_template build_ssc_02: <<: *build_ssc_template # If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template` build_esp_idf_tests: <<: *build_template artifacts: paths: - tools/unit-test-app/output - components/idf_test/unit_test/TestCaseAll.yml - components/idf_test/unit_test/CIConfigs/*.yml expire_in: 1 mos script: - cd tools/unit-test-app - make help # make sure kconfig tools are built in single process - make ut-clean-all-configs - export EXTRA_CFLAGS="-Werror -Werror=deprecated-declarations" - export EXTRA_CXXFLAGS=${EXTRA_CFLAGS} - make ut-build-all-configs TESTS_ALL=1 - python tools/UnitTestParser.py .build_examples_template: &build_examples_template <<: *build_template artifacts: when: always paths: - build_examples/*/*/*/build/*.bin - build_examples/*/*/*/build/*.elf - build_examples/*/*/*/build/*.map - build_examples/*/*/*/build/download.config - build_examples/*/*/*/build/bootloader/*.bin - $LOG_PATH expire_in: 1 week variables: IDF_CI_BUILD: "1" LOG_PATH: "$CI_PROJECT_DIR/log_examples" script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing - rm -rf build_examples - mkdir build_examples - cd build_examples # build some of examples - mkdir -p ${LOG_PATH} - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}" build_examples_00: <<: *build_examples_template build_examples_01: <<: *build_examples_template build_examples_02: <<: *build_examples_template build_examples_03: <<: *build_examples_template build_examples_04: <<: *build_examples_template build_examples_05: <<: *build_examples_template build_examples_06: <<: *build_examples_template build_examples_07: <<: *build_examples_template # If you want to add new build example jobs, please add it into dependencies of `.example_test_template` build_docs: stage: build image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build_docs artifacts: when: always paths: - docs/doxygen-warning-log.txt - docs/sphinx-warning-log.txt - docs/sphinx-warning-log-sanitized.txt - docs/_build/html expire_in: 1 mos script: - cd docs - doxygen # If there are Doxygen warnings, print them and bail out - test $(cat doxygen-warning-log.txt | wc -l) -eq 0 || ( echo "Doxygen pass had some warnings:" && cat doxygen-warning-log.txt && false ) - make gh-linkcheck - make html - ./check_doc_warnings.sh .host_test_template: &host_test_template stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - host_test dependencies: [] test_nvs_on_host: <<: *host_test_template script: - cd components/nvs_flash/test_nvs_host - make test test_partition_table_on_host: <<: *host_test_template tags: - build script: - cd components/partition_table/test_gen_esp32part_host - ./gen_esp32part_tests.py test_wl_on_host: <<: *host_test_template artifacts: paths: - components/wear_levelling/test_wl_host/coverage_report.zip script: - cd components/wear_levelling/test_wl_host - make test test_multi_heap_on_host: <<: *host_test_template script: - cd components/heap/test_multi_heap_host - ./test_all_configs.sh test_build_system: <<: *host_test_template script: - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - rm -rf test_build_system - mkdir test_build_system - cd test_build_system - ${IDF_PATH}/tools/ci/test_build_system.sh push_to_github: stage: deploy image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - deploy only: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ when: on_success dependencies: [] before_script: *do_nothing_before script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo -n $GH_PUSH_KEY > ~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config - git remote remove github &>/dev/null || true - git remote add github git@github.com:espressif/esp-idf.git - tools/ci/push_to_github.sh deploy_docs: stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - deploy only: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers dependencies: - build_docs before_script: *do_nothing_before script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo -n $DOCS_DEPLOY_KEY > ~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - echo -e "Host $DOCS_SERVER\n\tStrictHostKeyChecking no\n\tUser $DOCS_SERVER_USER\n" >> ~/.ssh/config - export GIT_VER=$(git describe --always) - cd docs/_build/ - mv html $GIT_VER - tar czvf $GIT_VER.tar.gz $GIT_VER - scp $GIT_VER.tar.gz $DOCS_SERVER:$DOCS_PATH - ssh $DOCS_SERVER -x "cd $DOCS_PATH && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest" check_doc_links: stage: host_test image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - check_doc_links only: # can only be triggered - triggers artifacts: paths: - docs/_build/linkcheck expire_in: 1 mos script: # must be triggered with CHECK_LINKS=Yes, otherwise exit without test - test "$CHECK_LINKS" = "Yes" || exit 0 # can only run on master branch (otherwise the commit is not on Github yet) - test "${CI_COMMIT_REF_NAME}" = "master" || exit 0 - cd docs - make linkcheck check_commit_msg: stage: deploy image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - build except: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] before_script: *do_nothing_before script: - git status - git log -n10 --oneline # commit start with "WIP: " need to be squashed before merge - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0' check_submodule_sync: stage: deploy image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: - github_sync except: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] variables: GIT_STRATEGY: clone retry: 2 before_script: *do_nothing_before script: # check if all submodules are correctly synced to public repostory - git submodule update --init --recursive assign_test: tags: - assign_test image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG stage: assign_test # gitlab ci do not support match job with RegEx or wildcard now in dependencies. # we have a lot build example jobs. now we don't use dependencies, just download all artificats of build stage. dependencies: - build_ssc_00 - build_ssc_01 - build_ssc_02 - build_esp_idf_tests variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" artifacts: paths: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite - $EXAMPLE_CONFIG_OUTPUT_PATH expire_in: 1 mos before_script: *add_gitlab_key_before script: # assign example tests - python $TEST_FW_PATH/CIAssignExampleTest.py $IDF_PATH/examples $IDF_PATH/.gitlab-ci.yml $EXAMPLE_CONFIG_OUTPUT_PATH # assign unit test cases - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $IDF_PATH/.gitlab-ci.yml $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - git clone $TEST_SCRIPT_REPOSITORY - cd auto_test_script - python $CHECKOUT_REF_SCRIPT auto_test_script # assgin integration test cases - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin .example_test_template: &example_test_template stage: integration_test when: on_success only: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers dependencies: - assign_test - build_examples_00 - build_examples_01 - build_examples_02 - build_examples_03 - build_examples_04 - build_examples_05 - build_examples_06 - build_examples_07 artifacts: when: always paths: - $LOG_PATH expire_in: 1 mos variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/examples" CONFIG_FILE: "$CI_PROJECT_DIR/examples/test_configs/$CI_JOB_NAME.yml" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" script: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 - cd $TEST_FW_PATH # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE .test_template: &test_template stage: integration_test when: on_success only: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers dependencies: - assign_test - build_ssc_00 - build_ssc_01 - build_ssc_02 artifacts: when: always paths: - $LOG_PATH expire_in: 1 mos variables: LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml" before_script: *add_gitlab_key_before script: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - cd ci-test-runner-configs - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY - cd auto_test_script - python $CHECKOUT_REF_SCRIPT auto_test_script # run test - python CIRunner.py -l "$LOG_PATH/$CI_JOB_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH -m $MODULE_UPDATE_FILE # template for unit test jobs .unit_test_template: &unit_test_template <<: *example_test_template stage: unit_test dependencies: - assign_test - build_esp_idf_tests only: refs: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - triggers variables: TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw" TEST_CASE_PATH: "$CI_PROJECT_DIR/tools/unit-test-app" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/unit_test/CIConfigs/$CI_JOB_NAME.yml" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" nvs_compatible_test: <<: *test_template artifacts: when: always paths: - $LOG_PATH - nvs_wifi.bin expire_in: 1 mos tags: - ESP32_IDF - NVS_Compatible script: # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - cd ci-test-runner-configs - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY - cd auto_test_script - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..." # prepare nvs bins - ./Tools/prepare_nvs_bin.sh # run test - python CIRunner.py -l "$LOG_PATH/$CI_JOB_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH -m $MODULE_UPDATE_FILE example_test_001_01: <<: *example_test_template tags: - ESP32 - Example_WIFI UT_001_01: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_02: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_03: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_04: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_05: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_06: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_07: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_08: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_09: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_10: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_11: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_12: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_13: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_14: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_15: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_16: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_17: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_18: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_19: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_20: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_001_21: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 UT_002_01: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_02: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_03: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_04: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_05: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_06: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_002_07: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - psram UT_003_01: <<: *unit_test_template tags: - ESP32_IDF - UT_T2_1 UT_003_02: <<: *unit_test_template tags: - ESP32_IDF - UT_T2_1 UT_003_03: <<: *unit_test_template tags: - ESP32_IDF - UT_T2_1 UT_004_01: <<: *unit_test_template tags: - ESP32_IDF - UT_T2_1 - psram IT_001_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_02: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_03: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_04: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_05: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_06: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_07: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_08: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_001_09: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 IT_002_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_2 IT_003_01: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_02: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_03: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_04: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_05: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_06: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_003_07: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 IT_004_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_APC IT_005_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_WEP IT_006_01: <<: *test_template tags: - ESP32_IDF - SSC_T3_PhyMode IT_007_01: <<: *test_template tags: - ESP32_IDF - SSC_T2_PhyMode IT_008_01: <<: *test_template tags: - ESP32_IDF - SSC_T2_PhyMode IT_009_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_3 IT_501_01: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - stress_test IT_501_02: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - stress_test IT_501_03: <<: *test_template tags: - ESP32_IDF - SSC_T1_1 - stress_test IT_502_01: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 - stress_test IT_502_02: <<: *test_template tags: - ESP32_IDF - SSC_T2_1 - stress_test IT_503_01: <<: *test_template tags: - ESP32_IDF - SSC_T5_1 - stress_test IT_503_02: <<: *test_template tags: - ESP32_IDF - SSC_T5_1 - stress_test IT_503_03: <<: *test_template tags: - ESP32_IDF - SSC_T5_1 - stress_test