diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3e8f7c11..3897135ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,10 @@ stages: + - pre_check - build - assign_test - host_test - target_test - - check + - post_check - deploy - post_check @@ -120,7 +121,7 @@ after_script: - *cleanup_custom_toolchain .check_job_template: - stage: check + stage: pre_check image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - host_test @@ -128,7 +129,7 @@ after_script: extends: .before_script_lesser_nofilter .check_job_template_with_filter: - stage: check + stage: pre_check image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG tags: - host_test @@ -147,9 +148,10 @@ after_script: - source tools/ci/configure_ci_environment.sh include: + - '/tools/ci/config/pre_check.yml' - '/tools/ci/config/build.yml' - '/tools/ci/config/assign-test.yml' - '/tools/ci/config/host-test.yml' - '/tools/ci/config/target-test.yml' - - '/tools/ci/config/check.yml' + - '/tools/ci/config/post_check.yml' - '/tools/ci/config/deploy.yml' diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index fd714a7c6..b758e5e42 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -22,72 +22,6 @@ - $BOT_LABEL_UNIT_TEST - $BOT_LABEL_REGULAR_TEST - -.build_with_make_and_cmake: &build_with_make_and_cmake | - echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make" - make defconfig - make all - make clean - echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake" - rm -rf build sdkconfig - idf.py build - echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K" - -build_template_app: - stage: build - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - build - variables: - BATCH_BUILD: "1" - only: - variables: - - $BOT_TRIGGER_WITH_LABEL == null - - $BOT_LABEL_BUILD - - $BOT_LABEL_REGULAR_TEST - script: - # Set the variable for 'esp-idf-template' testing - - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - - git clone ${ESP_IDF_TEMPLATE_GIT} - # Try to use the same branch name for esp-idf-template that we're - # using on esp-idf. If it doesn't exist then just stick to the default branch - - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template - - cd esp-idf-template - - export PATH="$IDF_PATH/tools:$PATH" - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - - # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og - - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults - - *build_with_make_and_cmake - - # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0 - - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults - - *build_with_make_and_cmake - - # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os - - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults - - *build_with_make_and_cmake - - # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2 - - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults - - *build_with_make_and_cmake - - # Same as above, but also disable assertions. - - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults - # Don't error out on -Wunused, when assertions are disabled - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} - - *build_with_make_and_cmake - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - - # Check if there are any stray printf/ets_printf references in WiFi libs - - pushd ../components/esp_wifi/lib - - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done; - - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done; - - popd - build_ssc: extends: .build_template parallel: 3 @@ -279,17 +213,6 @@ build_docs: - make html - ../check_doc_warnings.sh -verify_cmake_style: - extends: .check_job_template - stage: build - only: - variables: - - $BOT_TRIGGER_WITH_LABEL == null - - $BOT_LABEL_BUILD - - $BOT_LABEL_REGULAR_TEST - script: - tools/cmake/run_cmake_lint.sh - test_build_system: extends: .build_template script: diff --git a/tools/ci/config/check.yml b/tools/ci/config/post_check.yml similarity index 56% rename from tools/ci/config/check.yml rename to tools/ci/config/post_check.yml index 23746404b..2f9dacc12 100644 --- a/tools/ci/config/check.yml +++ b/tools/ci/config/post_check.yml @@ -1,100 +1,10 @@ - # copy from .gitlab-ci.yml as anchor is not global .show_submodule_urls: &show_submodule_urls | git config --get-regexp '^submodule\..*\.url$' || true -check_line_endings: - extends: .check_job_template - script: - - tools/ci/check-line-endings.sh ${IDF_PATH} - -check_commit_msg: - extends: .check_job_template - script: - - git status - - git log -n10 --oneline - # commit start with "WIP: " need to be squashed before merge - - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0' - -check_permissions: - extends: .check_job_template - script: - - tools/ci/check-executable.sh - -check_version: - extends: .check_job_template - # Don't run this for feature/bugfix branches, so that it is possible to modify - # esp_idf_version.h in a branch before tagging the next version. - only: - - master - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - script: - - export IDF_PATH=$PWD - - tools/ci/check_idf_version.sh - -check_examples_cmake_make: - extends: .check_job_template_with_filter - except: - - master - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - script: - - tools/ci/check_examples_cmake_make.sh - -check_examples_rom_header: - extends: .check_job_template_with_filter - script: - - tools/ci/check_examples_rom_header.sh - -check_python_style: - extends: .check_job_template_with_filter - artifacts: - when: on_failure - paths: - - flake8_output.txt - expire_in: 1 week - script: - # run it only under Python 3 (it is very slow under Python 2) - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH - -check_kconfigs: - extends: .check_job_template_with_filter - artifacts: - when: on_failure - paths: - - components/*/Kconfig*.new - - examples/*/*/*/Kconfig*.new - - examples/*/*/*/*/Kconfig*.new - - tools/*/Kconfig*.new - - tools/*/*/Kconfig*.new - - tools/*/*/*/Kconfig*.new - expire_in: 1 week - script: - - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py - - ${IDF_PATH}/tools/check_kconfigs.py - -check_deprecated_kconfig_options: - extends: .check_job_template_with_filter - script: - - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py - -check_ut_cmake_make: - extends: .check_job_template_with_filter - stage: check - image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG - tags: - - build - except: - - master - - /^release\/v/ - - /^v\d+\.\d+(\.\d+)?($|-)/ - dependencies: [] - script: - - tools/ci/check_ut_cmake_make.sh - check_submodule_sync: extends: .check_job_template + stage: post_check tags: - github_sync retry: 2 @@ -125,8 +35,23 @@ check_wifi_lib_md5: - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh - IDF_TARGET=esp32s2beta $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh +check_ut_cmake_make: + extends: .check_job_template_with_filter + stage: post_check + image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + tags: + - build + except: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + dependencies: [] + script: + - tools/ci/check_ut_cmake_make.sh + check_artifacts_expire_time: extends: .check_job_template + stage: post_check script: # check if we have set expire time for all artifacts - python tools/ci/check_artifacts_expire_time.py @@ -141,3 +66,12 @@ check_pipeline_triggered_by_label: # If the pipeline is triggered with label, the pipeline will only succeeded if "regular_test" label is added. # We want to make sure some jobs are always executed to detect regression. - test "$BOT_LABEL_REGULAR_TEST" = "true" || { echo "CI can only pass if 'regular_test' label is included"; exit -1; } + +check_commit_msg: + extends: .check_job_template + stage: post_check + script: + - git status + - git log -n10 --oneline + # commit start with "WIP: " need to be squashed before merge + - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0' diff --git a/tools/ci/config/pre_check.yml b/tools/ci/config/pre_check.yml new file mode 100644 index 000000000..7b67e917e --- /dev/null +++ b/tools/ci/config/pre_check.yml @@ -0,0 +1,142 @@ +check_line_endings: + extends: .check_job_template + script: + - tools/ci/check-line-endings.sh ${IDF_PATH} + +check_permissions: + extends: .check_job_template + script: + - tools/ci/check-executable.sh + +check_version: + extends: .check_job_template + # Don't run this for feature/bugfix branches, so that it is possible to modify + # esp_idf_version.h in a branch before tagging the next version. + only: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + script: + - export IDF_PATH=$PWD + - tools/ci/check_idf_version.sh + +check_examples_cmake_make: + extends: .check_job_template_with_filter + except: + - master + - /^release\/v/ + - /^v\d+\.\d+(\.\d+)?($|-)/ + script: + - tools/ci/check_examples_cmake_make.sh + +check_examples_rom_header: + extends: .check_job_template_with_filter + script: + - tools/ci/check_examples_rom_header.sh + +check_python_style: + extends: .check_job_template_with_filter + artifacts: + when: on_failure + paths: + - flake8_output.txt + expire_in: 1 week + script: + # run it only under Python 3 (it is very slow under Python 2) + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH + +check_kconfigs: + extends: .check_job_template_with_filter + artifacts: + when: on_failure + paths: + - components/*/Kconfig*.new + - examples/*/*/*/Kconfig*.new + - examples/*/*/*/*/Kconfig*.new + - tools/*/Kconfig*.new + - tools/*/*/Kconfig*.new + - tools/*/*/*/Kconfig*.new + expire_in: 1 week + script: + - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py + - ${IDF_PATH}/tools/check_kconfigs.py + +check_deprecated_kconfig_options: + extends: .check_job_template_with_filter + script: + - ${IDF_PATH}/tools/ci/check_deprecated_kconfigs.py + +check_cmake_style: + extends: .check_job_template + only: + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_BUILD + - $BOT_LABEL_REGULAR_TEST + script: + tools/cmake/run_cmake_lint.sh + +.build_with_make_and_cmake: &build_with_make_and_cmake | + echo -e "section_end:"`date +%s`":build_script\r\e[0Ksection_start:"`date +%s`":build_make\r\e[0KBuild with Make" + make defconfig + make all + make clean + echo -e "section_end:"`date +%s`":build_make\r\e[0Ksection_start:"`date +%s`":build_cmake\r\e[0KBuild with CMake" + rm -rf build sdkconfig + idf.py build + echo -e "section_end:"`date +%s`":build_cmake\r\e[0Ksection_start:"`date +%s`":build_script\r\e[0K" + +build_template_app: + stage: pre_check + image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG + tags: + - build + variables: + BATCH_BUILD: "1" + only: + variables: + - $BOT_TRIGGER_WITH_LABEL == null + - $BOT_LABEL_BUILD + - $BOT_LABEL_REGULAR_TEST + script: + # Set the variable for 'esp-idf-template' testing + - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} + - git clone ${ESP_IDF_TEMPLATE_GIT} + # Try to use the same branch name for esp-idf-template that we're + # using on esp-idf. If it doesn't exist then just stick to the default branch + - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template + - cd esp-idf-template + - export PATH="$IDF_PATH/tools:$PATH" + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} + + # CONFIG_COMPILER_OPTIMIZATION_DEFAULT with flag -Og + - echo "CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y" > sdkconfig.defaults + - *build_with_make_and_cmake + + # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0 + - echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > sdkconfig.defaults + - *build_with_make_and_cmake + + # CONFIG_COMPILER_OPTIMIZATION_SIZE with flag -Os + - echo "CONFIG_COMPILER_OPTIMIZATION_SIZE=y" > sdkconfig.defaults + - *build_with_make_and_cmake + + # CONFIG_COMPILER_OPTIMIZATION_PERF with flag -O2 + - echo "CONFIG_COMPILER_OPTIMIZATION_PERF=y" > sdkconfig.defaults + - *build_with_make_and_cmake + + # Same as above, but also disable assertions. + - echo "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y" >> sdkconfig.defaults + # Don't error out on -Wunused, when assertions are disabled + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} + - *build_with_make_and_cmake + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} + + # Check if there are any stray printf/ets_printf references in WiFi libs + - pushd ../components/esp_wifi/lib + - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w printf | wc -l) -eq 0; done; + - for dir in esp32 esp32s2beta; do test $(xtensa-esp32-elf-nm $dir/*.a | grep -w ets_printf | wc -l) -eq 0; done; + - popd