diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc2c59381..03480319a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,11 @@ variables: GET_SOURCES_ATTEMPTS: "10" ARTIFACT_DOWNLOAD_ATTEMPTS: "10" + # We use get_sources.sh script to fetch the submodules and/or re-fetch the repo + # if it was corrupted (if submodule update fails this can happen) + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: none + # IDF environment IDF_PATH: "$CI_PROJECT_DIR" @@ -34,10 +39,27 @@ before_script: # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not # # (the same regular expressions are used to set these are used in 'only:' sections below - - source make/configure_ci_environment.sh + - source tools/ci/configure_ci_environment.sh - # fetch all submodules - - git submodule update --init --recursive + # fetch the submodules (& if necessary re-fetch repo) from gitlab + - time ./tools/ci/get-full-sources.sh + +.do_nothing_before: + before_script: &do_nothing_before + - echo "Not setting up GitLab key, not fetching submodules" + - source tools/ci/configure_ci_environment.sh + +.add_gitlab_key_before: + before_script: &add_gitlab_key_before + - echo "Not fetching submodules" + - source tools/ci/configure_ci_environment.sh + # add gitlab ssh key + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 + - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config build_template_app: stage: build @@ -45,7 +67,6 @@ build_template_app: tags: - build variables: - GIT_STRATEGY: clone BATCH_BUILD: "1" IDF_CI_BUILD: "1" script: @@ -73,7 +94,6 @@ build_template_app: tags: - build variables: - GIT_STRATEGY: clone BATCH_BUILD: "1" V: "0" @@ -128,14 +148,13 @@ build_esp_idf_tests: expire_in: 1 week variables: IDF_CI_BUILD: "1" - GIT_STRATEGY: fetch script: # it's not possible to build 100% out-of-tree and have the "artifacts" # mechanism work, but this is the next best thing - mkdir build_examples - cd build_examples # build some of examples - - ${IDF_PATH}/make/build_examples.sh "${CI_JOB_NAME}" + - ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}" build_examples_00: <<: *build_examples_template @@ -209,8 +228,8 @@ test_build_system: - build_test dependencies: [] script: - - ./make/test_configure_ci_environment.sh - - ./make/test_build_system.sh + - ./tools/ci/test_configure_ci_environment.sh + - ./tools/ci/test_build_system.sh test_report: stage: test_report @@ -266,8 +285,6 @@ test_report: - test "${TEST_RESULT}" = "Pass" || exit 1 push_master_to_github: - before_script: - - echo "Not setting up GitLab key, not fetching submodules" stage: deploy image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: @@ -279,8 +296,8 @@ push_master_to_github: when: on_success dependencies: [] variables: - GIT_STRATEGY: clone GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master + before_script: *do_nothing_before script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -297,8 +314,6 @@ push_master_to_github: deploy_docs: - before_script: - - echo "Not setting up GitLab key, not fetching submodules" stage: deploy image: $CI_DOCKER_REGISTRY/esp32-ci-env tags: @@ -310,6 +325,7 @@ deploy_docs: - triggers dependencies: - build_docs + before_script: *do_nothing_before script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -354,12 +370,12 @@ check_commit_msg: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] - before_script: - - echo "skip update submodule" + before_script: *do_nothing_before script: - - git checkout ${CI_COMMIT_REF_NAME} + - git status + - git log -n10 --oneline # commit start with "WIP: " need to be squashed before merge - - 'git log --pretty=%s master..${CI_COMMIT_REF_NAME} | grep "^WIP: " || exit 0 && exit 1' + - 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0' check_submodule_sync: stage: deploy @@ -371,8 +387,9 @@ check_submodule_sync: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] - before_script: - - echo "do not use gitlab submodule repository" + variables: + GIT_STRATEGY: clone + before_script: *do_nothing_before script: # check if all submodules are correctly synced to public repostory - git submodule update --init --recursive @@ -389,6 +406,7 @@ assign_test: - components/idf_test/*/CIConfigs - components/idf_test/*/TC.sqlite expire_in: 1 mos + before_script: *add_gitlab_key_before script: # first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files - mkdir -p test_bins/ESP32_IDF/UT @@ -404,8 +422,6 @@ assign_test: - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins .test_template: &test_template - before_script: - - echo "Skip cloning submodule here" stage: test when: on_success only: @@ -422,25 +438,15 @@ assign_test: - $LOG_PATH expire_in: 6 mos variables: - # set git strategy to fetch so we can get esptool without update submodule - GIT_STRATEGY: fetch LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml" + before_script: *add_gitlab_key_before script: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 - # remove artifacts from the 'unit_test' stage - - rm -rf "$LOG_PATH" - # add gitlab ssh key - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY # clone test bench @@ -456,7 +462,6 @@ assign_test: allow_failure: false stage: unit_test variables: - GIT_STRATEGY: fetch LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test" @@ -475,15 +480,6 @@ nvs_compatible_test: - ESP32_IDF - NVS_Compatible script: - # remove artifacts from the 'unit_test' stage - - rm -rf "$LOG_PATH" - # add gitlab ssh key - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY # clone test bench diff --git a/make/build_examples.sh b/tools/ci/build_examples.sh similarity index 100% rename from make/build_examples.sh rename to tools/ci/build_examples.sh diff --git a/make/configure_ci_environment.sh b/tools/ci/configure_ci_environment.sh similarity index 59% rename from make/configure_ci_environment.sh rename to tools/ci/configure_ci_environment.sh index a421da9e4..8ddb64309 100644 --- a/make/configure_ci_environment.sh +++ b/tools/ci/configure_ci_environment.sh @@ -1,16 +1,15 @@ -#!/bin/bash -# -# Short script that is sourced in to the CI environment +# This file is sourced in to the CI environment # in .gitlab-ci.yml # -# Sets IS_PUBLIC and IS_PRIVATE based on branch type + +# Sets the error behaviour options for shell throughout the CI environment # -# Tweaks .gitmodules file for private builds +set -o errexit # Exit if command failed. [ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1 -REF=$CI_COMMIT_REF_NAME - +# Sets IS_PUBLIC and IS_PRIVATE based on branch type +# # Public branches are: # release branches - start with release/ # release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end @@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME # # These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file # +REF=$CI_COMMIT_REF_NAME if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then export IS_PUBLIC=1 else export IS_PRIVATE=1 fi - unset REF - -set -e - -if [[ $IS_PRIVATE ]]; then - # Redirect git submodules from public github to our private gitlab server - sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules - sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules -fi diff --git a/tools/ci/get-full-sources.sh b/tools/ci/get-full-sources.sh new file mode 100755 index 000000000..8ebaebbf3 --- /dev/null +++ b/tools/ci/get-full-sources.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Short script that is run as part of the CI environment +# in .gitlab-ci.yml +# +# Sets up submodules in the ESP-IDF source tree +# - Ideally, this just means doing a "git submodule update" +# - But if something goes wrong we re-clone the repo from scratch +# +# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone +# + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner." +[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment." + +SCRIPT_DIR=$(dirname -- "${0}") +update_submodules() { + if [ "${IS_PRIVATE}" ]; then + ${SCRIPT_DIR}/mirror-submodule-update.sh + else + git submodule foreach "git submodule deinit --force ." + git submodule deinit --force . + git submodule update --init --recursive + fi +} + +DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX) +del_files() { + # if non-empty + [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" ) +} +del_files_confirm() { + rm -rf "${DELETED_FILES}" +} + +RETRIES=10 +# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here... + +# For the first time, we try the fastest way. +for try in `seq $RETRIES`; do + echo "Trying to add submodules to existing repo..." + update_submodules && + echo "Fetch strategy submodules succeeded" && + exit 0 +done + +# Then we use the clean way. +for try in `seq $RETRIES`; do + cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain + echo "Trying a clean clone of IDF..." + del_files + git clone ${CI_REPOSITORY_URL} . && + git checkout ${CI_COMMIT_SHA} && + update_submodules && + echo "Clone strategy succeeded" && + del_files_confirm && + exit 0 + + echo "Clean clone failed..." +done + +die "Failed to clone repo & submodules together" diff --git a/tools/ci/mirror-list.txt b/tools/ci/mirror-list.txt new file mode 100644 index 000000000..549359f60 --- /dev/null +++ b/tools/ci/mirror-list.txt @@ -0,0 +1,10 @@ +components/esp32/lib @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git +components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git +components/aws_iot/aws-iot-device-sdk-embedded-C @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git ALLOW_TO_SYNC_FROM_PUBLIC +components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git ALLOW_TO_SYNC_FROM_PUBLIC +components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git ALLOW_TO_SYNC_FROM_PUBLIC +components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git ALLOW_TO_SYNC_FROM_PUBLIC +components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git ALLOW_TO_SYNC_FROM_PUBLIC +components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git ALLOW_TO_SYNC_FROM_PUBLIC +third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git ALLOW_TO_SYNC_FROM_PUBLIC +third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git ALLOW_TO_SYNC_FROM_PUBLIC diff --git a/tools/ci/mirror-submodule-update.sh b/tools/ci/mirror-submodule-update.sh new file mode 100755 index 000000000..38fa43dec --- /dev/null +++ b/tools/ci/mirror-submodule-update.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Redirects git submodules to the specified local mirrors and updates these recursively. +# +# To revert the changed URLs use 'git submodule deinit .' +# + +# ----------------------------------------------------------------------------- +# Common bash + +if [[ ! -z ${DEBUG} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. +set -o nounset # Exit if variable not set. + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +# ----------------------------------------------------------------------------- + +[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable" + +REPO_DIR=${1:-"${PWD}"} +REPO_DIR=$(readlink -f -- "${REPO_DIR}") + +SCRIPT_DIR=$(dirname -- "${0}") +SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}") + +SCRIPT_SH=$(readlink -f -- "${0}") + +MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt + +[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!" +[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!" +[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!" + +pushd ${REPO_DIR} >/dev/null + +# 0 +[ -f ".gitmodules" ] || exit 0 + +# 1 +git submodule init + + +# 2 +# Replacing each submodule URL of the current repository +# according to the one found in the MIRRORLIST + +# SED parses the strings like: +# +#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe +#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67) +# +for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/') +do + SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) + [ ${SUBMIRROR} ] || continue + SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}} + echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR" + + git config submodule.${SUBPATH}.url ${SUBMIRROR} +done + +# 3 +# Getting submodules of the current repository from the local mirrors +git submodule update + +# 4 +# Replacing URLs for each sub-submodule. +# The script runs recursively +git submodule foreach "${SCRIPT_SH}" # No '--recursive' + +popd >/dev/null diff --git a/tools/ci/mirror-synchronize.sh b/tools/ci/mirror-synchronize.sh new file mode 100755 index 000000000..163e7db7f --- /dev/null +++ b/tools/ci/mirror-synchronize.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Script for automate mirroring +# You can run this script manually +# + +# ----------------------------------------------------------------------------- +# Common bash + +if [[ ! -z ${DEBUG} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. +set -o nounset # Exit if variable not set. + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +# ----------------------------------------------------------------------------- + +[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable" + +REPO_DIR=${1:-"${PWD}"} +REPO_DIR=$(readlink -f -- "${REPO_DIR}") + +SCRIPT_DIR=$(dirname -- "${0}") +SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}") + +SCRIPT_SH=$(readlink -f -- "${0}") + +MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt + +[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!" +[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!" +[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!" + +pushd ${REPO_DIR} >/dev/null + +# 0 +[ -f ".gitmodules" ] || exit 0 + +# 1 +# Recursion +git submodule update --init +git submodule foreach "${SCRIPT_SH}" # No '--recursive' + +# 2 +git submodule deinit --force . +git submodule update --init + + +# 3 +# Mirroring from original URLs to ones listed in the MIRRORLIST + +# SED parses the strings like: +# +#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe +#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67) +# +for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/') +do + SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url) + SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) + [ ${SUBMIRROR} ] || continue + SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}} + + ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) + if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ] + then + echo "[should not to sync mirror] ${SUBMIRROR}" + continue + fi + + echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}" + + pushd ${SUBPATH} >/dev/null + git fetch --prune + git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/* + popd >/dev/null +done + +popd >/dev/null diff --git a/make/test_build_system.sh b/tools/ci/test_build_system.sh similarity index 100% rename from make/test_build_system.sh rename to tools/ci/test_build_system.sh diff --git a/make/test_configure_ci_environment.sh b/tools/ci/test_configure_ci_environment.sh similarity index 100% rename from make/test_configure_ci_environment.sh rename to tools/ci/test_configure_ci_environment.sh