From 52445ee5416a318d04bcf6e2e75cf9d72d378972 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Mon, 19 Jun 2017 09:59:18 +0800 Subject: [PATCH] CI: Use local mirrors for submodules The CI uses the mirror-submodule-update.sh for non-permanent reassignment of 3rdparty sub-modules to local mirrors (only for 'non-master' branches). List of the mirrors in mirror-list.txt You can use the script locally to speed up data retrieval on network problems check_submodule_sync job uses 'GIT_STRATEGY: clone' to check the availability of public sources --- .gitlab-ci.yml | 3 +- tools/ci/configure_ci_environment.sh | 23 +++----- tools/ci/get-full-sources.sh | 20 +++++-- tools/ci/mirror-list.txt | 10 ++++ tools/ci/mirror-submodule-update.sh | 80 ++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 22 deletions(-) create mode 100644 tools/ci/mirror-list.txt create mode 100755 tools/ci/mirror-submodule-update.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bde5db9e0..e0a3f5d8f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,7 +37,6 @@ before_script: - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config # Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not - # and adjust "our" submodule URLs if IS_PRIVATE # # (the same regular expressions are used to set these are used in 'only:' sections below - source tools/ci/configure_ci_environment.sh @@ -374,6 +373,8 @@ check_submodule_sync: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] + variables: + GIT_STRATEGY: clone before_script: - echo "Not setting up GitLab key, not fetching submodules" script: diff --git a/tools/ci/configure_ci_environment.sh b/tools/ci/configure_ci_environment.sh index a421da9e4..8ddb64309 100644 --- a/tools/ci/configure_ci_environment.sh +++ b/tools/ci/configure_ci_environment.sh @@ -1,16 +1,15 @@ -#!/bin/bash -# -# Short script that is sourced in to the CI environment +# This file is sourced in to the CI environment # in .gitlab-ci.yml # -# Sets IS_PUBLIC and IS_PRIVATE based on branch type + +# Sets the error behaviour options for shell throughout the CI environment # -# Tweaks .gitmodules file for private builds +set -o errexit # Exit if command failed. [ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1 -REF=$CI_COMMIT_REF_NAME - +# Sets IS_PUBLIC and IS_PRIVATE based on branch type +# # Public branches are: # release branches - start with release/ # release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end @@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME # # These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file # +REF=$CI_COMMIT_REF_NAME if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then export IS_PUBLIC=1 else export IS_PRIVATE=1 fi - unset REF - -set -e - -if [[ $IS_PRIVATE ]]; then - # Redirect git submodules from public github to our private gitlab server - sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules - sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules -fi diff --git a/tools/ci/get-full-sources.sh b/tools/ci/get-full-sources.sh index 35d794e97..8ebaebbf3 100755 --- a/tools/ci/get-full-sources.sh +++ b/tools/ci/get-full-sources.sh @@ -16,6 +16,18 @@ die() { } [ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner." +[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment." + +SCRIPT_DIR=$(dirname -- "${0}") +update_submodules() { + if [ "${IS_PRIVATE}" ]; then + ${SCRIPT_DIR}/mirror-submodule-update.sh + else + git submodule foreach "git submodule deinit --force ." + git submodule deinit --force . + git submodule update --init --recursive + fi +} DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX) del_files() { @@ -32,7 +44,7 @@ RETRIES=10 # For the first time, we try the fastest way. for try in `seq $RETRIES`; do echo "Trying to add submodules to existing repo..." - git submodule update --init --recursive && + update_submodules && echo "Fetch strategy submodules succeeded" && exit 0 done @@ -44,7 +56,7 @@ for try in `seq $RETRIES`; do del_files git clone ${CI_REPOSITORY_URL} . && git checkout ${CI_COMMIT_SHA} && - git submodule update --init --recursive && + update_submodules && echo "Clone strategy succeeded" && del_files_confirm && exit 0 @@ -52,6 +64,4 @@ for try in `seq $RETRIES`; do echo "Clean clone failed..." done -echo "Failed to clone repo & submodules together" - -exit 1 +die "Failed to clone repo & submodules together" diff --git a/tools/ci/mirror-list.txt b/tools/ci/mirror-list.txt new file mode 100644 index 000000000..15576eb10 --- /dev/null +++ b/tools/ci/mirror-list.txt @@ -0,0 +1,10 @@ +components/esp32/lib @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git +components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git +components/aws_iot/aws-iot-device-sdk-embedded-C @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git +components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git +components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git +components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git +components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git +components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git +third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git +third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git diff --git a/tools/ci/mirror-submodule-update.sh b/tools/ci/mirror-submodule-update.sh new file mode 100755 index 000000000..38fa43dec --- /dev/null +++ b/tools/ci/mirror-submodule-update.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Redirects git submodules to the specified local mirrors and updates these recursively. +# +# To revert the changed URLs use 'git submodule deinit .' +# + +# ----------------------------------------------------------------------------- +# Common bash + +if [[ ! -z ${DEBUG} ]] +then + set -x # Activate the expand mode if DEBUG is anything but empty. +fi + +set -o errexit # Exit if command failed. +set -o pipefail # Exit if pipe failed. +set -o nounset # Exit if variable not set. + +die() { + echo "${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +# ----------------------------------------------------------------------------- + +[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable" + +REPO_DIR=${1:-"${PWD}"} +REPO_DIR=$(readlink -f -- "${REPO_DIR}") + +SCRIPT_DIR=$(dirname -- "${0}") +SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}") + +SCRIPT_SH=$(readlink -f -- "${0}") + +MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt + +[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!" +[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!" +[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!" + +pushd ${REPO_DIR} >/dev/null + +# 0 +[ -f ".gitmodules" ] || exit 0 + +# 1 +git submodule init + + +# 2 +# Replacing each submodule URL of the current repository +# according to the one found in the MIRRORLIST + +# SED parses the strings like: +# +#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe +#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67) +# +for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/') +do + SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) + [ ${SUBMIRROR} ] || continue + SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}} + echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR" + + git config submodule.${SUBPATH}.url ${SUBMIRROR} +done + +# 3 +# Getting submodules of the current repository from the local mirrors +git submodule update + +# 4 +# Replacing URLs for each sub-submodule. +# The script runs recursively +git submodule foreach "${SCRIPT_SH}" # No '--recursive' + +popd >/dev/null