Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'feature/ci_mirror_for_submodules' into 'master'

Browse source 
Use custom fetch strategy. Use local mirrors for submodules

See merge request !890
This commit is contained in:
Anton Maklakov 2017-06-30 12:23:42 +08:00
parent 45c926f8f0 e55eeb0687
commit 6509947c1e
9 changed files with 290 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
.gitlab-ci.yml
View file

@ -18,6 +18,11 @@ variables:
GET_SOURCES_ATTEMPTS: "10"
ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
# We use get_sources.sh script to fetch the submodules and/or re-fetch the repo
# if it was corrupted (if submodule update fails this can happen)
GIT_STRATEGY: fetch
GIT_SUBMODULE_STRATEGY: none
# IDF environment
IDF_PATH: "$CI_PROJECT_DIR"
@ -34,10 +39,27 @@ before_script:
# Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
#
# (the same regular expressions are used to set these are used in 'only:' sections below
- source make/configure_ci_environment.sh
- source tools/ci/configure_ci_environment.sh
# fetch all submodules
- git submodule update --init --recursive
# fetch the submodules (& if necessary re-fetch repo) from gitlab
- time ./tools/ci/get-full-sources.sh
.do_nothing_before:
before_script: &do_nothing_before
- echo "Not setting up GitLab key, not fetching submodules"
- source tools/ci/configure_ci_environment.sh
.add_gitlab_key_before:
before_script: &add_gitlab_key_before
- echo "Not fetching submodules"
- source tools/ci/configure_ci_environment.sh
# add gitlab ssh key
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
build_template_app:
stage: build
@ -45,7 +67,6 @@ build_template_app:
tags:
- build
variables:
GIT_STRATEGY: clone
BATCH_BUILD: "1"
IDF_CI_BUILD: "1"
script:
@ -73,7 +94,6 @@ build_template_app:
tags:
- build
variables:
GIT_STRATEGY: clone
BATCH_BUILD: "1"
V: "0"
@ -128,14 +148,13 @@ build_esp_idf_tests:
expire_in: 1 week
variables:
IDF_CI_BUILD: "1"
GIT_STRATEGY: fetch
script:
# it's not possible to build 100% out-of-tree and have the "artifacts"
# mechanism work, but this is the next best thing
- mkdir build_examples
- cd build_examples
# build some of examples
- ${IDF_PATH}/make/build_examples.sh "${CI_JOB_NAME}"
- ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
build_examples_00:
<<: *build_examples_template
@ -209,8 +228,8 @@ test_build_system:
- build_test
dependencies: []
script:
- ./make/test_configure_ci_environment.sh
- ./make/test_build_system.sh
- ./tools/ci/test_configure_ci_environment.sh
- ./tools/ci/test_build_system.sh
test_report:
stage: test_report
@ -266,8 +285,6 @@ test_report:
- test "${TEST_RESULT}" = "Pass" || exit 1
push_master_to_github:
before_script:
- echo "Not setting up GitLab key, not fetching submodules"
stage: deploy
image: $CI_DOCKER_REGISTRY/esp32-ci-env
tags:
@ -279,8 +296,8 @@ push_master_to_github:
when: on_success
dependencies: []
variables:
GIT_STRATEGY: clone
GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master
before_script: *do_nothing_before
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
@ -297,8 +314,6 @@ push_master_to_github:
deploy_docs:
before_script:
- echo "Not setting up GitLab key, not fetching submodules"
stage: deploy
image: $CI_DOCKER_REGISTRY/esp32-ci-env
tags:
@ -310,6 +325,7 @@ deploy_docs:
- triggers
dependencies:
- build_docs
before_script: *do_nothing_before
script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
@ -354,12 +370,12 @@ check_commit_msg:
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
dependencies: []
before_script:
- echo "skip update submodule"
before_script: *do_nothing_before
script:
- git checkout ${CI_COMMIT_REF_NAME}
- git status
- git log -n10 --oneline
# commit start with "WIP: " need to be squashed before merge
- 'git log --pretty=%s master..${CI_COMMIT_REF_NAME} | grep "^WIP: " || exit 0 && exit 1'
- 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
check_submodule_sync:
stage: deploy
@ -371,8 +387,9 @@ check_submodule_sync:
- /^release\/v/
- /^v\d+\.\d+(\.\d+)?($|-)/
dependencies: []
before_script:
- echo "do not use gitlab submodule repository"
variables:
GIT_STRATEGY: clone
before_script: *do_nothing_before
script:
# check if all submodules are correctly synced to public repostory
- git submodule update --init --recursive
@ -389,6 +406,7 @@ assign_test:
- components/idf_test/*/CIConfigs
- components/idf_test/*/TC.sqlite
expire_in: 1 mos
before_script: *add_gitlab_key_before
script:
# first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
- mkdir -p test_bins/ESP32_IDF/UT
@ -404,8 +422,6 @@ assign_test:
- python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins
.test_template: &test_template
before_script:
- echo "Skip cloning submodule here"
stage: test
when: on_success
only:
@ -422,25 +438,15 @@ assign_test:
- $LOG_PATH
expire_in: 6 mos
variables:
# set git strategy to fetch so we can get esptool without update submodule
GIT_STRATEGY: fetch
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
before_script: *add_gitlab_key_before
script:
# first test if config file exists, if not exist, exit 0
- test -e $CONFIG_FILE || exit 0
# remove artifacts from the 'unit_test' stage
- rm -rf "$LOG_PATH"
# add gitlab ssh key
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
# clone local test env configs
- git clone $TEST_ENV_CONFIG_REPOSITORY
# clone test bench
@ -456,7 +462,6 @@ assign_test:
allow_failure: false
stage: unit_test
variables:
GIT_STRATEGY: fetch
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test"
@ -475,15 +480,6 @@ nvs_compatible_test:
- ESP32_IDF
- NVS_Compatible
script:
# remove artifacts from the 'unit_test' stage
- rm -rf "$LOG_PATH"
# add gitlab ssh key
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
# clone local test env configs
- git clone $TEST_ENV_CONFIG_REPOSITORY
# clone test bench

0
make/build_examples.sh → tools/ci/build_examples.sh
View file

23
make/configure_ci_environment.sh → tools/ci/configure_ci_environment.sh
View file

@ -1,16 +1,15 @@
#!/bin/bash
#
# Short script that is sourced in to the CI environment
# This file is sourced in to the CI environment
# in .gitlab-ci.yml
#
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
# Sets the error behaviour options for shell throughout the CI environment
#
# Tweaks .gitmodules file for private builds
set -o errexit # Exit if command failed.
[ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
REF=$CI_COMMIT_REF_NAME
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
#
# Public branches are:
# release branches - start with release/
# release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME
#
# These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
#
REF=$CI_COMMIT_REF_NAME
if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
export IS_PUBLIC=1
else
export IS_PRIVATE=1
fi
unset REF
set -e
if [[ $IS_PRIVATE ]]; then
# Redirect git submodules from public github to our private gitlab server
sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules
sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules
fi

67
tools/ci/get-full-sources.sh Executable file
View file

@ -0,0 +1,67 @@
#!/bin/bash
#
# Short script that is run as part of the CI environment
# in .gitlab-ci.yml
#
# Sets up submodules in the ESP-IDF source tree
# - Ideally, this just means doing a "git submodule update"
# - But if something goes wrong we re-clone the repo from scratch
#
# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
#
die() {
echo "${1:-"Unknown Error"}" 1>&2
exit 1
}
[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment."
SCRIPT_DIR=$(dirname -- "${0}")
update_submodules() {
if [ "${IS_PRIVATE}" ]; then
${SCRIPT_DIR}/mirror-submodule-update.sh
else
git submodule foreach "git submodule deinit --force ."
git submodule deinit --force .
git submodule update --init --recursive
fi
}
DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
del_files() {
# if non-empty
[ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
}
del_files_confirm() {
rm -rf "${DELETED_FILES}"
}
RETRIES=10
# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here...
# For the first time, we try the fastest way.
for try in `seq $RETRIES`; do
echo "Trying to add submodules to existing repo..."
update_submodules &&
echo "Fetch strategy submodules succeeded" &&
exit 0
done
# Then we use the clean way.
for try in `seq $RETRIES`; do
cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain
echo "Trying a clean clone of IDF..."
del_files
git clone ${CI_REPOSITORY_URL} . &&
git checkout ${CI_COMMIT_SHA} &&
update_submodules &&
echo "Clone strategy succeeded" &&
del_files_confirm &&
exit 0
echo "Clean clone failed..."
done
die "Failed to clone repo & submodules together"

10
tools/ci/mirror-list.txt Normal file
View file

@ -0,0 +1,10 @@
components/esp32/lib @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
components/aws_iot/aws-iot-device-sdk-embedded-C @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git ALLOW_TO_SYNC_FROM_PUBLIC
components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git ALLOW_TO_SYNC_FROM_PUBLIC
components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git ALLOW_TO_SYNC_FROM_PUBLIC
components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git ALLOW_TO_SYNC_FROM_PUBLIC
components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git ALLOW_TO_SYNC_FROM_PUBLIC
components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git ALLOW_TO_SYNC_FROM_PUBLIC
third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git ALLOW_TO_SYNC_FROM_PUBLIC
third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git ALLOW_TO_SYNC_FROM_PUBLIC

80
tools/ci/mirror-submodule-update.sh Executable file
View file

@ -0,0 +1,80 @@
#!/bin/bash
#
# Redirects git submodules to the specified local mirrors and updates these recursively.
#
# To revert the changed URLs use 'git submodule deinit .'
#
# -----------------------------------------------------------------------------
# Common bash
if [[ ! -z ${DEBUG} ]]
then
set -x # Activate the expand mode if DEBUG is anything but empty.
fi
set -o errexit # Exit if command failed.
set -o pipefail # Exit if pipe failed.
set -o nounset # Exit if variable not set.
die() {
echo "${1:-"Unknown Error"}" 1>&2
exit 1
}
# -----------------------------------------------------------------------------
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
REPO_DIR=${1:-"${PWD}"}
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
SCRIPT_DIR=$(dirname -- "${0}")
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
SCRIPT_SH=$(readlink -f -- "${0}")
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
pushd ${REPO_DIR} >/dev/null
# 0
[ -f ".gitmodules" ] || exit 0
# 1
git submodule init
# 2
# Replacing each submodule URL of the current repository
# according to the one found in the MIRRORLIST
# SED parses the strings like:
#
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
#
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
do
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
[ ${SUBMIRROR} ] || continue
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
git config submodule.${SUBPATH}.url ${SUBMIRROR}
done
# 3
# Getting submodules of the current repository from the local mirrors
git submodule update
# 4
# Replacing URLs for each sub-submodule.
# The script runs recursively
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
popd >/dev/null

87
tools/ci/mirror-synchronize.sh Executable file
View file

@ -0,0 +1,87 @@
#!/bin/bash
#
# Script for automate mirroring
# You can run this script manually
#
# -----------------------------------------------------------------------------
# Common bash
if [[ ! -z ${DEBUG} ]]
then
set -x # Activate the expand mode if DEBUG is anything but empty.
fi
set -o errexit # Exit if command failed.
set -o pipefail # Exit if pipe failed.
set -o nounset # Exit if variable not set.
die() {
echo "${1:-"Unknown Error"}" 1>&2
exit 1
}
# -----------------------------------------------------------------------------
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
REPO_DIR=${1:-"${PWD}"}
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
SCRIPT_DIR=$(dirname -- "${0}")
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
SCRIPT_SH=$(readlink -f -- "${0}")
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
pushd ${REPO_DIR} >/dev/null
# 0
[ -f ".gitmodules" ] || exit 0
# 1
# Recursion
git submodule update --init
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
# 2
git submodule deinit --force .
git submodule update --init
# 3
# Mirroring from original URLs to ones listed in the MIRRORLIST
# SED parses the strings like:
#
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
#
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
do
SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
[ ${SUBMIRROR} ] || continue
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
then
echo "[should not to sync mirror] ${SUBMIRROR}"
continue
fi
echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
pushd ${SUBPATH} >/dev/null
git fetch --prune
git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
popd >/dev/null
done
popd >/dev/null

0
make/test_build_system.sh → tools/ci/test_build_system.sh
View file

0
make/test_configure_ci_environment.sh → tools/ci/test_configure_ci_environment.sh
View file