Merge branch 'feature/ci_mirror_for_submodules' into 'master'
Use custom fetch strategy. Use local mirrors for submodules See merge request !890
This commit is contained in:
commit
6509947c1e
|
@ -18,6 +18,11 @@ variables:
|
||||||
GET_SOURCES_ATTEMPTS: "10"
|
GET_SOURCES_ATTEMPTS: "10"
|
||||||
ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
|
ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
|
||||||
|
|
||||||
|
# We use get_sources.sh script to fetch the submodules and/or re-fetch the repo
|
||||||
|
# if it was corrupted (if submodule update fails this can happen)
|
||||||
|
GIT_STRATEGY: fetch
|
||||||
|
GIT_SUBMODULE_STRATEGY: none
|
||||||
|
|
||||||
# IDF environment
|
# IDF environment
|
||||||
|
|
||||||
IDF_PATH: "$CI_PROJECT_DIR"
|
IDF_PATH: "$CI_PROJECT_DIR"
|
||||||
|
@ -34,10 +39,27 @@ before_script:
|
||||||
# Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
|
# Set IS_PRIVATE or IS_PUBLIC depending on if our branch is public or not
|
||||||
#
|
#
|
||||||
# (the same regular expressions are used to set these are used in 'only:' sections below
|
# (the same regular expressions are used to set these are used in 'only:' sections below
|
||||||
- source make/configure_ci_environment.sh
|
- source tools/ci/configure_ci_environment.sh
|
||||||
|
|
||||||
# fetch all submodules
|
# fetch the submodules (& if necessary re-fetch repo) from gitlab
|
||||||
- git submodule update --init --recursive
|
- time ./tools/ci/get-full-sources.sh
|
||||||
|
|
||||||
|
.do_nothing_before:
|
||||||
|
before_script: &do_nothing_before
|
||||||
|
- echo "Not setting up GitLab key, not fetching submodules"
|
||||||
|
- source tools/ci/configure_ci_environment.sh
|
||||||
|
|
||||||
|
.add_gitlab_key_before:
|
||||||
|
before_script: &add_gitlab_key_before
|
||||||
|
- echo "Not fetching submodules"
|
||||||
|
- source tools/ci/configure_ci_environment.sh
|
||||||
|
# add gitlab ssh key
|
||||||
|
- mkdir -p ~/.ssh
|
||||||
|
- chmod 700 ~/.ssh
|
||||||
|
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
|
||||||
|
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
||||||
|
- chmod 600 ~/.ssh/id_rsa
|
||||||
|
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
||||||
|
|
||||||
build_template_app:
|
build_template_app:
|
||||||
stage: build
|
stage: build
|
||||||
|
@ -45,7 +67,6 @@ build_template_app:
|
||||||
tags:
|
tags:
|
||||||
- build
|
- build
|
||||||
variables:
|
variables:
|
||||||
GIT_STRATEGY: clone
|
|
||||||
BATCH_BUILD: "1"
|
BATCH_BUILD: "1"
|
||||||
IDF_CI_BUILD: "1"
|
IDF_CI_BUILD: "1"
|
||||||
script:
|
script:
|
||||||
|
@ -73,7 +94,6 @@ build_template_app:
|
||||||
tags:
|
tags:
|
||||||
- build
|
- build
|
||||||
variables:
|
variables:
|
||||||
GIT_STRATEGY: clone
|
|
||||||
BATCH_BUILD: "1"
|
BATCH_BUILD: "1"
|
||||||
V: "0"
|
V: "0"
|
||||||
|
|
||||||
|
@ -128,14 +148,13 @@ build_esp_idf_tests:
|
||||||
expire_in: 1 week
|
expire_in: 1 week
|
||||||
variables:
|
variables:
|
||||||
IDF_CI_BUILD: "1"
|
IDF_CI_BUILD: "1"
|
||||||
GIT_STRATEGY: fetch
|
|
||||||
script:
|
script:
|
||||||
# it's not possible to build 100% out-of-tree and have the "artifacts"
|
# it's not possible to build 100% out-of-tree and have the "artifacts"
|
||||||
# mechanism work, but this is the next best thing
|
# mechanism work, but this is the next best thing
|
||||||
- mkdir build_examples
|
- mkdir build_examples
|
||||||
- cd build_examples
|
- cd build_examples
|
||||||
# build some of examples
|
# build some of examples
|
||||||
- ${IDF_PATH}/make/build_examples.sh "${CI_JOB_NAME}"
|
- ${IDF_PATH}/tools/ci/build_examples.sh "${CI_JOB_NAME}"
|
||||||
|
|
||||||
build_examples_00:
|
build_examples_00:
|
||||||
<<: *build_examples_template
|
<<: *build_examples_template
|
||||||
|
@ -209,8 +228,8 @@ test_build_system:
|
||||||
- build_test
|
- build_test
|
||||||
dependencies: []
|
dependencies: []
|
||||||
script:
|
script:
|
||||||
- ./make/test_configure_ci_environment.sh
|
- ./tools/ci/test_configure_ci_environment.sh
|
||||||
- ./make/test_build_system.sh
|
- ./tools/ci/test_build_system.sh
|
||||||
|
|
||||||
test_report:
|
test_report:
|
||||||
stage: test_report
|
stage: test_report
|
||||||
|
@ -266,8 +285,6 @@ test_report:
|
||||||
- test "${TEST_RESULT}" = "Pass" || exit 1
|
- test "${TEST_RESULT}" = "Pass" || exit 1
|
||||||
|
|
||||||
push_master_to_github:
|
push_master_to_github:
|
||||||
before_script:
|
|
||||||
- echo "Not setting up GitLab key, not fetching submodules"
|
|
||||||
stage: deploy
|
stage: deploy
|
||||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env
|
image: $CI_DOCKER_REGISTRY/esp32-ci-env
|
||||||
tags:
|
tags:
|
||||||
|
@ -279,8 +296,8 @@ push_master_to_github:
|
||||||
when: on_success
|
when: on_success
|
||||||
dependencies: []
|
dependencies: []
|
||||||
variables:
|
variables:
|
||||||
GIT_STRATEGY: clone
|
|
||||||
GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master
|
GITHUB_PUSH_REFS: refs/remotes/origin/release refs/remotes/origin/master
|
||||||
|
before_script: *do_nothing_before
|
||||||
script:
|
script:
|
||||||
- mkdir -p ~/.ssh
|
- mkdir -p ~/.ssh
|
||||||
- chmod 700 ~/.ssh
|
- chmod 700 ~/.ssh
|
||||||
|
@ -297,8 +314,6 @@ push_master_to_github:
|
||||||
|
|
||||||
|
|
||||||
deploy_docs:
|
deploy_docs:
|
||||||
before_script:
|
|
||||||
- echo "Not setting up GitLab key, not fetching submodules"
|
|
||||||
stage: deploy
|
stage: deploy
|
||||||
image: $CI_DOCKER_REGISTRY/esp32-ci-env
|
image: $CI_DOCKER_REGISTRY/esp32-ci-env
|
||||||
tags:
|
tags:
|
||||||
|
@ -310,6 +325,7 @@ deploy_docs:
|
||||||
- triggers
|
- triggers
|
||||||
dependencies:
|
dependencies:
|
||||||
- build_docs
|
- build_docs
|
||||||
|
before_script: *do_nothing_before
|
||||||
script:
|
script:
|
||||||
- mkdir -p ~/.ssh
|
- mkdir -p ~/.ssh
|
||||||
- chmod 700 ~/.ssh
|
- chmod 700 ~/.ssh
|
||||||
|
@ -354,12 +370,12 @@ check_commit_msg:
|
||||||
- /^release\/v/
|
- /^release\/v/
|
||||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||||
dependencies: []
|
dependencies: []
|
||||||
before_script:
|
before_script: *do_nothing_before
|
||||||
- echo "skip update submodule"
|
|
||||||
script:
|
script:
|
||||||
- git checkout ${CI_COMMIT_REF_NAME}
|
- git status
|
||||||
|
- git log -n10 --oneline
|
||||||
# commit start with "WIP: " need to be squashed before merge
|
# commit start with "WIP: " need to be squashed before merge
|
||||||
- 'git log --pretty=%s master..${CI_COMMIT_REF_NAME} | grep "^WIP: " || exit 0 && exit 1'
|
- 'git log --pretty=%s master.. -- | grep "^WIP: " && exit 1 || exit 0'
|
||||||
|
|
||||||
check_submodule_sync:
|
check_submodule_sync:
|
||||||
stage: deploy
|
stage: deploy
|
||||||
|
@ -371,8 +387,9 @@ check_submodule_sync:
|
||||||
- /^release\/v/
|
- /^release\/v/
|
||||||
- /^v\d+\.\d+(\.\d+)?($|-)/
|
- /^v\d+\.\d+(\.\d+)?($|-)/
|
||||||
dependencies: []
|
dependencies: []
|
||||||
before_script:
|
variables:
|
||||||
- echo "do not use gitlab submodule repository"
|
GIT_STRATEGY: clone
|
||||||
|
before_script: *do_nothing_before
|
||||||
script:
|
script:
|
||||||
# check if all submodules are correctly synced to public repostory
|
# check if all submodules are correctly synced to public repostory
|
||||||
- git submodule update --init --recursive
|
- git submodule update --init --recursive
|
||||||
|
@ -389,6 +406,7 @@ assign_test:
|
||||||
- components/idf_test/*/CIConfigs
|
- components/idf_test/*/CIConfigs
|
||||||
- components/idf_test/*/TC.sqlite
|
- components/idf_test/*/TC.sqlite
|
||||||
expire_in: 1 mos
|
expire_in: 1 mos
|
||||||
|
before_script: *add_gitlab_key_before
|
||||||
script:
|
script:
|
||||||
# first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
|
# first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
|
||||||
- mkdir -p test_bins/ESP32_IDF/UT
|
- mkdir -p test_bins/ESP32_IDF/UT
|
||||||
|
@ -404,8 +422,6 @@ assign_test:
|
||||||
- python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins
|
- python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/test_bins
|
||||||
|
|
||||||
.test_template: &test_template
|
.test_template: &test_template
|
||||||
before_script:
|
|
||||||
- echo "Skip cloning submodule here"
|
|
||||||
stage: test
|
stage: test
|
||||||
when: on_success
|
when: on_success
|
||||||
only:
|
only:
|
||||||
|
@ -422,25 +438,15 @@ assign_test:
|
||||||
- $LOG_PATH
|
- $LOG_PATH
|
||||||
expire_in: 6 mos
|
expire_in: 6 mos
|
||||||
variables:
|
variables:
|
||||||
# set git strategy to fetch so we can get esptool without update submodule
|
|
||||||
GIT_STRATEGY: fetch
|
|
||||||
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
||||||
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
|
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
|
||||||
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
|
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
|
||||||
MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
|
MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
|
||||||
CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
|
CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
|
||||||
|
before_script: *add_gitlab_key_before
|
||||||
script:
|
script:
|
||||||
# first test if config file exists, if not exist, exit 0
|
# first test if config file exists, if not exist, exit 0
|
||||||
- test -e $CONFIG_FILE || exit 0
|
- test -e $CONFIG_FILE || exit 0
|
||||||
# remove artifacts from the 'unit_test' stage
|
|
||||||
- rm -rf "$LOG_PATH"
|
|
||||||
# add gitlab ssh key
|
|
||||||
- mkdir -p ~/.ssh
|
|
||||||
- chmod 700 ~/.ssh
|
|
||||||
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
|
|
||||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
|
||||||
- chmod 600 ~/.ssh/id_rsa
|
|
||||||
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
|
||||||
# clone local test env configs
|
# clone local test env configs
|
||||||
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
||||||
# clone test bench
|
# clone test bench
|
||||||
|
@ -456,7 +462,6 @@ assign_test:
|
||||||
allow_failure: false
|
allow_failure: false
|
||||||
stage: unit_test
|
stage: unit_test
|
||||||
variables:
|
variables:
|
||||||
GIT_STRATEGY: fetch
|
|
||||||
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
|
||||||
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
|
LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
|
||||||
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test"
|
TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/unit_test"
|
||||||
|
@ -475,15 +480,6 @@ nvs_compatible_test:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- NVS_Compatible
|
- NVS_Compatible
|
||||||
script:
|
script:
|
||||||
# remove artifacts from the 'unit_test' stage
|
|
||||||
- rm -rf "$LOG_PATH"
|
|
||||||
# add gitlab ssh key
|
|
||||||
- mkdir -p ~/.ssh
|
|
||||||
- chmod 700 ~/.ssh
|
|
||||||
- echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
|
|
||||||
- base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
|
|
||||||
- chmod 600 ~/.ssh/id_rsa
|
|
||||||
- echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
|
|
||||||
# clone local test env configs
|
# clone local test env configs
|
||||||
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
- git clone $TEST_ENV_CONFIG_REPOSITORY
|
||||||
# clone test bench
|
# clone test bench
|
||||||
|
|
|
@ -1,16 +1,15 @@
|
||||||
#!/bin/bash
|
# This file is sourced in to the CI environment
|
||||||
#
|
|
||||||
# Short script that is sourced in to the CI environment
|
|
||||||
# in .gitlab-ci.yml
|
# in .gitlab-ci.yml
|
||||||
#
|
#
|
||||||
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
|
|
||||||
|
# Sets the error behaviour options for shell throughout the CI environment
|
||||||
#
|
#
|
||||||
# Tweaks .gitmodules file for private builds
|
set -o errexit # Exit if command failed.
|
||||||
|
|
||||||
[ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
|
[ -z $CI_COMMIT_REF_NAME ] && echo "This internal script should only be run by a Gitlab CI runner." && exit 1
|
||||||
|
|
||||||
REF=$CI_COMMIT_REF_NAME
|
# Sets IS_PUBLIC and IS_PRIVATE based on branch type
|
||||||
|
#
|
||||||
# Public branches are:
|
# Public branches are:
|
||||||
# release branches - start with release/
|
# release branches - start with release/
|
||||||
# release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
|
# release tags - look like vXX.YY or vXX.YY.ZZ with an optional dash followed by anything on the end
|
||||||
|
@ -18,18 +17,10 @@ REF=$CI_COMMIT_REF_NAME
|
||||||
#
|
#
|
||||||
# These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
|
# These POSIX REs are equivalent to the REs in some "only:" sections of the gitlab-ci.yml file
|
||||||
#
|
#
|
||||||
|
REF=$CI_COMMIT_REF_NAME
|
||||||
if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
|
if [[ $REF = "master" || $REF =~ ^release/v || $REF =~ ^v[0-9]+\.[0-9]+(\.[0-9]+)?(-|$) ]]; then
|
||||||
export IS_PUBLIC=1
|
export IS_PUBLIC=1
|
||||||
else
|
else
|
||||||
export IS_PRIVATE=1
|
export IS_PRIVATE=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
unset REF
|
unset REF
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [[ $IS_PRIVATE ]]; then
|
|
||||||
# Redirect git submodules from public github to our private gitlab server
|
|
||||||
sed -i "s%https://github.com/espressif/esp32-wifi-lib%${GITLAB_SSH_SERVER}/idf/esp32-wifi-lib%" .gitmodules
|
|
||||||
sed -i "s%https://github.com/espressif/esp32-bt-lib%${GITLAB_SSH_SERVER}/idf/esp32-bt-lib%" .gitmodules
|
|
||||||
fi
|
|
67
tools/ci/get-full-sources.sh
Executable file
67
tools/ci/get-full-sources.sh
Executable file
|
@ -0,0 +1,67 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Short script that is run as part of the CI environment
|
||||||
|
# in .gitlab-ci.yml
|
||||||
|
#
|
||||||
|
# Sets up submodules in the ESP-IDF source tree
|
||||||
|
# - Ideally, this just means doing a "git submodule update"
|
||||||
|
# - But if something goes wrong we re-clone the repo from scratch
|
||||||
|
#
|
||||||
|
# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
|
||||||
|
#
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "${1:-"Unknown Error"}" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
|
||||||
|
[[ ( -z ${IS_PRIVATE} ) && ( -z ${IS_PUBLIC} ) ]] && die "IS_PRIVATE or IS_PUBLIC should be defined in the CI environment."
|
||||||
|
|
||||||
|
SCRIPT_DIR=$(dirname -- "${0}")
|
||||||
|
update_submodules() {
|
||||||
|
if [ "${IS_PRIVATE}" ]; then
|
||||||
|
${SCRIPT_DIR}/mirror-submodule-update.sh
|
||||||
|
else
|
||||||
|
git submodule foreach "git submodule deinit --force ."
|
||||||
|
git submodule deinit --force .
|
||||||
|
git submodule update --init --recursive
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
|
||||||
|
del_files() {
|
||||||
|
# if non-empty
|
||||||
|
[ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
|
||||||
|
}
|
||||||
|
del_files_confirm() {
|
||||||
|
rm -rf "${DELETED_FILES}"
|
||||||
|
}
|
||||||
|
|
||||||
|
RETRIES=10
|
||||||
|
# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here...
|
||||||
|
|
||||||
|
# For the first time, we try the fastest way.
|
||||||
|
for try in `seq $RETRIES`; do
|
||||||
|
echo "Trying to add submodules to existing repo..."
|
||||||
|
update_submodules &&
|
||||||
|
echo "Fetch strategy submodules succeeded" &&
|
||||||
|
exit 0
|
||||||
|
done
|
||||||
|
|
||||||
|
# Then we use the clean way.
|
||||||
|
for try in `seq $RETRIES`; do
|
||||||
|
cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain
|
||||||
|
echo "Trying a clean clone of IDF..."
|
||||||
|
del_files
|
||||||
|
git clone ${CI_REPOSITORY_URL} . &&
|
||||||
|
git checkout ${CI_COMMIT_SHA} &&
|
||||||
|
update_submodules &&
|
||||||
|
echo "Clone strategy succeeded" &&
|
||||||
|
del_files_confirm &&
|
||||||
|
exit 0
|
||||||
|
|
||||||
|
echo "Clean clone failed..."
|
||||||
|
done
|
||||||
|
|
||||||
|
die "Failed to clone repo & submodules together"
|
10
tools/ci/mirror-list.txt
Normal file
10
tools/ci/mirror-list.txt
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
components/esp32/lib @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
|
||||||
|
components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
|
||||||
|
components/aws_iot/aws-iot-device-sdk-embedded-C @GENERAL_MIRROR_SERVER@/idf/aws-iot-device-sdk-embedded-C.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git ALLOW_TO_SYNC_FROM_PUBLIC
|
||||||
|
third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git ALLOW_TO_SYNC_FROM_PUBLIC
|
80
tools/ci/mirror-submodule-update.sh
Executable file
80
tools/ci/mirror-submodule-update.sh
Executable file
|
@ -0,0 +1,80 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Redirects git submodules to the specified local mirrors and updates these recursively.
|
||||||
|
#
|
||||||
|
# To revert the changed URLs use 'git submodule deinit .'
|
||||||
|
#
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Common bash
|
||||||
|
|
||||||
|
if [[ ! -z ${DEBUG} ]]
|
||||||
|
then
|
||||||
|
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -o errexit # Exit if command failed.
|
||||||
|
set -o pipefail # Exit if pipe failed.
|
||||||
|
set -o nounset # Exit if variable not set.
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "${1:-"Unknown Error"}" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
|
||||||
|
|
||||||
|
REPO_DIR=${1:-"${PWD}"}
|
||||||
|
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
|
||||||
|
|
||||||
|
SCRIPT_DIR=$(dirname -- "${0}")
|
||||||
|
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
|
||||||
|
|
||||||
|
SCRIPT_SH=$(readlink -f -- "${0}")
|
||||||
|
|
||||||
|
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
|
||||||
|
|
||||||
|
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
|
||||||
|
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
|
||||||
|
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
|
||||||
|
|
||||||
|
pushd ${REPO_DIR} >/dev/null
|
||||||
|
|
||||||
|
# 0
|
||||||
|
[ -f ".gitmodules" ] || exit 0
|
||||||
|
|
||||||
|
# 1
|
||||||
|
git submodule init
|
||||||
|
|
||||||
|
|
||||||
|
# 2
|
||||||
|
# Replacing each submodule URL of the current repository
|
||||||
|
# according to the one found in the MIRRORLIST
|
||||||
|
|
||||||
|
# SED parses the strings like:
|
||||||
|
#
|
||||||
|
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
|
||||||
|
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
|
||||||
|
#
|
||||||
|
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
|
||||||
|
do
|
||||||
|
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||||
|
[ ${SUBMIRROR} ] || continue
|
||||||
|
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
|
||||||
|
echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
|
||||||
|
|
||||||
|
git config submodule.${SUBPATH}.url ${SUBMIRROR}
|
||||||
|
done
|
||||||
|
|
||||||
|
# 3
|
||||||
|
# Getting submodules of the current repository from the local mirrors
|
||||||
|
git submodule update
|
||||||
|
|
||||||
|
# 4
|
||||||
|
# Replacing URLs for each sub-submodule.
|
||||||
|
# The script runs recursively
|
||||||
|
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
|
||||||
|
|
||||||
|
popd >/dev/null
|
87
tools/ci/mirror-synchronize.sh
Executable file
87
tools/ci/mirror-synchronize.sh
Executable file
|
@ -0,0 +1,87 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Script for automate mirroring
|
||||||
|
# You can run this script manually
|
||||||
|
#
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
# Common bash
|
||||||
|
|
||||||
|
if [[ ! -z ${DEBUG} ]]
|
||||||
|
then
|
||||||
|
set -x # Activate the expand mode if DEBUG is anything but empty.
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -o errexit # Exit if command failed.
|
||||||
|
set -o pipefail # Exit if pipe failed.
|
||||||
|
set -o nounset # Exit if variable not set.
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "${1:-"Unknown Error"}" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
|
||||||
|
|
||||||
|
REPO_DIR=${1:-"${PWD}"}
|
||||||
|
REPO_DIR=$(readlink -f -- "${REPO_DIR}")
|
||||||
|
|
||||||
|
SCRIPT_DIR=$(dirname -- "${0}")
|
||||||
|
SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
|
||||||
|
|
||||||
|
SCRIPT_SH=$(readlink -f -- "${0}")
|
||||||
|
|
||||||
|
MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
|
||||||
|
|
||||||
|
[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
|
||||||
|
[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
|
||||||
|
[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
|
||||||
|
|
||||||
|
pushd ${REPO_DIR} >/dev/null
|
||||||
|
|
||||||
|
# 0
|
||||||
|
[ -f ".gitmodules" ] || exit 0
|
||||||
|
|
||||||
|
# 1
|
||||||
|
# Recursion
|
||||||
|
git submodule update --init
|
||||||
|
git submodule foreach "${SCRIPT_SH}" # No '--recursive'
|
||||||
|
|
||||||
|
# 2
|
||||||
|
git submodule deinit --force .
|
||||||
|
git submodule update --init
|
||||||
|
|
||||||
|
|
||||||
|
# 3
|
||||||
|
# Mirroring from original URLs to ones listed in the MIRRORLIST
|
||||||
|
|
||||||
|
# SED parses the strings like:
|
||||||
|
#
|
||||||
|
#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
|
||||||
|
#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
|
||||||
|
#
|
||||||
|
for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
|
||||||
|
do
|
||||||
|
SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
|
||||||
|
SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||||
|
[ ${SUBMIRROR} ] || continue
|
||||||
|
SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
|
||||||
|
|
||||||
|
ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
|
||||||
|
if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
|
||||||
|
then
|
||||||
|
echo "[should not to sync mirror] ${SUBMIRROR}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
|
||||||
|
|
||||||
|
pushd ${SUBPATH} >/dev/null
|
||||||
|
git fetch --prune
|
||||||
|
git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
|
||||||
|
popd >/dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
popd >/dev/null
|
Loading…
Reference in a new issue