From 008ad9af47dc8d2f2bcbbd0adc61279db00db122 Mon Sep 17 00:00:00 2001 From: Anton Maklakov Date: Thu, 30 May 2019 01:39:02 +0700 Subject: [PATCH] ci: Use relative submodule URLs over IDF. Correct CI accordindly Clean up `before_script`s Update `check_submodule_sync` Remove tools/ci/mirror* stuff --- .gitlab-ci.yml | 95 +++++++++++-------------- .gitmodules | 40 ++++++----- tools/ci/executable-list.txt | 2 - tools/ci/get-full-sources.sh | 103 ---------------------------- tools/ci/mirror-list.txt | 20 ------ tools/ci/mirror-submodule-update.sh | 83 +--------------------- tools/ci/mirror-synchronize.sh | 87 ----------------------- 7 files changed, 65 insertions(+), 365 deletions(-) delete mode 100755 tools/ci/get-full-sources.sh delete mode 100644 tools/ci/mirror-list.txt delete mode 100755 tools/ci/mirror-synchronize.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba2f05626..264023579 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,10 +19,12 @@ variables: GET_SOURCES_ATTEMPTS: "10" ARTIFACT_DOWNLOAD_ATTEMPTS: "10" - # We use get-full-sources.sh script to fetch the submodules and/or re-fetch the repo - # if it was corrupted (if submodule update fails this can happen) - GIT_STRATEGY: fetch - GIT_SUBMODULE_STRATEGY: none + # GIT_STRATEGY is not defined here. + # Use an option from "CI / CD Settings" - "General pipelines". + + # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF. + # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules. + GIT_SUBMODULE_STRATEGY: normal UNIT_TEST_BUILD_SYSTEM: make # IDF environment @@ -37,19 +39,6 @@ variables: # Docker images BOT_DOCKER_IMAGE_TAG: ":latest" -# When 'fetch' strategy is used, Gitlab removes untracked files before checking out -# new revision. However if the new revision doesn't include some of the submodules -# which were present in the old revision, such submodule directories would not be -# removed by the checkout. This extra step ensures that these stale submodules -# are removed. -.git_clean_stale_submodules: &git_clean_stale_submodules > - find . -name '.git' -not -path './.git' -printf '%P\n' - | sed 's|/.git||' - | xargs -I {} sh -c ' - grep -q "path = {}" .gitmodules - || (echo "Removing {}, has .git directory but not in .gitmodules file" - && rm -rf {});' - # before each job, we need to check if this job is filtered by bot stage/job filter .apply_bot_filter: &apply_bot_filter python $APPLY_BOT_FILTER_SCRIPT || exit 0 @@ -78,9 +67,13 @@ variables: tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1 fi +.show_submodule_urls: &show_submodule_urls | + git config --get-regexp '^submodule\..*\.url$' || true + before_script: + - echo "Running common script" + - *show_submodule_urls - source tools/ci/setup_python.sh - - *git_clean_stale_submodules # apply bot filter in before script - *apply_bot_filter # add gitlab ssh key @@ -97,43 +90,27 @@ before_script: - *setup_custom_toolchain - # fetch the submodules (& if necessary re-fetch repo) from gitlab - - time ./tools/ci/get-full-sources.sh - # used for check scripts which we want to run unconditionally -.do_nothing_before_no_filter: - before_script: &do_nothing_before_no_filter +.before_script_lesser_nofilter: &before_script_lesser_nofilter + variables: + GIT_SUBMODULE_STRATEGY: none + before_script: + - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter" - source tools/ci/setup_python.sh - - *git_clean_stale_submodules + - source tools/ci/configure_ci_environment.sh - *setup_custom_toolchain # used for everything else where we want to do no prep, except for bot filter -.do_nothing_before: - before_script: &do_nothing_before - - source tools/ci/setup_python.sh - - *git_clean_stale_submodules - # apply bot filter in before script - - *apply_bot_filter +.before_script_lesser: &before_script_lesser + variables: + GIT_SUBMODULE_STRATEGY: none + before_script: - echo "Not setting up GitLab key, not fetching submodules" - - source tools/ci/configure_ci_environment.sh - - *setup_custom_toolchain - -.add_gitlab_key_before: - before_script: &add_gitlab_key_before - source tools/ci/setup_python.sh - - *git_clean_stale_submodules # apply bot filter in before script - *apply_bot_filter - - echo "Not fetching submodules" - source tools/ci/configure_ci_environment.sh - *setup_custom_toolchain - # add gitlab ssh key - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64 - - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config after_script: - *cleanup_custom_toolchain @@ -420,7 +397,7 @@ build_docs: tags: - host_test dependencies: [] - before_script: *do_nothing_before_no_filter + <<: *before_script_lesser_nofilter verify_cmake_style: <<: *check_job_template @@ -771,7 +748,7 @@ push_to_github: - /^v\d+\.\d+(\.\d+)?($|-)/ when: on_success dependencies: [] - before_script: *do_nothing_before + <<: *before_script_lesser script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -799,7 +776,7 @@ deploy_docs: - $BOT_LABEL_BUILD_DOCS dependencies: - build_docs - before_script: *do_nothing_before + <<: *before_script_lesser script: - mkdir -p ~/.ssh - chmod 700 ~/.ssh @@ -966,7 +943,7 @@ check_examples_cmake_make: - master - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ - before_script: *do_nothing_before + <<: *before_script_lesser script: - tools/ci/check_examples_cmake_make.sh @@ -977,14 +954,13 @@ check_python_style: paths: - flake8_output.txt expire_in: 1 week - before_script: *do_nothing_before + <<: *before_script_lesser script: # run it only under Python 3 (it is very slow under Python 2) - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH check_kconfigs: <<: *check_job_template - before_script: *do_nothing_before artifacts: when: on_failure paths: @@ -995,6 +971,7 @@ check_kconfigs: - tools/*/*/Kconfig*.new - tools/*/*/*/Kconfig*.new expire_in: 1 week + <<: *before_script_lesser script: - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py - ${IDF_PATH}/tools/check_kconfigs.py @@ -1009,7 +986,7 @@ check_ut_cmake_make: - /^release\/v/ - /^v\d+\.\d+(\.\d+)?($|-)/ dependencies: [] - before_script: *do_nothing_before + <<: *before_script_lesser script: - tools/ci/check_ut_cmake_make.sh @@ -1017,12 +994,22 @@ check_submodule_sync: <<: *check_job_template tags: - github_sync + retry: 2 variables: GIT_STRATEGY: clone - retry: 2 + GIT_SUBMODULE_STRATEGY: none + PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" + before_script: [] + after_script: [] script: + - git submodule deinit --force . + # setting the default remote URL to the public one, to resolve relative location URLs + - git config remote.origin.url ${PUBLIC_IDF_URL} # check if all submodules are correctly synced to public repostory - - git submodule update --init --recursive + - git submodule init + - *show_submodule_urls + - git submodule update --recursive + - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" check_artifacts_expire_time: <<: *check_job_template @@ -1219,12 +1206,12 @@ test_weekend_network: - $LOG_PATH expire_in: 1 week variables: + GIT_SUBMODULE_STRATEGY: none LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml" CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml" - before_script: *add_gitlab_key_before script: # first test if config file exists, if not exist, exit 0 - test -e $CONFIG_FILE || exit 0 diff --git a/.gitmodules b/.gitmodules index 6a656fa4c..b14915bc2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,71 +1,75 @@ +# +# All the relative URL paths are intended to be GitHub ones +# + [submodule "components/esptool_py/esptool"] path = components/esptool_py/esptool - url = https://github.com/espressif/esptool.git + url = ../esptool.git [submodule "components/bt/lib"] path = components/bt/lib - url = https://github.com/espressif/esp32-bt-lib.git + url = ../esp32-bt-lib.git [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"] path = components/bootloader/subproject/components/micro-ecc/micro-ecc - url = https://github.com/kmackay/micro-ecc.git + url = ../../kmackay/micro-ecc.git [submodule "components/coap/libcoap"] path = components/coap/libcoap - url = https://github.com/obgm/libcoap.git + url = ../../obgm/libcoap.git [submodule "components/nghttp/nghttp2"] path = components/nghttp/nghttp2 - url = https://github.com/nghttp2/nghttp2.git + url = ../../nghttp2/nghttp2.git [submodule "components/libsodium/libsodium"] path = components/libsodium/libsodium - url = https://github.com/jedisct1/libsodium.git + url = ../../jedisct1/libsodium.git [submodule "components/spiffs/spiffs"] path = components/spiffs/spiffs - url = https://github.com/pellepl/spiffs.git + url = ../../pellepl/spiffs.git [submodule "components/json/cJSON"] path = components/json/cJSON - url = https://github.com/DaveGamble/cJSON.git + url = ../../DaveGamble/cJSON.git [submodule "components/mbedtls/mbedtls"] path = components/mbedtls/mbedtls - url = https://github.com/espressif/mbedtls.git + url = ../mbedtls.git [submodule "components/asio/asio"] path = components/asio/asio - url = https://github.com/espressif/asio.git + url = ../asio.git [submodule "components/expat/expat"] path = components/expat/expat - url = https://github.com/libexpat/libexpat.git + url = ../../libexpat/libexpat.git [submodule "components/lwip/lwip"] path = components/lwip/lwip - url = https://github.com/espressif/esp-lwip.git + url = ../esp-lwip.git [submodule "components/mqtt/esp-mqtt"] path = components/mqtt/esp-mqtt - url = https://github.com/espressif/esp-mqtt.git + url = ../esp-mqtt.git [submodule "components/protobuf-c/protobuf-c"] path = components/protobuf-c/protobuf-c - url = https://github.com/protobuf-c/protobuf-c + url = ../../protobuf-c/protobuf-c [submodule "components/unity/unity"] path = components/unity/unity - url = https://github.com/ThrowTheSwitch/Unity + url = ../../ThrowTheSwitch/Unity [submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"] path = examples/build_system/cmake/import_lib/main/lib/tinyxml2 - url = https://github.com/leethomason/tinyxml2 + url = ../../leethomason/tinyxml2 [submodule "components/esp_wifi/lib_esp32"] path = components/esp_wifi/lib_esp32 - url = https://github.com/espressif/esp32-wifi-lib.git + url = ../esp32-wifi-lib.git [submodule "components/nimble/nimble"] path = components/nimble/nimble - url = https://github.com/espressif/esp-nimble.git + url = ../esp-nimble.git diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index dd001885a..ea430e4f2 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -38,9 +38,7 @@ tools/ci/check_idf_version.sh tools/ci/check_ut_cmake_make.sh tools/ci/checkout_project_ref.py tools/ci/envsubst.py -tools/ci/get-full-sources.sh tools/ci/mirror-submodule-update.sh -tools/ci/mirror-synchronize.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh tools/ci/test_build_system.sh diff --git a/tools/ci/get-full-sources.sh b/tools/ci/get-full-sources.sh deleted file mode 100755 index a332b18e2..000000000 --- a/tools/ci/get-full-sources.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# -# Short script that is run as part of the CI environment -# in .gitlab-ci.yml -# -# Sets up submodules in the ESP-IDF source tree -# - Ideally, this just means doing a "git submodule update" -# - But if something goes wrong we re-clone the repo from scratch -# -# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone -# - -# ----------------------------------------------------------------------------- -# Common bash -if [[ ! -z ${DEBUG_SHELL} ]]; then - set -x # Activate the expand mode if DEBUG is anything but empty. -fi - -set -o errexit # Exit if command failed. -set -o pipefail # Exit if pipe failed. -set -o nounset # Exit if variable not set. - -die() { - echo "${1:-"Unknown Error"}" 1>&2 - exit ${2:-1} -} -# ----------------------------------------------------------------------------- - -[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner." -[ -z ${GITLAB_SSH_SERVER} ] && die "GITLAB_SSH_SERVER should be defined to run mirror-submodule-update.sh" -[ -z ${CI_REPOSITORY_URL} ] && die "CI_REPOSITORY_URL should be defined to run mirror-submodule-update.sh" -[ -z ${CI_COMMIT_SHA} ] && die "CI_COMMIT_SHA should be defined to run mirror-submodule-update.sh" -DONT_USE_MIRROR=${DONT_USE_MIRROR:-"0"} - -ERR_CANNOT_UPDATE=13 - -SCRIPT_DIR=$(dirname -- "${0}") -update_submodules() { - if [ "${DONT_USE_MIRROR}" = "1" ]; then - git submodule update --init --recursive - else - ${SCRIPT_DIR}/mirror-submodule-update.sh || return $? - fi -} - -del_files() { - DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX) - # if non-empty - [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" ) - trap 'del_files_rollback' ERR -} -del_files_confirm() { - [ -d "${DELETED_FILES}" ] && rm -rf "${DELETED_FILES}" - trap ERR -} -del_files_rollback() { - [ "$(ls -A .)" ] && [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; rm -rf * ) - [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; mv "${DELETED_FILES}/"* . ) - [ -d "${DELETED_FILES}" ] && rmdir "${DELETED_FILES}" - trap ERR -} - -RETRIES=10 -# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here... - -# For the first time, we try the fastest way. -for try in `seq $RETRIES`; do - echo "Trying to add submodules to existing repo..." - update_submodules && - echo "Fetch strategy submodules succeeded" && - exit 0 - - git submodule foreach --recursive "git reset --hard HEAD && git submodule deinit --force -- . || true" - git reset --hard HEAD && git submodule deinit --force -- . || true -done - -# Then we use the clean way. -for try in `seq $RETRIES`; do - cd ${CI_PROJECT_DIR} # we are probably already here but pays to be certain - echo "Trying a clean clone of IDF..." - del_files - git clone ${CI_REPOSITORY_URL} . && - git checkout ${CI_COMMIT_SHA} && - update_submodules && - echo "Clone strategy succeeded" && - del_files_confirm && - exit 0 - ERR_RES=$? - del_files_rollback - echo "Clean clone failed..." - if [ $ERR_RES -eq $ERR_CANNOT_UPDATE ]; then - echo "###" - echo "### If you have updated one of the submodules," - echo "### you have to synchronize the local mirrors manually" - echo "###" - echo "### https://gitlab.espressif.cn:6688/idf/esp-idf/wikis/ci-use-guide#submodule-mirroring-for-private-branches" - echo "###" - - die "Failed to clone repo & submodules together" $ERR_RES - fi -done - -die "Failed to clone repo & submodules together" diff --git a/tools/ci/mirror-list.txt b/tools/ci/mirror-list.txt deleted file mode 100644 index cd18d7d10..000000000 --- a/tools/ci/mirror-list.txt +++ /dev/null @@ -1,20 +0,0 @@ -components/esp_wifi/lib_esp32 @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git -components/bt/lib @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git -components/esptool_py/esptool @GENERAL_MIRROR_SERVER@/idf/esptool.git -components/coap/libcoap @GENERAL_MIRROR_SERVER@/idf/libcoap.git ALLOW_TO_SYNC_FROM_PUBLIC -ext/tinydtls @GENERAL_MIRROR_SERVER@/idf/org.eclipse.tinydtls.git ALLOW_TO_SYNC_FROM_PUBLIC -components/json/cJSON @GENERAL_MIRROR_SERVER@/idf/cJSON.git ALLOW_TO_SYNC_FROM_PUBLIC -components/libsodium/libsodium @GENERAL_MIRROR_SERVER@/idf/libsodium.git ALLOW_TO_SYNC_FROM_PUBLIC -components/mbedtls/mbedtls @GENERAL_MIRROR_SERVER@/idf/mbedtls.git ALLOW_TO_SYNC_FROM_PUBLIC -components/expat/expat @GENERAL_MIRROR_SERVER@/idf/libexpat.git ALLOW_TO_SYNC_FROM_PUBLIC -components/bootloader/subproject/components/micro-ecc/micro-ecc @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git ALLOW_TO_SYNC_FROM_PUBLIC -components/nghttp/nghttp2 @GENERAL_MIRROR_SERVER@/idf/nghttp2.git ALLOW_TO_SYNC_FROM_PUBLIC -components/spiffs/spiffs @GENERAL_MIRROR_SERVER@/idf/spiffs.git ALLOW_TO_SYNC_FROM_PUBLIC -components/asio/asio @GENERAL_MIRROR_SERVER@/idf/asio.git -components/lwip/lwip @GENERAL_MIRROR_SERVER@/idf/esp-lwip.git -third-party/mruby @GENERAL_MIRROR_SERVER@/idf/mruby.git ALLOW_TO_SYNC_FROM_PUBLIC -third-party/neverbleed @GENERAL_MIRROR_SERVER@/idf/neverbleed.git ALLOW_TO_SYNC_FROM_PUBLIC -components/mqtt/esp-mqtt @GENERAL_MIRROR_SERVER@/idf/esp-mqtt.git ALLOW_TO_SYNC_FROM_PUBLIC -components/protobuf-c/protobuf-c @GENERAL_MIRROR_SERVER@/idf/protobuf-c.git ALLOW_TO_SYNC_FROM_PUBLIC -components/unity/unity @GENERAL_MIRROR_SERVER@/idf/Unity.git ALLOW_TO_SYNC_FROM_PUBLIC -examples/build_system/cmake/import_lib/main/lib/tinyxml2 @GENERAL_MIRROR_SERVER@/idf/tinyxml2.git ALLOW_TO_SYNC_FROM_PUBLIC diff --git a/tools/ci/mirror-submodule-update.sh b/tools/ci/mirror-submodule-update.sh index 3a15dbe99..f67949288 100755 --- a/tools/ci/mirror-submodule-update.sh +++ b/tools/ci/mirror-submodule-update.sh @@ -1,84 +1,5 @@ #!/bin/bash -# -# Redirects git submodules to the specified local mirrors and updates these recursively. -# -# To revert the changed URLs use 'git submodule deinit .' -# -# ----------------------------------------------------------------------------- -# Common bash +# Now git submodules are redirected to mirror automatically according to relative URLs in .gitmodules -if [[ ! -z ${DEBUG_SHELL} ]] -then - set -x # Activate the expand mode if DEBUG is anything but empty. -fi - -set -o errexit # Exit if command failed. -set -o pipefail # Exit if pipe failed. -set -o nounset # Exit if variable not set. - -die() { - echo "${1:-"Unknown Error"}" 1>&2 - exit 1 -} - -# ----------------------------------------------------------------------------- - -[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable" - -ERR_CANNOT_UPDATE=13 - -REPO_DIR=${1:-"${PWD}"} -REPO_DIR=$(readlink -f -- "${REPO_DIR}") - -SCRIPT_DIR=$(dirname -- "${0}") -SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}") - -SCRIPT_SH=$(readlink -f -- "${0}") - -MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt - -[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!" -[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!" -[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!" - -pushd ${REPO_DIR} >/dev/null - -# 0 -[ -f ".gitmodules" ] || exit 0 - -# 1 -git submodule init - - -# 2 -# Replacing each submodule URL of the current repository -# according to the one found in the MIRRORLIST -# -# Selecting paths among lines: -# ... -#submodule.components/esp32/lib.path -#submodule.components/esp32/lib.url -#submodule.components/esptool_py/esptool.path -#submodule.components/esptool_py/esptool.url -#... -for SUBPATH in $(git config -f .gitmodules --list --name-only | grep "\.path" | sed 's/^submodule\.\([^ ]*\)\.path$/\1/') -do - SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) - [ ${SUBMIRROR} ] || continue - SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}} - echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR" - - git config submodule.${SUBPATH}.url ${SUBMIRROR} -done - -# 3 -# Getting submodules of the current repository from the local mirrors -git submodule update || exit $ERR_CANNOT_UPDATE - -# 4 -# Replacing URLs for each sub-submodule. -# The script runs recursively -git submodule foreach "${SCRIPT_SH}" # No '--recursive' - -popd >/dev/null +echo "WARNING: Deprecated. This script is no longer required." diff --git a/tools/ci/mirror-synchronize.sh b/tools/ci/mirror-synchronize.sh deleted file mode 100755 index 5047744e5..000000000 --- a/tools/ci/mirror-synchronize.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -# Script for automate mirroring -# You can run this script manually -# - -# ----------------------------------------------------------------------------- -# Common bash - -if [[ ! -z ${DEBUG_SHELL} ]] -then - set -x # Activate the expand mode if DEBUG is anything but empty. -fi - -set -o errexit # Exit if command failed. -set -o pipefail # Exit if pipe failed. -set -o nounset # Exit if variable not set. - -die() { - echo "${1:-"Unknown Error"}" 1>&2 - exit 1 -} - -# ----------------------------------------------------------------------------- - -[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable" - -REPO_DIR=${1:-"${PWD}"} -REPO_DIR=$(readlink -f -- "${REPO_DIR}") - -SCRIPT_DIR=$(dirname -- "${0}") -SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}") - -SCRIPT_SH=$(readlink -f -- "${0}") - -MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt - -[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!" -[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!" -[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!" - -pushd ${REPO_DIR} >/dev/null - -# 0 -[ -f ".gitmodules" ] || exit 0 - -# 1 -# Recursion -git submodule update --init -git submodule foreach "${SCRIPT_SH}" # No '--recursive' - -# 2 -git submodule deinit --force . -git submodule update --init - - -# 3 -# Mirroring from original URLs to ones listed in the MIRRORLIST - -# SED parses the strings like: -# -#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe -#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67) -# -for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/') -do - SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url) - SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) - [ ${SUBMIRROR} ] || continue - SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}} - - ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST})) - if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ] - then - echo "[should not to sync mirror] ${SUBMIRROR}" - continue - fi - - echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}" - - pushd ${SUBPATH} >/dev/null - git fetch --prune - git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/* - popd >/dev/null -done - -popd >/dev/null