ci: Use relative submodule URLs over IDF. Correct CI accordindly

Clean up `before_script`s

    Update `check_submodule_sync`

    Remove tools/ci/mirror* stuff

.gitlab-ci.yml

@@ -19,10 +19,12 @@ variables:
   GET_SOURCES_ATTEMPTS: "10"
   ARTIFACT_DOWNLOAD_ATTEMPTS: "10"
 
-  # We use get-full-sources.sh script to fetch the submodules and/or re-fetch the repo
+  # GIT_STRATEGY is not defined here.
-  # if it was corrupted (if submodule update fails this can happen)
+  # Use an option from  "CI / CD Settings" - "General pipelines".
-  GIT_STRATEGY: fetch
+
-  GIT_SUBMODULE_STRATEGY: none
+  # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF.
+  # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules.
+  GIT_SUBMODULE_STRATEGY: normal
 
   UNIT_TEST_BUILD_SYSTEM: make
 # IDF environment
@@ -37,19 +39,6 @@ variables:
 # Docker images
   BOT_DOCKER_IMAGE_TAG: ":latest"
 
-# When 'fetch' strategy is used, Gitlab removes untracked files before checking out
-# new revision. However if the new revision doesn't include some of the submodules
-# which were present in the old revision, such submodule directories would not be
-# removed by the checkout. This extra step ensures that these stale submodules
-# are removed.
-.git_clean_stale_submodules:  &git_clean_stale_submodules >
-  find . -name '.git' -not -path './.git' -printf '%P\n'
-  | sed 's|/.git||'
-  | xargs -I {} sh -c '
-  grep -q "path = {}" .gitmodules
-  ||  (echo "Removing {}, has .git directory but not in .gitmodules file"
-  && rm -rf {});'
-
 # before each job, we need to check if this job is filtered by bot stage/job filter
 .apply_bot_filter: &apply_bot_filter
   python $APPLY_BOT_FILTER_SCRIPT || exit 0
@@ -78,9 +67,13 @@ variables:
   tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
 
+.show_submodule_urls: &show_submodule_urls |
+  git config --get-regexp '^submodule\..*\.url$' || true
+
 before_script:
+  - echo "Running common script"
+  - *show_submodule_urls
   - source tools/ci/setup_python.sh
-  - *git_clean_stale_submodules
   # apply bot filter in before script
   - *apply_bot_filter
   # add gitlab ssh key
@@ -97,43 +90,27 @@ before_script:
 
   - *setup_custom_toolchain
 
-  # fetch the submodules (& if necessary re-fetch repo) from gitlab
-  - time ./tools/ci/get-full-sources.sh
-
 # used for check scripts which we want to run unconditionally
-.do_nothing_before_no_filter:
+.before_script_lesser_nofilter: &before_script_lesser_nofilter
-  before_script: &do_nothing_before_no_filter
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  before_script:
+    - echo "Not setting up GitLab key, not fetching submodules, not applying bot filter"
     - source tools/ci/setup_python.sh
-    - *git_clean_stale_submodules
+    - source tools/ci/configure_ci_environment.sh
     - *setup_custom_toolchain
 
 # used for everything else where we want to do no prep, except for bot filter
-.do_nothing_before:
+.before_script_lesser: &before_script_lesser
-  before_script: &do_nothing_before
+  variables:
-    - source tools/ci/setup_python.sh
+    GIT_SUBMODULE_STRATEGY: none
-    - *git_clean_stale_submodules
+  before_script:
-    # apply bot filter in before script
-    - *apply_bot_filter
     - echo "Not setting up GitLab key, not fetching submodules"
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_custom_toolchain
-
-.add_gitlab_key_before:
-  before_script: &add_gitlab_key_before
     - source tools/ci/setup_python.sh
-    - *git_clean_stale_submodules
     # apply bot filter in before script
     - *apply_bot_filter
-    - echo "Not fetching submodules"
     - source tools/ci/configure_ci_environment.sh
     - *setup_custom_toolchain
-    # add gitlab ssh key
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo -n $GITLAB_KEY > ~/.ssh/id_rsa_base64
-    - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - echo -e "Host gitlab.espressif.cn\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
 
 after_script:
   - *cleanup_custom_toolchain
@@ -420,7 +397,7 @@ build_docs:
   tags:
     - host_test
   dependencies: []
-  before_script: *do_nothing_before_no_filter
+  <<: *before_script_lesser_nofilter
 
 verify_cmake_style:
   <<: *check_job_template
@@ -771,7 +748,7 @@ push_to_github:
     - /^v\d+\.\d+(\.\d+)?($|-)/
   when: on_success
   dependencies: []
-  before_script: *do_nothing_before
+  <<: *before_script_lesser
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
@@ -799,7 +776,7 @@ deploy_docs:
       - $BOT_LABEL_BUILD_DOCS
   dependencies:
     - build_docs
-  before_script: *do_nothing_before
+  <<: *before_script_lesser
   script:
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
@@ -966,7 +943,7 @@ check_examples_cmake_make:
     - master
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
-  before_script: *do_nothing_before
+  <<: *before_script_lesser
   script:
     - tools/ci/check_examples_cmake_make.sh
 
@@ -977,14 +954,13 @@ check_python_style:
     paths:
       - flake8_output.txt
     expire_in: 1 week
-  before_script: *do_nothing_before
+  <<: *before_script_lesser
   script:
     # run it only under Python 3 (it is very slow under Python 2)
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 check_kconfigs:
   <<: *check_job_template
-  before_script: *do_nothing_before
   artifacts:
     when: on_failure
     paths:
@@ -995,6 +971,7 @@ check_kconfigs:
       - tools/*/*/Kconfig*.new
       - tools/*/*/*/Kconfig*.new
     expire_in: 1 week
+  <<: *before_script_lesser
   script:
     - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/test_check_kconfigs.py
     - ${IDF_PATH}/tools/check_kconfigs.py
@@ -1009,7 +986,7 @@ check_ut_cmake_make:
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
   dependencies: []
-  before_script: *do_nothing_before
+  <<: *before_script_lesser
   script:
     - tools/ci/check_ut_cmake_make.sh
 
@@ -1017,12 +994,22 @@ check_submodule_sync:
   <<: *check_job_template
   tags:
     - github_sync
+  retry: 2
   variables:
     GIT_STRATEGY: clone
-  retry: 2
+    GIT_SUBMODULE_STRATEGY: none
+    PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
+  before_script: []
+  after_script: []
   script:
+    - git submodule deinit --force .
+    # setting the default remote URL to the public one, to resolve relative location URLs
+    - git config remote.origin.url ${PUBLIC_IDF_URL}
     # check if all submodules are correctly synced to public repostory
-    - git submodule update --init --recursive
+    - git submodule init
+    - *show_submodule_urls
+    - git submodule update --recursive
+    - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
 
 check_artifacts_expire_time:
   <<: *check_job_template
@@ -1219,12 +1206,12 @@ test_weekend_network:
       - $LOG_PATH
     expire_in: 1 week
   variables:
+    GIT_SUBMODULE_STRATEGY: none
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
     MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/components/idf_test/ModuleDefinition.yml"
     CONFIG_FILE: "$CI_PROJECT_DIR/components/idf_test/integration_test/CIConfigs/$CI_JOB_NAME.yml"
-  before_script: *add_gitlab_key_before
   script:
     # first test if config file exists, if not exist, exit 0
     - test -e $CONFIG_FILE || exit 0

.gitmodules

@@ -1,71 +1,75 @@
+#
+# All the relative URL paths are intended to be GitHub ones
+#
+
 [submodule "components/esptool_py/esptool"]
 	path = components/esptool_py/esptool
-	url = https://github.com/espressif/esptool.git
+	url = ../esptool.git
 
 [submodule "components/bt/lib"]
 	path = components/bt/lib
-	url = https://github.com/espressif/esp32-bt-lib.git
+	url = ../esp32-bt-lib.git
 
 [submodule "components/bootloader/subproject/components/micro-ecc/micro-ecc"]
 	path = components/bootloader/subproject/components/micro-ecc/micro-ecc
-	url = https://github.com/kmackay/micro-ecc.git
+	url = ../../kmackay/micro-ecc.git
 
 [submodule "components/coap/libcoap"]
 	path = components/coap/libcoap
-	url = https://github.com/obgm/libcoap.git
+	url = ../../obgm/libcoap.git
 
 [submodule "components/nghttp/nghttp2"]
 	path = components/nghttp/nghttp2
-	url = https://github.com/nghttp2/nghttp2.git
+	url = ../../nghttp2/nghttp2.git
 
 [submodule "components/libsodium/libsodium"]
 	path = components/libsodium/libsodium
-	url = https://github.com/jedisct1/libsodium.git
+	url = ../../jedisct1/libsodium.git
 
 [submodule "components/spiffs/spiffs"]
 	path = components/spiffs/spiffs
-	url = https://github.com/pellepl/spiffs.git
+	url = ../../pellepl/spiffs.git
 
 [submodule "components/json/cJSON"]
 	path = components/json/cJSON
-	url = https://github.com/DaveGamble/cJSON.git
+	url = ../../DaveGamble/cJSON.git
 
 [submodule "components/mbedtls/mbedtls"]
 	path = components/mbedtls/mbedtls
-	url = https://github.com/espressif/mbedtls.git
+	url = ../mbedtls.git
 
 [submodule "components/asio/asio"]
 	path = components/asio/asio
-	url = https://github.com/espressif/asio.git
+	url = ../asio.git
 
 [submodule "components/expat/expat"]
 	path = components/expat/expat
-	url = https://github.com/libexpat/libexpat.git
+	url = ../../libexpat/libexpat.git
 
 [submodule "components/lwip/lwip"]
 	path = components/lwip/lwip
-	url = https://github.com/espressif/esp-lwip.git
+	url = ../esp-lwip.git
 
 [submodule "components/mqtt/esp-mqtt"]
 	path = components/mqtt/esp-mqtt
-	url = https://github.com/espressif/esp-mqtt.git
+	url = ../esp-mqtt.git
 
 [submodule "components/protobuf-c/protobuf-c"]
 	path = components/protobuf-c/protobuf-c
-	url = https://github.com/protobuf-c/protobuf-c
+	url = ../../protobuf-c/protobuf-c
 
 [submodule "components/unity/unity"]
 	path = components/unity/unity
-	url = https://github.com/ThrowTheSwitch/Unity
+	url = ../../ThrowTheSwitch/Unity
 
 [submodule "examples/build_system/cmake/import_lib/main/lib/tinyxml2"]
 	path = examples/build_system/cmake/import_lib/main/lib/tinyxml2
-	url = https://github.com/leethomason/tinyxml2
+	url = ../../leethomason/tinyxml2
 
 [submodule "components/esp_wifi/lib_esp32"]
 	path = components/esp_wifi/lib_esp32
-	url = https://github.com/espressif/esp32-wifi-lib.git
+	url = ../esp32-wifi-lib.git
 
 [submodule "components/nimble/nimble"]
 	path = components/nimble/nimble
-	url = https://github.com/espressif/esp-nimble.git
+	url = ../esp-nimble.git

tools/ci/executable-list.txt

@@ -38,9 +38,7 @@ tools/ci/check_idf_version.sh
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
 tools/ci/envsubst.py
-tools/ci/get-full-sources.sh
 tools/ci/mirror-submodule-update.sh
-tools/ci/mirror-synchronize.sh
 tools/ci/multirun_with_pyenv.sh
 tools/ci/push_to_github.sh
 tools/ci/test_build_system.sh

tools/ci/get-full-sources.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-#
-# Short script that is run as part of the CI environment
-# in .gitlab-ci.yml
-#
-# Sets up submodules in the ESP-IDF source tree
-# - Ideally, this just means doing a "git submodule update"
-# - But if something goes wrong we re-clone the repo from scratch
-#
-# This is a "best of both worlds" for GIT_STRATEGY: fetch & GIT_STRATEGY: clone
-#
-
-# -----------------------------------------------------------------------------
-# Common bash
-if [[ ! -z ${DEBUG_SHELL} ]]; then
-  set -x # Activate the expand mode if DEBUG is anything but empty.
-fi
-
-set -o errexit # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-set -o nounset # Exit if variable not set.
-
-die() {
-    echo "${1:-"Unknown Error"}" 1>&2
-    exit ${2:-1}
-}
-# -----------------------------------------------------------------------------
-
-[ -z ${CI_PROJECT_DIR} ] && die "This internal script should only be run by a Gitlab CI runner."
-[ -z ${GITLAB_SSH_SERVER} ] && die "GITLAB_SSH_SERVER should be defined to run mirror-submodule-update.sh"
-[ -z ${CI_REPOSITORY_URL} ] && die "CI_REPOSITORY_URL should be defined to run mirror-submodule-update.sh"
-[ -z ${CI_COMMIT_SHA} ] && die "CI_COMMIT_SHA should be defined to run mirror-submodule-update.sh"
-DONT_USE_MIRROR=${DONT_USE_MIRROR:-"0"}
-
-ERR_CANNOT_UPDATE=13
-
-SCRIPT_DIR=$(dirname -- "${0}")
-update_submodules() {
-    if [ "${DONT_USE_MIRROR}" = "1" ]; then
-        git submodule update --init --recursive
-    else
-        ${SCRIPT_DIR}/mirror-submodule-update.sh || return $?
-    fi
-}
-
-del_files() {
-    DELETED_FILES=$(mktemp --tmpdir -d tmp_XXXX)
-    # if non-empty
-    [ "$(ls -A .)" ] && ( shopt -s dotglob; mv * "${DELETED_FILES}/" )
-    trap 'del_files_rollback' ERR
-}
-del_files_confirm() {
-    [ -d "${DELETED_FILES}" ] && rm -rf "${DELETED_FILES}"
-    trap ERR
-}
-del_files_rollback() {
-    [ "$(ls -A .)" ] && [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; rm -rf * )
-    [ "$(ls -A ${DELETED_FILES}/)" ] && ( shopt -s dotglob; mv "${DELETED_FILES}/"* . )
-    [ -d "${DELETED_FILES}" ] && rmdir "${DELETED_FILES}"
-    trap ERR
-}
-
-RETRIES=10
-# we're in gitlab-ci's build phase, so GET_SOURCES_ATTEMPTS doesn't apply here...
-
-# For the first time, we try the fastest way.
-for try in `seq $RETRIES`; do
-    echo "Trying to add submodules to existing repo..."
-    update_submodules &&
-        echo "Fetch strategy submodules succeeded" &&
-        exit 0
-
-    git submodule foreach --recursive "git reset --hard HEAD && git submodule deinit --force -- . || true"
-    git reset --hard HEAD && git submodule deinit --force -- . || true
-done
-
-# Then we use the clean way.
-for try in `seq $RETRIES`; do
-    cd ${CI_PROJECT_DIR}  # we are probably already here but pays to be certain
-    echo "Trying a clean clone of IDF..."
-    del_files
-    git clone ${CI_REPOSITORY_URL} . &&
-        git checkout ${CI_COMMIT_SHA} &&
-        update_submodules &&
-        echo "Clone strategy succeeded" &&
-        del_files_confirm &&
-        exit 0
-    ERR_RES=$?
-    del_files_rollback
-    echo "Clean clone failed..."
-    if [ $ERR_RES -eq $ERR_CANNOT_UPDATE ]; then
-        echo "###"
-        echo "### If you have updated one of the submodules,"
-        echo "### you have to synchronize the local mirrors manually"
-        echo "###"
-        echo "###     https://gitlab.espressif.cn:6688/idf/esp-idf/wikis/ci-use-guide#submodule-mirroring-for-private-branches"
-        echo "###"
-
-        die "Failed to clone repo & submodules together" $ERR_RES
-    fi
-done
-
-die "Failed to clone repo & submodules together"

tools/ci/mirror-list.txt

@@ -1,20 +0,0 @@
-components/esp_wifi/lib_esp32                               @GENERAL_MIRROR_SERVER@/idf/esp32-wifi-lib.git
-components/bt/lib                                           @GENERAL_MIRROR_SERVER@/idf/esp32-bt-lib.git
-components/esptool_py/esptool                               @GENERAL_MIRROR_SERVER@/idf/esptool.git
-components/coap/libcoap                                     @GENERAL_MIRROR_SERVER@/idf/libcoap.git                         ALLOW_TO_SYNC_FROM_PUBLIC
-ext/tinydtls                                                @GENERAL_MIRROR_SERVER@/idf/org.eclipse.tinydtls.git            ALLOW_TO_SYNC_FROM_PUBLIC
-components/json/cJSON                                       @GENERAL_MIRROR_SERVER@/idf/cJSON.git                           ALLOW_TO_SYNC_FROM_PUBLIC
-components/libsodium/libsodium                              @GENERAL_MIRROR_SERVER@/idf/libsodium.git                       ALLOW_TO_SYNC_FROM_PUBLIC
-components/mbedtls/mbedtls                                  @GENERAL_MIRROR_SERVER@/idf/mbedtls.git                         ALLOW_TO_SYNC_FROM_PUBLIC
-components/expat/expat                                      @GENERAL_MIRROR_SERVER@/idf/libexpat.git                        ALLOW_TO_SYNC_FROM_PUBLIC
-components/bootloader/subproject/components/micro-ecc/micro-ecc                              @GENERAL_MIRROR_SERVER@/idf/micro-ecc.git                       ALLOW_TO_SYNC_FROM_PUBLIC
-components/nghttp/nghttp2                                   @GENERAL_MIRROR_SERVER@/idf/nghttp2.git                         ALLOW_TO_SYNC_FROM_PUBLIC
-components/spiffs/spiffs                                    @GENERAL_MIRROR_SERVER@/idf/spiffs.git                          ALLOW_TO_SYNC_FROM_PUBLIC
-components/asio/asio                                        @GENERAL_MIRROR_SERVER@/idf/asio.git
-components/lwip/lwip                                        @GENERAL_MIRROR_SERVER@/idf/esp-lwip.git
-third-party/mruby                                           @GENERAL_MIRROR_SERVER@/idf/mruby.git                           ALLOW_TO_SYNC_FROM_PUBLIC
-third-party/neverbleed                                      @GENERAL_MIRROR_SERVER@/idf/neverbleed.git                      ALLOW_TO_SYNC_FROM_PUBLIC
-components/mqtt/esp-mqtt                                    @GENERAL_MIRROR_SERVER@/idf/esp-mqtt.git                        ALLOW_TO_SYNC_FROM_PUBLIC
-components/protobuf-c/protobuf-c                            @GENERAL_MIRROR_SERVER@/idf/protobuf-c.git                      ALLOW_TO_SYNC_FROM_PUBLIC
-components/unity/unity                                      @GENERAL_MIRROR_SERVER@/idf/Unity.git                           ALLOW_TO_SYNC_FROM_PUBLIC
-examples/build_system/cmake/import_lib/main/lib/tinyxml2    @GENERAL_MIRROR_SERVER@/idf/tinyxml2.git                        ALLOW_TO_SYNC_FROM_PUBLIC

tools/ci/mirror-submodule-update.sh

@@ -1,84 +1,5 @@
 #!/bin/bash
-#
-# Redirects git submodules to the specified local mirrors and updates these recursively.
-#
-# To revert the changed URLs use 'git submodule deinit .'
-#
 
-# -----------------------------------------------------------------------------
+# Now git submodules are redirected to mirror automatically according to relative URLs in .gitmodules
-# Common bash
 
-if [[ ! -z ${DEBUG_SHELL} ]]
+echo "WARNING: Deprecated. This script is no longer required."
-then
-  set -x # Activate the expand mode if DEBUG is anything but empty.
-fi
-
-set -o errexit # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-set -o nounset # Exit if variable not set.
-
-die() {
-    echo "${1:-"Unknown Error"}" 1>&2
-    exit 1
-}
-
-# -----------------------------------------------------------------------------
-
-[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
-
-ERR_CANNOT_UPDATE=13
-
-REPO_DIR=${1:-"${PWD}"}
-REPO_DIR=$(readlink -f -- "${REPO_DIR}")
-
-SCRIPT_DIR=$(dirname -- "${0}")
-SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
-
-SCRIPT_SH=$(readlink -f -- "${0}")
-
-MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
-
-[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
-[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
-[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
-
-pushd ${REPO_DIR} >/dev/null
-
-# 0
-[ -f ".gitmodules" ] || exit 0
-
-# 1
-git submodule init
-
-
-# 2
-# Replacing each submodule URL of the current repository
-# according to the one found in the MIRRORLIST
-#
-# Selecting paths among lines:
-# ...
-#submodule.components/esp32/lib.path
-#submodule.components/esp32/lib.url
-#submodule.components/esptool_py/esptool.path
-#submodule.components/esptool_py/esptool.url
-#...
-for SUBPATH in $(git config -f .gitmodules --list --name-only | grep "\.path" | sed 's/^submodule\.\([^ ]*\)\.path$/\1/')
-do
-    SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
-    [ ${SUBMIRROR} ] || continue
-    SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
-    echo -e "[switch mirror] $SUBPATH \tto\t $SUBMIRROR"
-
-    git config submodule.${SUBPATH}.url ${SUBMIRROR}
-done
-
-# 3
-# Getting submodules of the current repository from the local mirrors
-git submodule update || exit $ERR_CANNOT_UPDATE
-
-# 4
-# Replacing URLs for each sub-submodule.
-# The script runs recursively
-git submodule foreach "${SCRIPT_SH}" # No '--recursive'
-
-popd >/dev/null

tools/ci/mirror-synchronize.sh

@@ -1,87 +0,0 @@
-#!/bin/bash
-#
-# Script for automate mirroring
-# You can run this script manually
-#
-
-# -----------------------------------------------------------------------------
-# Common bash
-
-if [[ ! -z ${DEBUG_SHELL} ]]
-then
-  set -x # Activate the expand mode if DEBUG is anything but empty.
-fi
-
-set -o errexit # Exit if command failed.
-set -o pipefail # Exit if pipe failed.
-set -o nounset # Exit if variable not set.
-
-die() {
-    echo "${1:-"Unknown Error"}" 1>&2
-    exit 1
-}
-
-# -----------------------------------------------------------------------------
-
-[ -z ${GITLAB_SSH_SERVER:-} ] && die "Have to set up GITLAB_SSH_SERVER environment variable"
-
-REPO_DIR=${1:-"${PWD}"}
-REPO_DIR=$(readlink -f -- "${REPO_DIR}")
-
-SCRIPT_DIR=$(dirname -- "${0}")
-SCRIPT_DIR=$(readlink -f -- "${SCRIPT_DIR}")
-
-SCRIPT_SH=$(readlink -f -- "${0}")
-
-MIRRORLIST=${SCRIPT_DIR}/mirror-list.txt
-
-[ -d "${REPO_DIR}" ] || die "${REPO_DIR} is not directory!"
-[ -f "${SCRIPT_SH}" ] || die "${SCRIPT_SH} does not exist!"
-[ -f "${MIRRORLIST}" ] || die "${MIRRORLIST} does not exist!"
-
-pushd ${REPO_DIR} >/dev/null
-
-# 0
-[ -f ".gitmodules" ] || exit 0
-
-# 1
-# Recursion
-git submodule update --init
-git submodule foreach "${SCRIPT_SH}" # No '--recursive'
-
-# 2
-git submodule deinit --force .
-git submodule update --init
-
-
-# 3
-# Mirroring from original URLs to ones listed in the MIRRORLIST
-
-#   SED parses the strings like:
-#
-#-b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe
-#b991c67c1d91574ef22336cc3a5944d1e63230c9 roms/ipxe (v1.0.0-2388-gb991c67)
-#
-for SUBPATH in $(git submodule status | sed -E 's/.*[[:space:]](.*)([[:space:]].*|$)/\1/')
-do
-    SUBURL=$(git config -f .gitmodules --get submodule.$SUBPATH.url)
-    SUBMIRROR=$(join -o"2.2" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
-    [ ${SUBMIRROR} ] || continue
-    SUBMIRROR=${SUBMIRROR//@GENERAL_MIRROR_SERVER@/${GITLAB_SSH_SERVER}}
-
-    ALLOW_TO_SYNC=$(join -o"2.3" <(echo ${SUBPATH}) <(sort ${MIRRORLIST}))
-    if [ "${ALLOW_TO_SYNC}" != "ALLOW_TO_SYNC_FROM_PUBLIC" ]
-    then
-        echo "[should not to sync mirror] ${SUBMIRROR}"
-        continue
-    fi
-
-    echo -e "[mirror sync] ${SUBURL} \tto\t ${SUBMIRROR}"
-
-    pushd ${SUBPATH} >/dev/null
-        git fetch --prune
-        git push --prune ${SUBMIRROR} +refs/remotes/origin/*:refs/heads/* +refs/tags/*:refs/tags/*
-    popd >/dev/null
-done
-
-popd >/dev/null