CI: modify fetch submodule method:

download archive for submodules instead of clone

.gitlab-ci.yml

@@ -23,9 +23,14 @@ variables:
   # GIT_STRATEGY is not defined here.
   # Use an option from  "CI / CD Settings" - "General pipelines".
 
-  # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF.
-  # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules.
-  GIT_SUBMODULE_STRATEGY: normal
+  # we will download archive for each submodule instead of clone.
+  # we don't do "recursive" when fetch submodule as they're not used in CI now.
+  GIT_SUBMODULE_STRATEGY: none
+  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
+  # by default we will fetch all submodules
+  # jobs can overwrite this variable to only fetch submodules they required
+  # set to "none" if don't need to fetch submodules
+  SUBMODULES_TO_FETCH: "all"
 
   UNIT_TEST_BUILD_SYSTEM: cmake
   EXAMPLE_TEST_BUILD_SYSTEM: cmake
@@ -44,6 +49,7 @@ variables:
   CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml"
 
 
+
 # before each job, we need to check if this job is filtered by bot stage/job filter
 .apply_bot_filter: &apply_bot_filter
   python $APPLY_BOT_FILTER_SCRIPT || exit 0
@@ -72,12 +78,10 @@ variables:
   tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
   fi
 
-.show_submodule_urls: &show_submodule_urls |
-  git config --get-regexp '^submodule\..*\.url$' || true
+.fetch_submodules: &fetch_submodules |
+  python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH
 
 before_script:
-  - echo "Running common script"
-  - *show_submodule_urls
   - source tools/ci/setup_python.sh
   # apply bot filter in before script
   - *apply_bot_filter
@@ -93,6 +97,8 @@ before_script:
 
   - *setup_tools_unless_target_test
 
+  - *fetch_submodules
+
   - *setup_custom_toolchain
 
 # used for check scripts which we want to run unconditionally

tools/ci/ci_fetch_submodule.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+
+# internal use only for CI
+# download archive of one commit instead of cloning entire submodule repo
+
+import re
+import os
+import subprocess
+import argparse
+import shutil
+import time
+
+import gitlab_api
+
+SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]")
+PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)")
+URL_PATTERN = re.compile(r"url\s+=\s+(\S+)")
+
+SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive"
+
+
+class SubModule(object):
+    # We don't need to support recursive submodule clone now
+
+    GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+")
+
+    def __init__(self, gitlab_inst, path, url):
+        self.path = path
+        self.gitlab_inst = gitlab_inst
+        self.project_id = self._get_project_id(url)
+        self.commit_id = self._get_commit_id(path)
+
+    def _get_commit_id(self, path):
+        output = subprocess.check_output(["git", "ls-tree", "HEAD", path])
+        # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0	components/esp_wifi/lib
+        match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output)
+        return match.group(1)
+
+    def _get_project_id(self, url):
+        base_name = os.path.basename(url)
+        project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0],  # remove .git
+                                                     namespace="espressif")
+        return project_id
+
+    def download_archive(self):
+        print("Update submodule: {}: {}".format(self.path, self.commit_id))
+        path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER,
+                                                      self.project_id)
+        renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path))
+        os.rename(path_name, renamed_path)
+        shutil.rmtree(self.path, ignore_errors=True)
+        shutil.move(renamed_path, os.path.dirname(self.path))
+
+
+def update_submodule(git_module_file, submodules_to_update):
+    gitlab_inst = gitlab_api.Gitlab()
+    submodules = []
+    with open(git_module_file, "r") as f:
+        data = f.read()
+    match = SUBMODULE_PATTERN.search(data)
+    while True:
+        next_match = SUBMODULE_PATTERN.search(data, pos=match.end())
+        if next_match:
+            end_pos = next_match.start()
+        else:
+            end_pos = len(data)
+        path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+        url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos)
+        path = path_match.group(1)
+        url = url_match.group(1)
+
+        filter_result = True
+        if submodules_to_update:
+            if path not in submodules_to_update:
+                filter_result = False
+        if filter_result:
+            submodules.append(SubModule(gitlab_inst, path, url))
+
+        match = next_match
+        if not match:
+            break
+
+    shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True)
+
+    for submodule in submodules:
+        submodule.download_archive()
+
+
+if __name__ == '__main__':
+    start_time = time.time()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo_path", "-p", default=".", help="repo path")
+    parser.add_argument("--submodule", "-s", default="all",
+                        help="Submodules to update. By default update all submodules. "
+                             "For multiple submodules, separate them with `;`. "
+                             "`all` and `none` are special values that indicates we fetch all / none submodules")
+    args = parser.parse_args()
+    if args.submodule == "none":
+        print("don't need to update submodules")
+        exit(0)
+    if args.submodule == "all":
+        _submodules = []
+    else:
+        _submodules = args.submodule.split(";")
+    update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules)
+    print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time))

tools/ci/config/assign-test.yml

@@ -10,6 +10,7 @@ assign_test:
     - build_ssc
     - build_esp_idf_tests_cmake
   variables:
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
   artifacts:
@@ -55,6 +56,7 @@ update_test_cases:
       - ${CI_PROJECT_DIR}/test-management/*.log
     expire_in: 1 week
   variables:
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml"
     BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml"
     AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script"

tools/ci/config/post_check.yml

@@ -10,7 +10,7 @@ check_submodule_sync:
   retry: 2
   variables:
     GIT_STRATEGY: clone
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   before_script: []
   after_script: []

tools/ci/config/target-test.yml

@@ -33,7 +33,7 @@
     CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs"
     LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS"
     ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml"
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
   script:
     - *define_config_file_name
     # first test if config file exists, if not exist, exit 0
@@ -41,9 +41,6 @@
     # clone test env configs
     - git clone $TEST_ENV_CONFIG_REPOSITORY
     - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
-    # Get esptool
-    - git submodule sync
-    - git submodule update --init components/esptool_py/esptool
     - cd tools/ci/python_packages/tiny_test_fw/bin
     # run test
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE
@@ -134,7 +131,7 @@
       - $LOG_PATH
     expire_in: 1 week
   variables:
-    GIT_SUBMODULE_STRATEGY: none
+    SUBMODULES_TO_FETCH: "components/esptool_py/esptool"
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}"
     TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test"
@@ -195,8 +192,6 @@ example_test_001:
 example_test_002:
   extends: .example_test_template
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
-  variables:
-    GIT_SUBMODULE_STRATEGY: normal
   tags:
     - ESP32
     - Example_ShieldBox_Basic