diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3897135ce..2b5e8cf79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,9 +23,14 @@ variables: # GIT_STRATEGY is not defined here. # Use an option from "CI / CD Settings" - "General pipelines". - # "normal" strategy for fetching only top-level submodules since nothing requires the sub-submodules code for building IDF. - # If the "recursive" strategy is used we have a problem with using relative URLs for sub-submodules. - GIT_SUBMODULE_STRATEGY: normal + # we will download archive for each submodule instead of clone. + # we don't do "recursive" when fetch submodule as they're not used in CI now. + GIT_SUBMODULE_STRATEGY: none + SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" + # by default we will fetch all submodules + # jobs can overwrite this variable to only fetch submodules they required + # set to "none" if don't need to fetch submodules + SUBMODULES_TO_FETCH: "all" UNIT_TEST_BUILD_SYSTEM: cmake EXAMPLE_TEST_BUILD_SYSTEM: cmake @@ -44,6 +49,7 @@ variables: CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/tools/ci/config/target-test.yml" + # before each job, we need to check if this job is filtered by bot stage/job filter .apply_bot_filter: &apply_bot_filter python $APPLY_BOT_FILTER_SCRIPT || exit 0 @@ -72,12 +78,10 @@ variables: tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1 fi -.show_submodule_urls: &show_submodule_urls | - git config --get-regexp '^submodule\..*\.url$' || true +.fetch_submodules: &fetch_submodules | + python $SUBMODULE_FETCH_TOOL -s $SUBMODULES_TO_FETCH before_script: - - echo "Running common script" - - *show_submodule_urls - source tools/ci/setup_python.sh # apply bot filter in before script - *apply_bot_filter @@ -93,6 +97,8 @@ before_script: - *setup_tools_unless_target_test + - *fetch_submodules + - *setup_custom_toolchain # used for check scripts which we want to run unconditionally diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py new file mode 100644 index 000000000..89dce82ea --- /dev/null +++ b/tools/ci/ci_fetch_submodule.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +# internal use only for CI +# download archive of one commit instead of cloning entire submodule repo + +import re +import os +import subprocess +import argparse +import shutil +import time + +import gitlab_api + +SUBMODULE_PATTERN = re.compile(r"\[submodule \"([^\"]+)\"]") +PATH_PATTERN = re.compile(r"path\s+=\s+(\S+)") +URL_PATTERN = re.compile(r"url\s+=\s+(\S+)") + +SUBMODULE_ARCHIVE_TEMP_FOLDER = "submodule_archive" + + +class SubModule(object): + # We don't need to support recursive submodule clone now + + GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r"\d+\s+commit\s+([0-9a-f]+)\s+") + + def __init__(self, gitlab_inst, path, url): + self.path = path + self.gitlab_inst = gitlab_inst + self.project_id = self._get_project_id(url) + self.commit_id = self._get_commit_id(path) + + def _get_commit_id(self, path): + output = subprocess.check_output(["git", "ls-tree", "HEAD", path]) + # example output: 160000 commit d88a262fbdf35e5abb372280eb08008749c3faa0 components/esp_wifi/lib + match = self.GIT_LS_TREE_OUTPUT_PATTERN.search(output) + return match.group(1) + + def _get_project_id(self, url): + base_name = os.path.basename(url) + project_id = self.gitlab_inst.get_project_id(os.path.splitext(base_name)[0], # remove .git + namespace="espressif") + return project_id + + def download_archive(self): + print("Update submodule: {}: {}".format(self.path, self.commit_id)) + path_name = self.gitlab_inst.download_archive(self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, + self.project_id) + renamed_path = os.path.join(os.path.dirname(path_name), os.path.basename(self.path)) + os.rename(path_name, renamed_path) + shutil.rmtree(self.path, ignore_errors=True) + shutil.move(renamed_path, os.path.dirname(self.path)) + + +def update_submodule(git_module_file, submodules_to_update): + gitlab_inst = gitlab_api.Gitlab() + submodules = [] + with open(git_module_file, "r") as f: + data = f.read() + match = SUBMODULE_PATTERN.search(data) + while True: + next_match = SUBMODULE_PATTERN.search(data, pos=match.end()) + if next_match: + end_pos = next_match.start() + else: + end_pos = len(data) + path_match = PATH_PATTERN.search(data, pos=match.end(), endpos=end_pos) + url_match = URL_PATTERN.search(data, pos=match.end(), endpos=end_pos) + path = path_match.group(1) + url = url_match.group(1) + + filter_result = True + if submodules_to_update: + if path not in submodules_to_update: + filter_result = False + if filter_result: + submodules.append(SubModule(gitlab_inst, path, url)) + + match = next_match + if not match: + break + + shutil.rmtree(SUBMODULE_ARCHIVE_TEMP_FOLDER, ignore_errors=True) + + for submodule in submodules: + submodule.download_archive() + + +if __name__ == '__main__': + start_time = time.time() + parser = argparse.ArgumentParser() + parser.add_argument("--repo_path", "-p", default=".", help="repo path") + parser.add_argument("--submodule", "-s", default="all", + help="Submodules to update. By default update all submodules. " + "For multiple submodules, separate them with `;`. " + "`all` and `none` are special values that indicates we fetch all / none submodules") + args = parser.parse_args() + if args.submodule == "none": + print("don't need to update submodules") + exit(0) + if args.submodule == "all": + _submodules = [] + else: + _submodules = args.submodule.split(";") + update_submodule(os.path.join(args.repo_path, ".gitmodules"), _submodules) + print("total time spent on update submodule: {:.02f}s".format(time.time() - start_time)) diff --git a/tools/ci/config/assign-test.yml b/tools/ci/config/assign-test.yml index 9846b8d00..e997062e3 100644 --- a/tools/ci/config/assign-test.yml +++ b/tools/ci/config/assign-test.yml @@ -10,6 +10,7 @@ assign_test: - build_ssc - build_esp_idf_tests_cmake variables: + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" EXAMPLE_CONFIG_OUTPUT_PATH: "$CI_PROJECT_DIR/examples/test_configs" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" artifacts: @@ -55,6 +56,7 @@ update_test_cases: - ${CI_PROJECT_DIR}/test-management/*.log expire_in: 1 week variables: + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" UNIT_TEST_CASE_FILE: "${CI_PROJECT_DIR}/components/idf_test/unit_test/TestCaseAll.yml" BOT_ACCOUNT_CONFIG_FILE: "${CI_PROJECT_DIR}/test-management/Config/Account.local.yml" AUTO_TEST_SCRIPT_PATH: "${CI_PROJECT_DIR}/auto_test_script" diff --git a/tools/ci/config/post_check.yml b/tools/ci/config/post_check.yml index 2f9dacc12..d3d577892 100644 --- a/tools/ci/config/post_check.yml +++ b/tools/ci/config/post_check.yml @@ -10,7 +10,7 @@ check_submodule_sync: retry: 2 variables: GIT_STRATEGY: clone - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "none" PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" before_script: [] after_script: [] diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 0ec079ee4..63bffea83 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -33,7 +33,7 @@ CONFIG_FILE_PATH: "${CI_PROJECT_DIR}/examples/test_configs" LOG_PATH: "$CI_PROJECT_DIR/TEST_LOGS" ENV_FILE: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/EnvConfig.yml" - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" script: - *define_config_file_name # first test if config file exists, if not exist, exit 0 @@ -41,9 +41,6 @@ # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - # Get esptool - - git submodule sync - - git submodule update --init components/esptool_py/esptool - cd tools/ci/python_packages/tiny_test_fw/bin # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -134,7 +131,7 @@ - $LOG_PATH expire_in: 1 week variables: - GIT_SUBMODULE_STRATEGY: none + SUBMODULES_TO_FETCH: "components/esptool_py/esptool" LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF" LOG_PATH: "${CI_PROJECT_DIR}/${CI_COMMIT_SHA}" TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test/integration_test" @@ -195,8 +192,6 @@ example_test_001: example_test_002: extends: .example_test_template image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG - variables: - GIT_SUBMODULE_STRATEGY: normal tags: - ESP32 - Example_ShieldBox_Basic