From 6eca80a9899ec02325fe9d8a5c9bfb5c9d2c68d1 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Sun, 28 Apr 2019 16:57:59 +0800 Subject: [PATCH] CI: try to use the correct branch of other projects used in CI: 1. revision defined in bot message 2. branch name (or tag name) of current IDF 3. CI_MERGE_REQUEST_TARGET_BRANCH_NAME 4. branch name parsed from `git describe` 5. default branch --- .gitlab-ci.yml | 23 +++++------- tools/ci/checkout_project_ref.py | 63 +++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99a21cbb8..a42b47fa4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,11 +131,11 @@ build_template_app: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - git clone ${ESP_IDF_TEMPLATE_GIT} + - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template - cd esp-idf-template # Try to use the same branch name for esp-idf-template that we're # using on esp-idf. If it doesn't exist then just stick to the default # branch - - python $CHECKOUT_REF_SCRIPT esp-idf-template - make defconfig # Test debug build (default) - make all V=1 @@ -175,8 +175,8 @@ build_ssc: - $BOT_LABEL_REGULAR_TEST script: - git clone $SSC_REPOSITORY + - python $CHECKOUT_REF_SCRIPT SSC SSC - cd SSC - - python $CHECKOUT_REF_SCRIPT SSC - MAKEFLAGS= ./ci_build_ssc.sh # If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template` @@ -771,8 +771,8 @@ update_test_cases: script: - export GIT_SHA=$(echo ${CI_COMMIT_SHA} | cut -c 1-8) - git clone $TEST_MANAGEMENT_REPO + - python $CHECKOUT_REF_SCRIPT test-management test-management - cd test-management - - python $CHECKOUT_REF_SCRIPT test-management - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} # update unit test cases - python ImportTestCase.py $JIRA_TEST_MANAGEMENT_PROJECT unity -d $UNIT_TEST_CASE_FILE -r $GIT_SHA @@ -821,8 +821,8 @@ deploy_test_result: # we need to remove it so we can clone test-management folder again - rm -r test-management - git clone $TEST_MANAGEMENT_REPO + - python3 $CHECKOUT_REF_SCRIPT test-management test-management - cd test-management - - python3 $CHECKOUT_REF_SCRIPT test-management - echo $BOT_JIRA_ACCOUNT > ${BOT_ACCOUNT_CONFIG_FILE} # update test results - python3 ImportTestResult.py -r "$GIT_SHA (r${REV_COUNT})" -j $JIRA_TEST_MANAGEMENT_PROJECT -s "$SUMMARY" -l CI -p ${CI_PROJECT_DIR}/TEST_LOGS ${CI_PROJECT_DIR}/${CI_COMMIT_SHA} --pipeline_url ${CI_PIPELINE_URL} @@ -1005,8 +1005,8 @@ assign_test: - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $IDF_PATH/.gitlab-ci.yml $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - python $CHECKOUT_REF_SCRIPT auto_test_script # assgin integration test cases - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin @@ -1045,8 +1045,7 @@ assign_test: - test -e $CONFIG_FILE || exit 0 # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - cd $TEST_FW_PATH # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -1143,12 +1142,11 @@ test_weekend_network: - test -e $CONFIG_FILE || exit 0 # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - python $CHECKOUT_REF_SCRIPT auto_test_script # run test - python CIRunner.py -l "$LOG_PATH/$CI_JOB_NAME_$CI_NODE_INDEX" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH -m $MODULE_UPDATE_FILE @@ -1166,12 +1164,11 @@ nvs_compatible_test: script: # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..." # prepare nvs bins - ./Tools/prepare_nvs_bin.sh # run test diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 5218a7cc2..5b6b332c9 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -7,32 +7,67 @@ import os import json import argparse import subprocess +import re -def checkout_branch(proj_name, customized_revision, default_ref_name): +IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)") + + +def target_branch_candidates(proj_name): + """ + :return: a list of target branch candidates, from highest priority to lowest priority. + """ + candidates = [ + # branch name (or tag name) of current IDF + os.getenv("CI_COMMIT_REF_NAME"), + # CI_MERGE_REQUEST_TARGET_BRANCH_NAME + os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"), + ] + # revision defined in bot message + customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") + if customized_project_revisions: + customized_project_revisions = json.loads(customized_project_revisions) try: - ref_to_use = customized_revision[proj_name.lower()] + ref_to_use = customized_project_revisions[proj_name.lower()] + # highest priority, insert to head of list + candidates.insert(0, ref_to_use) except (KeyError, TypeError): - ref_to_use = default_ref_name + pass + # branch name read from IDF + git_describe = subprocess.check_output(["git", "describe", "--tags", "HEAD"]) + match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe) + if match: + major_revision = match.group(1) + minor_revision = match.group(2) + # release branch + candidates.append("release/v{}.{}".format(major_revision, minor_revision)) + # branch to match all major branches, like v3.x or v3 + candidates.append("release/v{}.x".format(major_revision)) + candidates.append("release/v{}".format(major_revision)) - try: - subprocess.check_call(["git", "checkout", ref_to_use]) - print("CI using ref {} for project {}".format(ref_to_use, proj_name)) - except subprocess.CalledProcessError: - print("using default branch") + return [c for c in candidates if c] # filter out null value if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("project", help="the name of project") + parser.add_argument("project_relative_path", + help="relative path of project to IDF repository directory") args = parser.parse_args() - project_name = args.project - customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") - if customized_project_revisions: - customized_project_revisions = json.loads(customized_project_revisions) - ci_ref_name = os.getenv("CI_COMMIT_REF_NAME") + candidate_branches = target_branch_candidates(args.project) - checkout_branch(project_name, customized_project_revisions, ci_ref_name) + # change to project dir for checkout + os.chdir(args.project_relative_path) + + for candidate in candidate_branches: + try: + subprocess.check_call(["git", "checkout", candidate]) + print("CI using ref {} for project {}".format(candidate, args.project)) + break + except subprocess.CalledProcessError: + pass + else: + print("using default branch")