From ce9c7e4f7901d7f86cf87ed9c24ddc787dde1143 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Tue, 19 Nov 2019 12:06:27 +0800 Subject: [PATCH] CI: try to use the correct branch of other projects used in CI: 1. revision defined in bot message 2. branch name (or tag name) of current IDF 3. CI_MERGE_REQUEST_TARGET_BRANCH_NAME 4. branch name parsed from `git describe` 5. default branch --- .gitlab-ci.yml | 19 +++++----- tools/ci/checkout_project_ref.py | 65 +++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7dd932d4..8ac163fbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -122,11 +122,11 @@ build_template_app: # Set the variable for 'esp-idf-template' testing - ESP_IDF_TEMPLATE_GIT=${ESP_IDF_TEMPLATE_GIT:-"https://github.com/espressif/esp-idf-template.git"} - git clone ${ESP_IDF_TEMPLATE_GIT} - - cd esp-idf-template # Try to use the same branch name for esp-idf-template that we're # using on esp-idf. If it doesn't exist then just stick to the default # branch - - python $CHECKOUT_REF_SCRIPT esp-idf-template + - python $CHECKOUT_REF_SCRIPT esp-idf-template esp-idf-template + - cd esp-idf-template - make defconfig # Test debug build (default) - make all V=1 @@ -166,8 +166,8 @@ build_ssc: - $BOT_LABEL_REGULAR_TEST script: - git clone $SSC_REPOSITORY + - python $CHECKOUT_REF_SCRIPT SSC SSC - cd SSC - - python $CHECKOUT_REF_SCRIPT SSC - MAKEFLAGS= ./ci_build_ssc.sh .build_esp_idf_unit_test_template: &build_esp_idf_unit_test_template @@ -779,8 +779,8 @@ assign_test: - python $TEST_FW_PATH/CIAssignUnitTest.py $IDF_PATH/components/idf_test/unit_test/TestCaseAll.yml $IDF_PATH/.gitlab-ci.yml $IDF_PATH/components/idf_test/unit_test/CIConfigs # clone test script to assign tests - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - python $CHECKOUT_REF_SCRIPT auto_test_script # assgin integration test cases - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin @@ -825,8 +825,7 @@ assign_test: - test -e $CONFIG_FILE || exit 0 # clone test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY - - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - cd $TEST_FW_PATH # run test - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE -e $ENV_FILE @@ -889,12 +888,12 @@ assign_test: - test -e $CONFIG_FILE || exit 0 # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - python $CHECKOUT_REF_SCRIPT auto_test_script # run test - python CIRunner.py -l "$LOG_PATH/$JOG_FULL_NAME" -c $CONFIG_FILE -e $LOCAL_ENV_CONFIG_PATH -t $TEST_CASE_FILE_PATH @@ -915,12 +914,12 @@ nvs_compatible_test: - test -e $CONFIG_FILE || exit 0 # clone local test env configs - git clone $TEST_ENV_CONFIG_REPOSITORY + - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs - cd ci-test-runner-configs - - python $CHECKOUT_REF_SCRIPT ci-test-runner-configs # clone test bench - git clone $TEST_SCRIPT_REPOSITORY + - python $CHECKOUT_REF_SCRIPT auto_test_script auto_test_script - cd auto_test_script - - git checkout ${CI_COMMIT_REF_NAME} || echo "Using default branch..." # prepare nvs bins - ./Tools/prepare_nvs_bin.sh # run test diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 5218a7cc2..e9c060a1b 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -7,32 +7,71 @@ import os import json import argparse import subprocess +import re -def checkout_branch(proj_name, customized_revision, default_ref_name): +IDF_GIT_DESCRIBE_PATTERN = re.compile(r"^v(\d)\.(\d)") + + +def target_branch_candidates(proj_name): + """ + :return: a list of target branch candidates, from highest priority to lowest priority. + """ + candidates = [ + # branch name (or tag name) of current IDF + os.getenv("CI_COMMIT_REF_NAME"), + # CI_MERGE_REQUEST_TARGET_BRANCH_NAME + os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"), + ] + # revision defined in bot message + customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") + if customized_project_revisions: + customized_project_revisions = json.loads(customized_project_revisions) try: - ref_to_use = customized_revision[proj_name.lower()] + ref_to_use = customized_project_revisions[proj_name.lower()] + # highest priority, insert to head of list + candidates.insert(0, ref_to_use) except (KeyError, TypeError): - ref_to_use = default_ref_name - + pass + # branch name read from IDF try: - subprocess.check_call(["git", "checkout", ref_to_use]) - print("CI using ref {} for project {}".format(ref_to_use, proj_name)) + git_describe = subprocess.check_output(["git", "describe", "--tags", "HEAD"]) + match = IDF_GIT_DESCRIBE_PATTERN.search(git_describe.decode()) + if match: + major_revision = match.group(1) + minor_revision = match.group(2) + # release branch + candidates.append("release/v{}.{}".format(major_revision, minor_revision)) + # branch to match all major branches, like v3.x or v3 + candidates.append("release/v{}.x".format(major_revision)) + candidates.append("release/v{}".format(major_revision)) except subprocess.CalledProcessError: - print("using default branch") + # this should not happen as IDF should have describe message + pass + + return [c for c in candidates if c] # filter out null value if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("project", help="the name of project") + parser.add_argument("project_relative_path", + help="relative path of project to IDF repository directory") args = parser.parse_args() - project_name = args.project - customized_project_revisions = os.getenv("BOT_CUSTOMIZED_REVISION") - if customized_project_revisions: - customized_project_revisions = json.loads(customized_project_revisions) - ci_ref_name = os.getenv("CI_COMMIT_REF_NAME") + candidate_branches = target_branch_candidates(args.project) - checkout_branch(project_name, customized_project_revisions, ci_ref_name) + # change to project dir for checkout + os.chdir(args.project_relative_path) + + for candidate in candidate_branches: + try: + subprocess.check_call(["git", "checkout", candidate]) + print("CI using ref {} for project {}".format(candidate, args.project)) + break + except subprocess.CalledProcessError: + pass + else: + print("using default branch")