CI: support only download artifacts by request:

use CI dependencies could waste a lot bandwidth for target test jobs, as
example binary artifacts are very large. Now we will parse required
artifacts first, then use API to download required files in artifacts.

tools/ci/config/target-test.yml

@@ -21,7 +21,6 @@
       - $BOT_LABEL_EXAMPLE_TEST
   dependencies:
     - assign_test
-    - build_examples_cmake_esp32
   artifacts:
     when: always
     paths:
@@ -64,8 +63,6 @@
       - $BOT_LABEL_EXAMPLE_TEST
   dependencies:
     - assign_test
-    - build_examples_make
-    - build_examples_cmake_esp32
   artifacts:
     when: always
     paths:
@@ -282,7 +279,6 @@ example_test_010:
 
 example_test_011:
   extends: .example_debug_template
-  parallel: 4
   tags:
     - ESP32
     - Example_T2_RS485

tools/ci/python_packages/ttfw_idf/CIAssignExampleTest.py

@@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs.
 
 # TODO: Need to handle running examples on different chips
 import os
-import sys
 import re
 import argparse
+import json
 
 import gitlab_api
 from tiny_test_fw.Utility import CIAssignTest
 
 
+EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
+IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
+if IDF_PATH_FROM_ENV:
+    ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
+                                       "build_examples", "artifact_index.json")
+else:
+    ARTIFACT_INDEX_FILE = "artifact_index.json"
+
+
 class ExampleGroup(CIAssignTest.Group):
     SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
 
@@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest):
     CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
 
 
-class ArtifactFile(object):
+def create_artifact_index_file(project_id=None, pipeline_id=None):
-    def __init__(self, project_id, job_name, artifact_file_path):
+    if project_id is None:
-        self.gitlab_api = gitlab_api.Gitlab(project_id)
+        project_id = os.getenv("CI_PROJECT_ID")
+    if pipeline_id is None:
+        pipeline_id = os.getenv("CI_PIPELINE_ID")
+    gitlab_inst = gitlab_api.Gitlab(project_id)
+    artifact_index_list = []
 
-    def process(self):
+    def format_build_log_path():
+        return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
+
+    for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
+        job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
+        for job_info in job_info_list:
+            raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
+            build_info_list = [json.loads(line) for line in raw_data.splitlines()]
+            for build_info in build_info_list:
+                build_info["ci_job_id"] = job_info["id"]
+                artifact_index_list.append(build_info)
+    try:
+        os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
+    except OSError:
+        # already created
         pass
 
-    def output(self):
+    with open(ARTIFACT_INDEX_FILE, "w") as f:
-        pass
+        json.dump(artifact_index_list, f)
 
 
 if __name__ == '__main__':
@@ -53,8 +80,11 @@ if __name__ == '__main__':
                         help="gitlab ci config file")
     parser.add_argument("output_path",
                         help="output path of config files")
+    parser.add_argument("--pipeline_id", "-p", type=int, default=None,
+                        help="pipeline_id")
     args = parser.parse_args()
 
     assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
     assign_test.assign_cases()
     assign_test.output_configs(args.output_path)
+    create_artifact_index_file()

tools/ci/python_packages/ttfw_idf/IDFApp.py

@@ -17,7 +17,102 @@ import subprocess
 
 import os
 import json
+
 from tiny_test_fw import App
+from . import CIAssignExampleTest
+
+try:
+    import gitlab_api
+except ImportError:
+    gitlab_api = None
+
+
+def parse_flash_settings(path):
+    file_name = os.path.basename(path)
+    if file_name == "flasher_args.json":
+        # CMake version using build metadata file
+        with open(path, "r") as f:
+            args = json.load(f)
+            flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""]
+            flash_settings = args["flash_settings"]
+    else:
+        # GNU Make version uses download.config arguments file
+        with open(path, "r") as f:
+            args = f.readlines()[-1].split(" ")
+            flash_files = []
+            flash_settings = {}
+            for idx in range(0, len(args), 2):  # process arguments in pairs
+                if args[idx].startswith("--"):
+                    # strip the -- from the command line argument
+                    flash_settings[args[idx][2:]] = args[idx + 1]
+                else:
+                    # offs, filename
+                    flash_files.append((args[idx], args[idx + 1]))
+    return flash_files, flash_settings
+
+
+class Artifacts(object):
+    def __init__(self, dest_root_path):
+        assert gitlab_api
+        self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
+        self.dest_root_path = dest_root_path
+
+    @staticmethod
+    def _find_artifact(artifact_index, app_path, config_name, target):
+        for artifact_info in artifact_index:
+            match_result = True
+            if app_path:
+                match_result = app_path in artifact_info["app_dir"]
+            if config_name:
+                match_result = match_result and config_name == artifact_info["config"]
+            if target:
+                match_result = match_result and target == artifact_info["target"]
+            if match_result:
+                ret = artifact_info
+                break
+        else:
+            ret = None
+        return ret
+
+    def download_artifact(self, artifact_index_file, app_path, config_name, target):
+        # at least one of app_path or config_name is not None. otherwise we can't match artifact
+        assert app_path or config_name
+        assert os.path.exists(artifact_index_file)
+        with open(artifact_index_file, "r") as f:
+            artifact_index = json.load(f)
+
+        artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
+
+        if artifact_info:
+            base_path = os.path.join(artifact_info["work_dir"], artifact_info["build_dir"])
+            job_id = artifact_info["ci_job_id"]
+
+            # 1. download flash args file
+            if artifact_info["build_system"] == "cmake":
+                flash_arg_file = os.path.join(base_path, "flasher_args.json")
+            else:
+                flash_arg_file = os.path.join(base_path, "download.config")
+
+            self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
+
+            # 2. download all binary files
+            flash_files, flash_settings = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
+            artifact_files = []
+            for p in flash_files:
+                artifact_files.append(os.path.join(base_path, p[1]))
+                if not os.path.dirname(p[1]):
+                    # find app bin and also download elf
+                    elf_file = os.path.splitext(p[1])[0] + ".elf"
+                    artifact_files.append(os.path.join(base_path, elf_file))
+
+            self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
+
+            # 3. download sdkconfig file
+            self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
+                                               self.dest_root_path)
+        else:
+            base_path = None
+        return base_path
 
 
 class IDFApp(App.BaseApp):
@@ -34,7 +129,7 @@ class IDFApp(App.BaseApp):
         self.config_name = config_name
         self.target = target
         self.idf_path = self.get_sdk_path()
-        self.binary_path = self.get_binary_path(app_path, config_name)
+        self.binary_path = self.get_binary_path(app_path, config_name, target)
         self.elf_file = self._get_elf_file_path(self.binary_path)
         assert os.path.exists(self.binary_path)
         sdkconfig_dict = self.get_sdkconfig()
@@ -72,7 +167,6 @@ class IDFApp(App.BaseApp):
         """
         reads sdkconfig and returns a dictionary with all configuredvariables
 
-        :param sdkconfig_file: location of sdkconfig
         :raise: AssertionError: if sdkconfig file does not exist in defined paths
         """
         d = {}
@@ -89,14 +183,15 @@ class IDFApp(App.BaseApp):
                     d[configs[0]] = configs[1].rstrip()
         return d
 
-    def get_binary_path(self, app_path, config_name=None):
+    def get_binary_path(self, app_path, config_name=None, target=None):
         """
         get binary path according to input app_path.
 
         subclass must overwrite this method.
 
         :param app_path: path of application
-        :param config_name: name of the application build config
+        :param config_name: name of the application build config. Will match any config if None
+        :param target: target name. Will match for target if None
         :return: abs app binary path
         """
         pass
@@ -123,24 +218,12 @@ class IDFApp(App.BaseApp):
 
         if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path):
             # CMake version using build metadata file
-            with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f:
+            path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE)
-                args = json.load(f)
-                flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""]
-                flash_settings = args["flash_settings"]
         else:
             # GNU Make version uses download.config arguments file
-            with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f:
+            path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
-                args = f.readlines()[-1].split(" ")
-                flash_files = []
-                flash_settings = {}
-                for idx in range(0, len(args), 2):  # process arguments in pairs
-                    if args[idx].startswith("--"):
-                        # strip the -- from the command line argument
-                        flash_settings[args[idx][2:]] = args[idx + 1]
-                    else:
-                        # offs, filename
-                        flash_files.append((args[idx], args[idx + 1]))
 
+        flash_files, flash_settings = parse_flash_settings(path)
         # The build metadata file does not currently have details, which files should be encrypted and which not.
         # Assume that all files should be encrypted if flash encryption is enabled in development mode.
         sdkconfig_dict = self.get_sdkconfig()
@@ -149,7 +232,7 @@ class IDFApp(App.BaseApp):
         # make file offsets into integers, make paths absolute
         flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files]
 
-        return (flash_files, flash_settings)
+        return flash_files, flash_settings
 
     def _parse_partition_table(self):
         """
@@ -209,7 +292,7 @@ class Example(IDFApp):
         """
         return [os.path.join(self.binary_path, "..", "sdkconfig")]
 
-    def get_binary_path(self, app_path, config_name=None):
+    def get_binary_path(self, app_path, config_name=None, target=None):
         # build folder of example path
         path = os.path.join(self.idf_path, app_path, "build")
         if os.path.exists(path):
@@ -227,18 +310,23 @@ class Example(IDFApp):
         for dirpath in os.listdir(example_path):
             if os.path.basename(dirpath) == app_path_underscored:
                 path = os.path.join(example_path, dirpath, config_name, self.target, "build")
-                return path
+                if os.path.exists(path):
+                    return path
+                else:
+                    # app path exists, but config name not exists. try to download artifacts.
+                    break
 
-        raise OSError("Failed to find example binary")
+        artifacts = Artifacts(self.idf_path)
+        path = artifacts.download_artifact(CIAssignExampleTest.ARTIFACT_INDEX_FILE,
+                                           app_path, config_name, target)
+        if path:
+            return os.path.join(self.idf_path, path)
+        else:
+            raise OSError("Failed to find example binary")
 
 
 class UT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
+    def get_binary_path(self, app_path, config_name=None, target=None):
-        """
-        :param app_path: app path
-        :param config_name: config name
-        :return: binary path
-        """
         if not config_name:
             config_name = "default"
 
@@ -262,12 +350,12 @@ class UT(IDFApp):
 
 
 class SSC(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
+    def get_binary_path(self, app_path, config_name=None, target=None):
         # TODO: to implement SSC get binary path
         return app_path
 
 
 class AT(IDFApp):
-    def get_binary_path(self, app_path, config_name=None):
+    def get_binary_path(self, app_path, config_name=None, target=None):
         # TODO: to implement AT get binary path
         return app_path

tools/ci/python_packages/ttfw_idf/__init__.py

@@ -16,7 +16,7 @@ import re
 
 from tiny_test_fw import TinyFW, Utility
 from IDFApp import IDFApp, Example, UT
-from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT
+from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT  # noqa: export DUTs for users
 
 
 def format_case_id(chip, case_name):