CI: support only download artifacts by request:
use CI dependencies could waste a lot bandwidth for target test jobs, as example binary artifacts are very large. Now we will parse required artifacts first, then use API to download required files in artifacts.
This commit is contained in:
parent
c906e2afee
commit
89f8e19850
|
@ -21,7 +21,6 @@
|
|||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
dependencies:
|
||||
- assign_test
|
||||
- build_examples_cmake_esp32
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
|
@ -64,8 +63,6 @@
|
|||
- $BOT_LABEL_EXAMPLE_TEST
|
||||
dependencies:
|
||||
- assign_test
|
||||
- build_examples_make
|
||||
- build_examples_cmake_esp32
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
|
@ -282,7 +279,6 @@ example_test_010:
|
|||
|
||||
example_test_011:
|
||||
extends: .example_debug_template
|
||||
parallel: 4
|
||||
tags:
|
||||
- ESP32
|
||||
- Example_T2_RS485
|
||||
|
|
|
@ -18,14 +18,23 @@ Command line tool to assign example tests to CI test jobs.
|
|||
|
||||
# TODO: Need to handle running examples on different chips
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import argparse
|
||||
import json
|
||||
|
||||
import gitlab_api
|
||||
from tiny_test_fw.Utility import CIAssignTest
|
||||
|
||||
|
||||
EXAMPLE_BUILD_JOB_NAMES = ["build_examples_cmake_esp32", "build_examples_cmake_esp32s2"]
|
||||
IDF_PATH_FROM_ENV = os.getenv("IDF_PATH")
|
||||
if IDF_PATH_FROM_ENV:
|
||||
ARTIFACT_INDEX_FILE = os.path.join(IDF_PATH_FROM_ENV,
|
||||
"build_examples", "artifact_index.json")
|
||||
else:
|
||||
ARTIFACT_INDEX_FILE = "artifact_index.json"
|
||||
|
||||
|
||||
class ExampleGroup(CIAssignTest.Group):
|
||||
SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"]
|
||||
|
||||
|
@ -34,15 +43,33 @@ class CIExampleAssignTest(CIAssignTest.AssignTest):
|
|||
CI_TEST_JOB_PATTERN = re.compile(r"^example_test_.+")
|
||||
|
||||
|
||||
class ArtifactFile(object):
|
||||
def __init__(self, project_id, job_name, artifact_file_path):
|
||||
self.gitlab_api = gitlab_api.Gitlab(project_id)
|
||||
def create_artifact_index_file(project_id=None, pipeline_id=None):
|
||||
if project_id is None:
|
||||
project_id = os.getenv("CI_PROJECT_ID")
|
||||
if pipeline_id is None:
|
||||
pipeline_id = os.getenv("CI_PIPELINE_ID")
|
||||
gitlab_inst = gitlab_api.Gitlab(project_id)
|
||||
artifact_index_list = []
|
||||
|
||||
def process(self):
|
||||
def format_build_log_path():
|
||||
return "build_examples/list_job_{}.json".format(job_info["parallel_num"])
|
||||
|
||||
for build_job_name in EXAMPLE_BUILD_JOB_NAMES:
|
||||
job_info_list = gitlab_inst.find_job_id(build_job_name, pipeline_id=pipeline_id)
|
||||
for job_info in job_info_list:
|
||||
raw_data = gitlab_inst.download_artifact(job_info["id"], [format_build_log_path()])[0]
|
||||
build_info_list = [json.loads(line) for line in raw_data.splitlines()]
|
||||
for build_info in build_info_list:
|
||||
build_info["ci_job_id"] = job_info["id"]
|
||||
artifact_index_list.append(build_info)
|
||||
try:
|
||||
os.makedirs(os.path.dirname(ARTIFACT_INDEX_FILE))
|
||||
except OSError:
|
||||
# already created
|
||||
pass
|
||||
|
||||
def output(self):
|
||||
pass
|
||||
with open(ARTIFACT_INDEX_FILE, "w") as f:
|
||||
json.dump(artifact_index_list, f)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
@ -53,8 +80,11 @@ if __name__ == '__main__':
|
|||
help="gitlab ci config file")
|
||||
parser.add_argument("output_path",
|
||||
help="output path of config files")
|
||||
parser.add_argument("--pipeline_id", "-p", type=int, default=None,
|
||||
help="pipeline_id")
|
||||
args = parser.parse_args()
|
||||
|
||||
assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup)
|
||||
assign_test.assign_cases()
|
||||
assign_test.output_configs(args.output_path)
|
||||
create_artifact_index_file()
|
||||
|
|
|
@ -17,7 +17,102 @@ import subprocess
|
|||
|
||||
import os
|
||||
import json
|
||||
|
||||
from tiny_test_fw import App
|
||||
from . import CIAssignExampleTest
|
||||
|
||||
try:
|
||||
import gitlab_api
|
||||
except ImportError:
|
||||
gitlab_api = None
|
||||
|
||||
|
||||
def parse_flash_settings(path):
|
||||
file_name = os.path.basename(path)
|
||||
if file_name == "flasher_args.json":
|
||||
# CMake version using build metadata file
|
||||
with open(path, "r") as f:
|
||||
args = json.load(f)
|
||||
flash_files = [(offs, binary) for (offs, binary) in args["flash_files"].items() if offs != ""]
|
||||
flash_settings = args["flash_settings"]
|
||||
else:
|
||||
# GNU Make version uses download.config arguments file
|
||||
with open(path, "r") as f:
|
||||
args = f.readlines()[-1].split(" ")
|
||||
flash_files = []
|
||||
flash_settings = {}
|
||||
for idx in range(0, len(args), 2): # process arguments in pairs
|
||||
if args[idx].startswith("--"):
|
||||
# strip the -- from the command line argument
|
||||
flash_settings[args[idx][2:]] = args[idx + 1]
|
||||
else:
|
||||
# offs, filename
|
||||
flash_files.append((args[idx], args[idx + 1]))
|
||||
return flash_files, flash_settings
|
||||
|
||||
|
||||
class Artifacts(object):
|
||||
def __init__(self, dest_root_path):
|
||||
assert gitlab_api
|
||||
self.gitlab_inst = gitlab_api.Gitlab(os.getenv("CI_PROJECT_ID"))
|
||||
self.dest_root_path = dest_root_path
|
||||
|
||||
@staticmethod
|
||||
def _find_artifact(artifact_index, app_path, config_name, target):
|
||||
for artifact_info in artifact_index:
|
||||
match_result = True
|
||||
if app_path:
|
||||
match_result = app_path in artifact_info["app_dir"]
|
||||
if config_name:
|
||||
match_result = match_result and config_name == artifact_info["config"]
|
||||
if target:
|
||||
match_result = match_result and target == artifact_info["target"]
|
||||
if match_result:
|
||||
ret = artifact_info
|
||||
break
|
||||
else:
|
||||
ret = None
|
||||
return ret
|
||||
|
||||
def download_artifact(self, artifact_index_file, app_path, config_name, target):
|
||||
# at least one of app_path or config_name is not None. otherwise we can't match artifact
|
||||
assert app_path or config_name
|
||||
assert os.path.exists(artifact_index_file)
|
||||
with open(artifact_index_file, "r") as f:
|
||||
artifact_index = json.load(f)
|
||||
|
||||
artifact_info = self._find_artifact(artifact_index, app_path, config_name, target)
|
||||
|
||||
if artifact_info:
|
||||
base_path = os.path.join(artifact_info["work_dir"], artifact_info["build_dir"])
|
||||
job_id = artifact_info["ci_job_id"]
|
||||
|
||||
# 1. download flash args file
|
||||
if artifact_info["build_system"] == "cmake":
|
||||
flash_arg_file = os.path.join(base_path, "flasher_args.json")
|
||||
else:
|
||||
flash_arg_file = os.path.join(base_path, "download.config")
|
||||
|
||||
self.gitlab_inst.download_artifact(job_id, [flash_arg_file], self.dest_root_path)
|
||||
|
||||
# 2. download all binary files
|
||||
flash_files, flash_settings = parse_flash_settings(os.path.join(self.dest_root_path, flash_arg_file))
|
||||
artifact_files = []
|
||||
for p in flash_files:
|
||||
artifact_files.append(os.path.join(base_path, p[1]))
|
||||
if not os.path.dirname(p[1]):
|
||||
# find app bin and also download elf
|
||||
elf_file = os.path.splitext(p[1])[0] + ".elf"
|
||||
artifact_files.append(os.path.join(base_path, elf_file))
|
||||
|
||||
self.gitlab_inst.download_artifact(job_id, artifact_files, self.dest_root_path)
|
||||
|
||||
# 3. download sdkconfig file
|
||||
self.gitlab_inst.download_artifact(job_id, [os.path.join(os.path.dirname(base_path), "sdkconfig")],
|
||||
self.dest_root_path)
|
||||
else:
|
||||
base_path = None
|
||||
return base_path
|
||||
|
||||
|
||||
class IDFApp(App.BaseApp):
|
||||
|
@ -34,7 +129,7 @@ class IDFApp(App.BaseApp):
|
|||
self.config_name = config_name
|
||||
self.target = target
|
||||
self.idf_path = self.get_sdk_path()
|
||||
self.binary_path = self.get_binary_path(app_path, config_name)
|
||||
self.binary_path = self.get_binary_path(app_path, config_name, target)
|
||||
self.elf_file = self._get_elf_file_path(self.binary_path)
|
||||
assert os.path.exists(self.binary_path)
|
||||
sdkconfig_dict = self.get_sdkconfig()
|
||||
|
@ -72,7 +167,6 @@ class IDFApp(App.BaseApp):
|
|||
"""
|
||||
reads sdkconfig and returns a dictionary with all configuredvariables
|
||||
|
||||
:param sdkconfig_file: location of sdkconfig
|
||||
:raise: AssertionError: if sdkconfig file does not exist in defined paths
|
||||
"""
|
||||
d = {}
|
||||
|
@ -89,14 +183,15 @@ class IDFApp(App.BaseApp):
|
|||
d[configs[0]] = configs[1].rstrip()
|
||||
return d
|
||||
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
"""
|
||||
get binary path according to input app_path.
|
||||
|
||||
subclass must overwrite this method.
|
||||
|
||||
:param app_path: path of application
|
||||
:param config_name: name of the application build config
|
||||
:param config_name: name of the application build config. Will match any config if None
|
||||
:param target: target name. Will match for target if None
|
||||
:return: abs app binary path
|
||||
"""
|
||||
pass
|
||||
|
@ -123,24 +218,12 @@ class IDFApp(App.BaseApp):
|
|||
|
||||
if self.IDF_FLASH_ARGS_FILE in os.listdir(self.binary_path):
|
||||
# CMake version using build metadata file
|
||||
with open(os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE), "r") as f:
|
||||
args = json.load(f)
|
||||
flash_files = [(offs,file) for (offs,file) in args["flash_files"].items() if offs != ""]
|
||||
flash_settings = args["flash_settings"]
|
||||
path = os.path.join(self.binary_path, self.IDF_FLASH_ARGS_FILE)
|
||||
else:
|
||||
# GNU Make version uses download.config arguments file
|
||||
with open(os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE), "r") as f:
|
||||
args = f.readlines()[-1].split(" ")
|
||||
flash_files = []
|
||||
flash_settings = {}
|
||||
for idx in range(0, len(args), 2): # process arguments in pairs
|
||||
if args[idx].startswith("--"):
|
||||
# strip the -- from the command line argument
|
||||
flash_settings[args[idx][2:]] = args[idx + 1]
|
||||
else:
|
||||
# offs, filename
|
||||
flash_files.append((args[idx], args[idx + 1]))
|
||||
path = os.path.join(self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE)
|
||||
|
||||
flash_files, flash_settings = parse_flash_settings(path)
|
||||
# The build metadata file does not currently have details, which files should be encrypted and which not.
|
||||
# Assume that all files should be encrypted if flash encryption is enabled in development mode.
|
||||
sdkconfig_dict = self.get_sdkconfig()
|
||||
|
@ -149,7 +232,7 @@ class IDFApp(App.BaseApp):
|
|||
# make file offsets into integers, make paths absolute
|
||||
flash_files = [(int(offs, 0), os.path.join(self.binary_path, path.strip())) for (offs, path) in flash_files]
|
||||
|
||||
return (flash_files, flash_settings)
|
||||
return flash_files, flash_settings
|
||||
|
||||
def _parse_partition_table(self):
|
||||
"""
|
||||
|
@ -209,7 +292,7 @@ class Example(IDFApp):
|
|||
"""
|
||||
return [os.path.join(self.binary_path, "..", "sdkconfig")]
|
||||
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
# build folder of example path
|
||||
path = os.path.join(self.idf_path, app_path, "build")
|
||||
if os.path.exists(path):
|
||||
|
@ -227,18 +310,23 @@ class Example(IDFApp):
|
|||
for dirpath in os.listdir(example_path):
|
||||
if os.path.basename(dirpath) == app_path_underscored:
|
||||
path = os.path.join(example_path, dirpath, config_name, self.target, "build")
|
||||
if os.path.exists(path):
|
||||
return path
|
||||
else:
|
||||
# app path exists, but config name not exists. try to download artifacts.
|
||||
break
|
||||
|
||||
artifacts = Artifacts(self.idf_path)
|
||||
path = artifacts.download_artifact(CIAssignExampleTest.ARTIFACT_INDEX_FILE,
|
||||
app_path, config_name, target)
|
||||
if path:
|
||||
return os.path.join(self.idf_path, path)
|
||||
else:
|
||||
raise OSError("Failed to find example binary")
|
||||
|
||||
|
||||
class UT(IDFApp):
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
"""
|
||||
:param app_path: app path
|
||||
:param config_name: config name
|
||||
:return: binary path
|
||||
"""
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
if not config_name:
|
||||
config_name = "default"
|
||||
|
||||
|
@ -262,12 +350,12 @@ class UT(IDFApp):
|
|||
|
||||
|
||||
class SSC(IDFApp):
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
# TODO: to implement SSC get binary path
|
||||
return app_path
|
||||
|
||||
|
||||
class AT(IDFApp):
|
||||
def get_binary_path(self, app_path, config_name=None):
|
||||
def get_binary_path(self, app_path, config_name=None, target=None):
|
||||
# TODO: to implement AT get binary path
|
||||
return app_path
|
||||
|
|
|
@ -16,7 +16,7 @@ import re
|
|||
|
||||
from tiny_test_fw import TinyFW, Utility
|
||||
from IDFApp import IDFApp, Example, UT
|
||||
from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT
|
||||
from IDFDUT import IDFDUT, ESP32DUT, ESP32S2DUT, ESP8266DUT # noqa: export DUTs for users
|
||||
|
||||
|
||||
def format_case_id(chip, case_name):
|
||||
|
|
Loading…
Reference in a new issue