CI: support test UT with different config by CI
This commit is contained in:
parent
3d150be4e1
commit
0a5b678e58
2 changed files with 189 additions and 24 deletions
129
.gitlab-ci.yml
129
.gitlab-ci.yml
|
@ -412,6 +412,9 @@ assign_test:
|
||||||
dependencies:
|
dependencies:
|
||||||
- build_esp_idf_tests
|
- build_esp_idf_tests
|
||||||
- build_ssc
|
- build_ssc
|
||||||
|
variables:
|
||||||
|
UT_BIN_PATH: "tools/unit-test-app/output"
|
||||||
|
OUTPUT_BIN_PATH: "test_bins/ESP32_IDF"
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- test_bins
|
- test_bins
|
||||||
|
@ -421,9 +424,10 @@ assign_test:
|
||||||
before_script: *add_gitlab_key_before
|
before_script: *add_gitlab_key_before
|
||||||
script:
|
script:
|
||||||
# first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
|
# first move test bins together: test_bins/CHIP_SDK/TestApp/bin_files
|
||||||
- mkdir -p test_bins/ESP32_IDF/UT
|
- mkdir -p $OUTPUT_BIN_PATH
|
||||||
- cp -r tools/unit-test-app/build/* test_bins/ESP32_IDF/UT
|
# copy and rename folder name to "UT_config"
|
||||||
- cp -r SSC/ssc_bin/* test_bins/ESP32_IDF
|
- for CONFIG in $(ls $UT_BIN_PATH); do cp -r "$UT_BIN_PATH/$CONFIG" "$OUTPUT_BIN_PATH/UT_$CONFIG"; done
|
||||||
|
- cp -r SSC/ssc_bin/* $OUTPUT_BIN_PATH
|
||||||
# clone test script to assign tests
|
# clone test script to assign tests
|
||||||
- git clone $TEST_SCRIPT_REPOSITORY
|
- git clone $TEST_SCRIPT_REPOSITORY
|
||||||
- cd auto_test_script
|
- cd auto_test_script
|
||||||
|
@ -508,42 +512,161 @@ UT_001_01:
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_1
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_02:
|
UT_001_02:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_1
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_03:
|
UT_001_03:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_1
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_04:
|
UT_001_04:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_1
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_05:
|
UT_001_05:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_SDMODE
|
- UT_T1_SDMODE
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_06:
|
UT_001_06:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_SPIMODE
|
- UT_T1_SPIMODE
|
||||||
|
- UT_default
|
||||||
|
|
||||||
UT_001_07:
|
UT_001_07:
|
||||||
<<: *unit_test_template
|
<<: *unit_test_template
|
||||||
tags:
|
tags:
|
||||||
- ESP32_IDF
|
- ESP32_IDF
|
||||||
- UT_T1_1
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
|
UT_001_08:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_default
|
||||||
|
|
||||||
|
UT_002_01:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_02:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_03:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_04:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_05:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_SDMODE
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_06:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_SPIMODE
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_07:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_002_08:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_release
|
||||||
|
|
||||||
|
UT_003_01:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_02:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_03:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_04:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_05:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_SDMODE
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_06:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_SPIMODE
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
|
UT_003_07:
|
||||||
|
<<: *unit_test_template
|
||||||
|
tags:
|
||||||
|
- ESP32_IDF
|
||||||
|
- UT_T1_1
|
||||||
|
- UT_single_core
|
||||||
|
|
||||||
IT_001_01:
|
IT_001_01:
|
||||||
<<: *test_template
|
<<: *test_template
|
||||||
|
|
|
@ -14,7 +14,6 @@ TEST_CASE_PATTERN = {
|
||||||
"SDK": "ESP32_IDF",
|
"SDK": "ESP32_IDF",
|
||||||
"level": "Unit",
|
"level": "Unit",
|
||||||
"execution time": 0,
|
"execution time": 0,
|
||||||
"Test App": "UT",
|
|
||||||
"auto test": "Yes",
|
"auto test": "Yes",
|
||||||
"category": "Function",
|
"category": "Function",
|
||||||
"test point 1": "basic function",
|
"test point 1": "basic function",
|
||||||
|
@ -36,20 +35,31 @@ class Parser(object):
|
||||||
TAG_PATTERN = re.compile("([^=]+)(=)?(.+)?")
|
TAG_PATTERN = re.compile("([^=]+)(=)?(.+)?")
|
||||||
DESCRIPTION_PATTERN = re.compile("\[([^]\[]+)\]")
|
DESCRIPTION_PATTERN = re.compile("\[([^]\[]+)\]")
|
||||||
|
|
||||||
|
# file path (relative to idf path)
|
||||||
|
TAG_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "TagDefinition.yml")
|
||||||
|
MODULE_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "ModuleDefinition.yml")
|
||||||
|
MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml")
|
||||||
|
TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml")
|
||||||
|
UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "builds")
|
||||||
|
ELF_FILE = "unit-test-app.elf"
|
||||||
|
APP_NAME_PREFIX = "UT_"
|
||||||
|
|
||||||
def __init__(self, idf_path=os.getenv("IDF_PATH")):
|
def __init__(self, idf_path=os.getenv("IDF_PATH")):
|
||||||
self.test_env_tags = {}
|
self.test_env_tags = {}
|
||||||
self.unit_jobs = {}
|
self.unit_jobs = {}
|
||||||
self.file_name_cache = {}
|
self.file_name_cache = {}
|
||||||
self.idf_path = idf_path
|
self.idf_path = idf_path
|
||||||
self.tag_def = yaml.load(open(os.path.join(idf_path, "tools", "unit-test-app", "tools",
|
self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), "r"))
|
||||||
"TagDefinition.yml"), "r"))
|
self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), "r"))
|
||||||
self.module_map = yaml.load(open(os.path.join(idf_path, "tools", "unit-test-app", "tools",
|
# used to check if duplicated test case names
|
||||||
"ModuleDefinition.yml"), "r"))
|
self.test_case_names = set()
|
||||||
|
self.parsing_errors = []
|
||||||
|
|
||||||
def parse_test_cases_from_elf(self, elf_file):
|
def parse_test_cases_from_elf(self, elf_file, app_name):
|
||||||
"""
|
"""
|
||||||
parse test cases from elf and save test cases to unit test folder
|
parse test cases from elf and save test cases need to be executed to unit test folder
|
||||||
:param elf_file: elf file path
|
:param elf_file: elf file path
|
||||||
|
:param app_name: built unit test app name
|
||||||
"""
|
"""
|
||||||
subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep \ test_desc > case_address.tmp'.format(elf_file),
|
subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep \ test_desc > case_address.tmp'.format(elf_file),
|
||||||
shell=True)
|
shell=True)
|
||||||
|
@ -71,19 +81,31 @@ class Parser(object):
|
||||||
desc = table.get_string("any", desc_addr)
|
desc = table.get_string("any", desc_addr)
|
||||||
file_name = table.get_string("any", file_name_addr)
|
file_name = table.get_string("any", file_name_addr)
|
||||||
|
|
||||||
tc = self.parse_one_test_case(name, desc, file_name)
|
tc = self.parse_one_test_case(name, desc, file_name, app_name)
|
||||||
|
|
||||||
|
# check if duplicated case names
|
||||||
|
# we need to use it to select case,
|
||||||
|
# if duplicated IDs, Unity could select incorrect case to run
|
||||||
|
# and we need to check all cases no matter if it's going te be executed by CI
|
||||||
|
# also add app_name here, we allow same case for different apps
|
||||||
|
if (tc["summary"] + app_name) in self.test_case_names:
|
||||||
|
self.parsing_errors.append("duplicated test case ID: " + tc["summary"])
|
||||||
|
else:
|
||||||
|
self.test_case_names.add(tc["summary"] + app_name)
|
||||||
|
|
||||||
if tc["CI ready"] == "Yes":
|
if tc["CI ready"] == "Yes":
|
||||||
# update test env list and the cases of same env list
|
# update test env list and the cases of same env list
|
||||||
if tc["test environment"] in self.test_env_tags:
|
if tc["test environment"] in self.test_env_tags:
|
||||||
self.test_env_tags[tc["test environment"]].append(tc["ID"])
|
self.test_env_tags[tc["test environment"]].append(tc["ID"])
|
||||||
else:
|
else:
|
||||||
self.test_env_tags.update({tc["test environment"]: [tc["ID"]]})
|
self.test_env_tags.update({tc["test environment"]: [tc["ID"]]})
|
||||||
|
# only add cases need to be executed
|
||||||
test_cases.append(tc)
|
test_cases.append(tc)
|
||||||
|
|
||||||
os.remove("section_table.tmp")
|
os.remove("section_table.tmp")
|
||||||
os.remove("case_address.tmp")
|
os.remove("case_address.tmp")
|
||||||
|
|
||||||
self.dump_test_cases(test_cases)
|
return test_cases
|
||||||
|
|
||||||
def parse_case_properities(self, tags_raw):
|
def parse_case_properities(self, tags_raw):
|
||||||
"""
|
"""
|
||||||
|
@ -123,12 +145,13 @@ class Parser(object):
|
||||||
pass
|
pass
|
||||||
return p
|
return p
|
||||||
|
|
||||||
def parse_one_test_case(self, name, description, file_name):
|
def parse_one_test_case(self, name, description, file_name, app_name):
|
||||||
"""
|
"""
|
||||||
parse one test case
|
parse one test case
|
||||||
:param name: test case name (summary)
|
:param name: test case name (summary)
|
||||||
:param description: test case description (tag string)
|
:param description: test case description (tag string)
|
||||||
:param file_name: the file defines this test case
|
:param file_name: the file defines this test case
|
||||||
|
:param app_name: built unit test app name
|
||||||
:return: parsed test case
|
:return: parsed test case
|
||||||
"""
|
"""
|
||||||
prop = self.parse_case_properities(description)
|
prop = self.parse_case_properities(description)
|
||||||
|
@ -149,8 +172,10 @@ class Parser(object):
|
||||||
self.module_map[prop["module"]]['sub module abbr'],
|
self.module_map[prop["module"]]['sub module abbr'],
|
||||||
file_name_hash,
|
file_name_hash,
|
||||||
self.file_name_cache[file_name_hash])
|
self.file_name_cache[file_name_hash])
|
||||||
|
|
||||||
test_case = deepcopy(TEST_CASE_PATTERN)
|
test_case = deepcopy(TEST_CASE_PATTERN)
|
||||||
test_case.update({"module": self.module_map[prop["module"]]['module'],
|
test_case.update({"Test App": self.APP_NAME_PREFIX + app_name,
|
||||||
|
"module": self.module_map[prop["module"]]['module'],
|
||||||
"CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
|
"CI ready": "No" if prop["ignore"] == "Yes" else "Yes",
|
||||||
"cmd set": ["IDFUnitTest/UnitTest", [name]],
|
"cmd set": ["IDFUnitTest/UnitTest", [name]],
|
||||||
"ID": tc_id,
|
"ID": tc_id,
|
||||||
|
@ -166,15 +191,28 @@ class Parser(object):
|
||||||
dump parsed test cases to YAML file for test bench input
|
dump parsed test cases to YAML file for test bench input
|
||||||
:param test_cases: parsed test cases
|
:param test_cases: parsed test cases
|
||||||
"""
|
"""
|
||||||
with open(os.path.join(self.idf_path, "components", "idf_test", "unit_test", "TestCaseAll.yml"), "wb+") as f:
|
with open(os.path.join(self.idf_path, self.TEST_CASE_FILE), "wb+") as f:
|
||||||
yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False)
|
yaml.dump({"test cases": test_cases}, f, allow_unicode=True, default_flow_style=False)
|
||||||
|
|
||||||
def copy_module_def_file(self):
|
def copy_module_def_file(self):
|
||||||
""" copy module def file to artifact path """
|
""" copy module def file to artifact path """
|
||||||
src = os.path.join(self.idf_path, "tools", "unit-test-app", "tools", "ModuleDefinition.yml")
|
src = os.path.join(self.idf_path, self.MODULE_DEF_FILE)
|
||||||
dst = os.path.join(self.idf_path, "components", "idf_test")
|
dst = os.path.join(self.idf_path, self.MODULE_ARTIFACT_FILE)
|
||||||
shutil.copy(src, dst)
|
shutil.copy(src, dst)
|
||||||
|
|
||||||
|
def parse_test_cases(self):
|
||||||
|
""" parse test cases from multiple built unit test apps """
|
||||||
|
test_cases = []
|
||||||
|
|
||||||
|
test_app_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER)
|
||||||
|
test_apps = os.listdir(test_app_folder)
|
||||||
|
for app in test_apps:
|
||||||
|
elf_file = os.path.join(test_app_folder, app, self.ELF_FILE)
|
||||||
|
if os.path.exists(elf_file):
|
||||||
|
test_cases.extend(self.parse_test_cases_from_elf(elf_file, app))
|
||||||
|
|
||||||
|
self.dump_test_cases(test_cases)
|
||||||
|
|
||||||
|
|
||||||
def test_parser():
|
def test_parser():
|
||||||
parser = Parser()
|
parser = Parser()
|
||||||
|
@ -210,12 +248,16 @@ def main():
|
||||||
test_parser()
|
test_parser()
|
||||||
|
|
||||||
idf_path = os.getenv("IDF_PATH")
|
idf_path = os.getenv("IDF_PATH")
|
||||||
elf_path = os.path.join(idf_path, "tools", "unit-test-app", "build", "unit-test-app.elf")
|
|
||||||
|
|
||||||
parser = Parser(idf_path)
|
parser = Parser(idf_path)
|
||||||
parser.parse_test_cases_from_elf(elf_path)
|
parser.parse_test_cases()
|
||||||
parser.copy_module_def_file()
|
parser.copy_module_def_file()
|
||||||
|
if len(parser.parsing_errors) > 0:
|
||||||
|
for error in parser.parsing_errors:
|
||||||
|
print error
|
||||||
|
exit(-1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue