diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7bff45e3..9c83055ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -662,287 +662,250 @@ UT_001_01: tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_02: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_03: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_04: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_05: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_SDMODE - - UT_default + - UT_T1_1 UT_001_06: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_SPIMODE - - UT_default + - UT_T1_1 UT_001_07: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_08: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_09: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default UT_001_10: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_default + +UT_001_11: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_12: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_13: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_14: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_15: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_16: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_17: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_18: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_19: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_20: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_21: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_22: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_23: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 + +UT_001_24: + <<: *unit_test_template + tags: + - ESP32_IDF + - UT_T1_1 UT_002_01: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_release + - UT_T1_SDMODE UT_002_02: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_release + - UT_T1_SDMODE UT_002_03: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release - -UT_002_04: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release - -UT_002_05: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_SDMODE - - UT_release - -UT_002_06: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_SPIMODE - - UT_release - -UT_002_07: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release - -UT_002_08: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release - -UT_002_09: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release - -UT_002_10: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_release UT_003_01: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_single_core + - UT_T1_SPIMODE UT_003_02: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_single_core + - UT_T1_SPIMODE UT_003_03: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core - -UT_003_04: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core - -UT_003_05: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_SDMODE - - UT_single_core - -UT_003_06: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_SPIMODE - - UT_single_core - -UT_003_07: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core - -UT_003_08: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core - -UT_003_09: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core - -UT_003_10: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_single_core UT_004_01: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram UT_004_02: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram UT_004_03: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram UT_004_04: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram UT_004_05: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_SDMODE - - UT_psram + - UT_T1_1 + - psram UT_004_06: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_SPIMODE - - UT_psram + - UT_T1_1 + - psram UT_004_07: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram UT_004_08: <<: *unit_test_template tags: - ESP32_IDF - UT_T1_1 - - UT_psram + - psram -UT_004_09: +UT_005_01: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_psram + - UT_T1_SDMODE + - psram -UT_004_10: +UT_005_02: <<: *unit_test_template tags: - ESP32_IDF - - UT_T1_1 - - UT_psram - -UT_004_11: - <<: *unit_test_template - tags: - - ESP32_IDF - - UT_T1_1 - - UT_psram + - UT_T1_SPIMODE + - psram IT_001_01: <<: *test_template diff --git a/tools/tiny-test-fw/CIAssignExampleTest.py b/tools/tiny-test-fw/CIAssignExampleTest.py index f13a803e4..3d9df0360 100644 --- a/tools/tiny-test-fw/CIAssignExampleTest.py +++ b/tools/tiny-test-fw/CIAssignExampleTest.py @@ -26,7 +26,11 @@ test_fw_path = os.getenv("TEST_FW_PATH") if test_fw_path: sys.path.insert(0, test_fw_path) -from Utility.CIAssignTest import AssignTest +from Utility.CIAssignTest import AssignTest, Group + + +class ExampleGroup(Group): + SORT_KEYS = CI_JOB_MATCH_KEYS = ["env_tag", "chip"] class CIExampleAssignTest(AssignTest): @@ -43,6 +47,6 @@ if __name__ == '__main__': help="output path of config files") args = parser.parse_args() - assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file) + assign_test = CIExampleAssignTest(args.test_case, args.ci_config_file, case_group=ExampleGroup) assign_test.assign_cases() assign_test.output_configs(args.output_path) diff --git a/tools/tiny-test-fw/CIAssignUnitTest.py b/tools/tiny-test-fw/CIAssignUnitTest.py index 3c3151458..9885d4fd7 100644 --- a/tools/tiny-test-fw/CIAssignUnitTest.py +++ b/tools/tiny-test-fw/CIAssignUnitTest.py @@ -17,12 +17,18 @@ from Utility import CIAssignTest class Group(CIAssignTest.Group): - SORT_KEYS = ["Test App", "SDK", "test environment", "multi_device", "multi_stage"] - CI_JOB_MATCH_KEYS = ["Test App", "SDK", "test environment"] + SORT_KEYS = ["config", "SDK", "test environment", "multi_device", "multi_stage", "tags"] MAX_CASE = 30 ATTR_CONVERT_TABLE = { "execution_time": "execution time" } + # when IDF support multiple chips, SDK will be moved into tags, we can remove it + CI_JOB_MATCH_KEYS = ["test environment", "SDK"] + + def __init__(self, case): + super(Group, self).__init__(case) + for tag in self._get_case_attr(case, "tags"): + self.ci_job_match_keys.add(tag) @staticmethod def _get_case_attr(case, attr): @@ -30,13 +36,6 @@ class Group(CIAssignTest.Group): attr = Group.ATTR_CONVERT_TABLE[attr] return case[attr] - @staticmethod - def _get_ut_config(test_app): - # we format test app "UT_ + config" when parsing test cases - # now we need to extract config - assert test_app[:3] == "UT_" - return test_app[3:] - def _create_extra_data(self, test_function): """ For unit test case, we need to copy some attributes of test cases into config file. @@ -45,7 +44,7 @@ class Group(CIAssignTest.Group): case_data = [] for case in self.case_list: one_case_data = { - "config": self._get_ut_config(self._get_case_attr(case, "Test App")), + "config": self._get_case_attr(case, "config"), "name": self._get_case_attr(case, "summary"), "reset": self._get_case_attr(case, "reset"), "timeout": self._get_case_attr(case, "timeout"), @@ -101,8 +100,7 @@ class UnitTestAssignTest(CIAssignTest.AssignTest): def __init__(self, test_case_path, ci_config_file): CIAssignTest.AssignTest.__init__(self, test_case_path, ci_config_file, case_group=Group) - @staticmethod - def _search_cases(test_case_path, case_filter=None): + def _search_cases(self, test_case_path, case_filter=None): """ For unit test case, we don't search for test functions. The unit test cases is stored in a yaml file which is created in job build-idf-test. diff --git a/tools/tiny-test-fw/Utility/CIAssignTest.py b/tools/tiny-test-fw/Utility/CIAssignTest.py index 3e570cc31..ca0204aab 100644 --- a/tools/tiny-test-fw/Utility/CIAssignTest.py +++ b/tools/tiny-test-fw/Utility/CIAssignTest.py @@ -63,8 +63,8 @@ class Group(object): self.execution_time = 0 self.case_list = [case] self.filters = dict(zip(self.SORT_KEYS, [self._get_case_attr(case, x) for x in self.SORT_KEYS])) - self.ci_job_match_keys = dict(zip(self.CI_JOB_MATCH_KEYS, - [self._get_case_attr(case, x) for x in self.CI_JOB_MATCH_KEYS])) + # we use ci_job_match_keys to match CI job tags. It's a set of required tags. + self.ci_job_match_keys = set([self._get_case_attr(case, x) for x in self.CI_JOB_MATCH_KEYS]) @staticmethod def _get_case_attr(case, attr): diff --git a/tools/tiny-test-fw/Utility/GitlabCIJob.py b/tools/tiny-test-fw/Utility/GitlabCIJob.py index 79676cf00..e6173b035 100644 --- a/tools/tiny-test-fw/Utility/GitlabCIJob.py +++ b/tools/tiny-test-fw/Utility/GitlabCIJob.py @@ -27,6 +27,7 @@ class Job(dict): def __init__(self, job, job_name): super(Job, self).__init__(job) self["name"] = job_name + self.tags = set(self["tags"]) def match_group(self, group): """ @@ -37,17 +38,8 @@ class Job(dict): :return: True or False """ match_result = False - for _ in range(1): - if "case group" in self: - # this job is already assigned - break - for value in group.ci_job_match_keys.values(): - if value not in self["tags"]: - break - else: - continue - break - else: + if "case group" not in self and group.ci_job_match_keys == self.tags: + # group not assigned and all tags match match_result = True return match_result diff --git a/tools/unit-test-app/tools/ConfigDependency.yml b/tools/unit-test-app/tools/ConfigDependency.yml new file mode 100644 index 000000000..3af3e565b --- /dev/null +++ b/tools/unit-test-app/tools/ConfigDependency.yml @@ -0,0 +1 @@ +"psram": "CONFIG_SPIRAM_SUPPORT=y" diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py index 688e4ce37..1898bf659 100644 --- a/tools/unit-test-app/tools/UnitTestParser.py +++ b/tools/unit-test-app/tools/UnitTestParser.py @@ -20,13 +20,8 @@ TEST_CASE_PATTERN = { "test environment": "UT_T1_1", "reset": "", "expected result": "1. set succeed", - "cmd set": "test_unit_test_case" -} - -CONFIG_FILE_PATTERN = { - "Config": {"execute count": 1, "execute order": "in order"}, - "DUT": [], - "Filter": [{"Add": {"ID": []}}] + "cmd set": "test_unit_test_case", + "Test App": "UT", } @@ -39,11 +34,12 @@ class Parser(object): # file path (relative to idf path) TAG_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "TagDefinition.yml") MODULE_DEF_FILE = os.path.join("tools", "unit-test-app", "tools", "ModuleDefinition.yml") + CONFIG_DEPENDENCY_FILE = os.path.join("tools", "unit-test-app", "tools", "ConfigDependency.yml") MODULE_ARTIFACT_FILE = os.path.join("components", "idf_test", "ModuleDefinition.yml") TEST_CASE_FILE = os.path.join("components", "idf_test", "unit_test", "TestCaseAll.yml") - UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "builds") + UT_BIN_FOLDER = os.path.join("tools", "unit-test-app", "output") ELF_FILE = "unit-test-app.elf" - APP_NAME_PREFIX = "UT_" + SDKCONFIG_FILE = "sdkconfig" def __init__(self, idf_path=os.getenv("IDF_PATH")): self.test_env_tags = {} @@ -52,21 +48,24 @@ class Parser(object): self.idf_path = idf_path self.tag_def = yaml.load(open(os.path.join(idf_path, self.TAG_DEF_FILE), "r")) self.module_map = yaml.load(open(os.path.join(idf_path, self.MODULE_DEF_FILE), "r")) + self.config_dependency = yaml.load(open(os.path.join(idf_path, self.CONFIG_DEPENDENCY_FILE), "r")) # used to check if duplicated test case names self.test_case_names = set() self.parsing_errors = [] - def parse_test_cases_from_elf(self, elf_file, app_name): + def parse_test_cases_for_one_config(self, config_output_folder, config_name): """ parse test cases from elf and save test cases need to be executed to unit test folder - :param elf_file: elf file path - :param app_name: built unit test app name + :param config_output_folder: build folder of this config + :param config_name: built unit test config name """ + elf_file = os.path.join(config_output_folder, self.ELF_FILE) subprocess.check_output('xtensa-esp32-elf-objdump -t {} | grep test_desc > case_address.tmp'.format(elf_file), shell=True) subprocess.check_output('xtensa-esp32-elf-objdump -s {} > section_table.tmp'.format(elf_file), shell=True) table = CreateSectionTable.SectionTable("section_table.tmp") + tags = self.parse_tags(os.path.join(config_output_folder, self.SDKCONFIG_FILE)) test_cases = [] with open("case_address.tmp", "r") as f: for line in f: @@ -82,17 +81,17 @@ class Parser(object): name = table.get_string("any", name_addr) desc = table.get_string("any", desc_addr) file_name = table.get_string("any", file_name_addr) - tc = self.parse_one_test_case(name, desc, file_name, app_name) + tc = self.parse_one_test_case(name, desc, file_name, config_name, tags) # check if duplicated case names # we need to use it to select case, # if duplicated IDs, Unity could select incorrect case to run # and we need to check all cases no matter if it's going te be executed by CI # also add app_name here, we allow same case for different apps - if (tc["summary"] + app_name) in self.test_case_names: + if (tc["summary"] + config_name) in self.test_case_names: self.parsing_errors.append("duplicated test case ID: " + tc["summary"]) else: - self.test_case_names.add(tc["summary"] + app_name) + self.test_case_names.add(tc["summary"] + config_name) if tc["CI ready"] == "Yes": # update test env list and the cases of same env list @@ -150,13 +149,32 @@ class Parser(object): pass return p - def parse_one_test_case(self, name, description, file_name, app_name): + def parse_tags(self, sdkconfig_file): + """ + Some test configs could requires different DUTs. + For example, if CONFIG_SPIRAM_SUPPORT is enabled, we need WROVER-Kit to run test. + This method will get tags for runners according to ConfigDependency.yml(maps tags to sdkconfig). + + :param sdkconfig_file: sdkconfig file of the unit test config + :return: required tags for runners + """ + required_tags = [] + with open(sdkconfig_file, "r") as f: + configs_raw_data = f.read() + configs = configs_raw_data.splitlines(False) + for tag in self.config_dependency: + if self.config_dependency[tag] in configs: + required_tags.append(tag) + return required_tags + + def parse_one_test_case(self, name, description, file_name, config_name, tags): """ parse one test case :param name: test case name (summary) :param description: test case description (tag string) :param file_name: the file defines this test case - :param app_name: built unit test app name + :param config_name: built unit test app name + :param tags: tags to select runners :return: parsed test case """ prop = self.parse_case_properities(description) @@ -179,7 +197,7 @@ class Parser(object): self.file_name_cache[file_name_hash]) test_case = deepcopy(TEST_CASE_PATTERN) - test_case.update({"Test App": self.APP_NAME_PREFIX + app_name, + test_case.update({"config": config_name, "module": self.module_map[prop["module"]]['module'], "CI ready": "No" if prop["ignore"] == "Yes" else "Yes", "ID": tc_id, @@ -191,7 +209,8 @@ class Parser(object): "summary": name, "multi_device": prop["multi_device"], "multi_stage": prop["multi_stage"], - "timeout": int(prop["timeout"])}) + "timeout": int(prop["timeout"]), + "tags": tags}) return test_case def dump_test_cases(self, test_cases): @@ -212,12 +231,12 @@ class Parser(object): """ parse test cases from multiple built unit test apps """ test_cases = [] - test_app_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER) - test_apps = os.listdir(test_app_folder) - for app in test_apps: - elf_file = os.path.join(test_app_folder, app, self.ELF_FILE) - if os.path.exists(elf_file): - test_cases.extend(self.parse_test_cases_from_elf(elf_file, app)) + output_folder = os.path.join(self.idf_path, self.UT_BIN_FOLDER) + test_configs = os.listdir(output_folder) + for config in test_configs: + config_output_folder = os.path.join(output_folder, config) + if os.path.exists(config_output_folder): + test_cases.extend(self.parse_test_cases_for_one_config(config_output_folder, config)) self.dump_test_cases(test_cases) diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index e049e7505..b7453722e 100644 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -131,6 +131,7 @@ def run_unit_test_cases(env, extra_data): failed_cases = [] for ut_config in case_config: + Utility.console_log("Running unit test for config: " + ut_config, "O") dut = env.get_dut("unit-test-app", app_path=ut_config) dut.start_app() @@ -365,6 +366,7 @@ def run_multiple_devices_cases(env, extra_data): case_config = format_test_case_config(extra_data) DUTS = {} for ut_config in case_config: + Utility.console_log("Running unit test for config: " + ut_config, "O") for one_case in case_config[ut_config]: case_run(DUTS, ut_config, env, one_case, failed_cases) @@ -396,6 +398,7 @@ def run_multiple_stage_cases(env, extra_data): failed_cases = [] for ut_config in case_config: + Utility.console_log("Running unit test for config: " + ut_config, "O") dut = env.get_dut("unit-test-app", app_path=ut_config) dut.start_app()