Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

ci: optimize unit test assignment:

Browse source 
Currently we use config and test function as filter when assign cases to one CI job. It's not necessary as the runner can run test with different configs / test functions. Now we will try to assign as many cases to a job as possible, to reduce the amount of jobs required.
This commit is contained in:
He Yin Ling 2019-11-21 17:33:32 +08:00
parent 74ca3fc571
commit b12a8bce5f
1 changed files with 30 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
tools/tiny-test-fw/CIAssignUnitTest.py
View file

@ -24,8 +24,8 @@ except ImportError:
class Group(CIAssignTest.Group):
SORT_KEYS = ["config", "test environment", "multi_device", "multi_stage", "tags", "chip_target"]
MAX_CASE = 30
SORT_KEYS = ["test environment", "tags", "chip_target"]
MAX_CASE = 50
ATTR_CONVERT_TABLE = {
"execution_time": "execution time"
}
@ -47,13 +47,13 @@ class Group(CIAssignTest.Group):
attr = Group.ATTR_CONVERT_TABLE[attr]
return case[attr]
def _create_extra_data(self, test_function):
def _create_extra_data(self, test_cases, test_function):
"""
For unit test case, we need to copy some attributes of test cases into config file.
So unit test function knows how to run the case.
"""
case_data = []
for case in self.case_list:
for case in test_cases:
one_case_data = {
"config": self._get_case_attr(case, "config"),
"name": self._get_case_attr(case, "summary"),
@ -72,19 +72,26 @@ class Group(CIAssignTest.Group):
case_data.append(one_case_data)
return case_data
def _map_test_function(self):
def _divide_case_by_test_function(self):
"""
determine which test function to use according to current test case
divide cases of current test group by test function they need to use
:return: test function name to use
:return: dict of list of cases for each test functions
"""
if self.filters["multi_device"] == "Yes":
test_function = "run_multiple_devices_cases"
elif self.filters["multi_stage"] == "Yes":
test_function = "run_multiple_stage_cases"
else:
test_function = "run_unit_test_cases"
return test_function
case_by_test_function = {
"run_multiple_devices_cases": [],
"run_multiple_stage_cases": [],
"run_unit_test_cases": [],
}
for case in self.case_list:
if case["multi_device"] == "Yes":
case_by_test_function["run_multiple_devices_cases"].append(case)
elif case["multi_stage"] == "Yes":
case_by_test_function["run_multiple_stage_cases"].append(case)
else:
case_by_test_function["run_unit_test_cases"].append(case)
return case_by_test_function
def output(self):
"""
@ -92,7 +99,6 @@ class Group(CIAssignTest.Group):
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
"""
test_function = self._map_test_function()
target = self._get_case_attr(self.case_list[0], "chip_target")
if target:
@ -104,14 +110,17 @@ class Group(CIAssignTest.Group):
}
else:
overwrite = dict()
case_by_test_function = self._divide_case_by_test_function()
output_data = {
# we don't need filter for test function, as UT uses a few test functions for all cases
"CaseConfig": [
{
"name": test_function,
"extra_data": self._create_extra_data(test_function),
"extra_data": self._create_extra_data(test_cases, test_function),
"overwrite": overwrite,
}
} for test_function, test_cases in case_by_test_function.iteritems() if test_cases
],
}
return output_data
@ -158,6 +167,10 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
# case don't have this key, regard as filter success
filtered_cases.append(case)
test_cases = filtered_cases
# sort cases with configs and test functions
# in later stage cases with similar attributes are more likely to be assigned to the same job
# it will reduce the count of flash DUT operations
test_cases.sort(key=lambda x: x["config"] + x["multi_stage"] + x["multi_device"])
return test_cases