Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

ci: optimize unit test assignment:

Browse source 
Currently we use config and test function as filter when assign cases to one CI job. It's not necessary as the runner can run test with different configs / test functions. Now we will try to assign as many cases to a job as possible, to reduce the amount of jobs required.
This commit is contained in:
He Yin Ling 2019-11-21 17:33:32 +08:00 committed by Angus Gratton
parent 9bf656e12e
commit 6ee7492512
1 changed files with 30 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
tools/tiny-test-fw/CIAssignUnitTest.py
View file

@ -19,8 +19,8 @@ except ImportError:
class Group(CIAssignTest.Group): class Group(CIAssignTest.Group):
SORT_KEYS = ["config", "SDK", "test environment", "multi_device", "multi_stage", "tags"] SORT_KEYS = ["test environment", "tags"]
MAX_CASE = 30 MAX_CASE = 50
ATTR_CONVERT_TABLE = { ATTR_CONVERT_TABLE = {
"execution_time": "execution time" "execution_time": "execution time"
} }
@ -38,13 +38,13 @@ class Group(CIAssignTest.Group):
attr = Group.ATTR_CONVERT_TABLE[attr] attr = Group.ATTR_CONVERT_TABLE[attr]
return case[attr] return case[attr]
def _create_extra_data(self, test_function): def _create_extra_data(self, test_cases, test_function):
""" """
For unit test case, we need to copy some attributes of test cases into config file. For unit test case, we need to copy some attributes of test cases into config file.
So unit test function knows how to run the case. So unit test function knows how to run the case.
""" """
case_data = [] case_data = []
for case in self.case_list: for case in test_cases:
one_case_data = { one_case_data = {
"config": self._get_case_attr(case, "config"), "config": self._get_case_attr(case, "config"),
"name": self._get_case_attr(case, "summary"), "name": self._get_case_attr(case, "summary"),
@ -63,19 +63,26 @@ class Group(CIAssignTest.Group):
case_data.append(one_case_data) case_data.append(one_case_data)
return case_data return case_data
def _map_test_function(self): def _divide_case_by_test_function(self):
""" """
determine which test function to use according to current test case divide cases of current test group by test function they need to use
:return: test function name to use :return: dict of list of cases for each test functions
""" """
if self.filters["multi_device"] == "Yes": case_by_test_function = {
test_function = "run_multiple_devices_cases" "run_multiple_devices_cases": [],
elif self.filters["multi_stage"] == "Yes": "run_multiple_stage_cases": [],
test_function = "run_multiple_stage_cases" "run_unit_test_cases": [],
else: }
test_function = "run_unit_test_cases"
return test_function for case in self.case_list:
if case["multi_device"] == "Yes":
case_by_test_function["run_multiple_devices_cases"].append(case)
elif case["multi_stage"] == "Yes":
case_by_test_function["run_multiple_stage_cases"].append(case)
else:
case_by_test_function["run_unit_test_cases"].append(case)
return case_by_test_function
def output(self): def output(self):
""" """
@ -83,15 +90,16 @@ class Group(CIAssignTest.Group):
:return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group} :return: {"Filter": case filter, "CaseConfig": list of case configs for cases in this group}
""" """
test_function = self._map_test_function() case_by_test_function = self._divide_case_by_test_function()
output_data = { output_data = {
# we don't need filter for test function, as UT uses a few test functions for all cases # we don't need filter for test function, as UT uses a few test functions for all cases
"CaseConfig": [ "CaseConfig": [
{ {
"name": test_function, "name": test_function,
"extra_data": self._create_extra_data(test_function), "extra_data": self._create_extra_data(test_cases, test_function),
} } for test_function, test_cases in case_by_test_function.iteritems() if test_cases
] ],
} }
return output_data return output_data
@ -137,6 +145,10 @@ class UnitTestAssignTest(CIAssignTest.AssignTest):
# case don't have this key, regard as filter success # case don't have this key, regard as filter success
filtered_cases.append(case) filtered_cases.append(case)
test_cases = filtered_cases test_cases = filtered_cases
# sort cases with configs and test functions
# in later stage cases with similar attributes are more likely to be assigned to the same job
# it will reduce the count of flash DUT operations
test_cases.sort(key=lambda x: x["config"] + x["multi_stage"] + x["multi_device"])
return test_cases return test_cases