Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

tiny-test-fw: print useful debug info when assign test:

Browse source 
* total requirements (number of required jobs vs tags)
* unused CI jobs
* missing jobs (number of missing jobs vs tags)
This commit is contained in:
He Yin Ling 2019-11-21 17:43:04 +08:00
parent b12a8bce5f
commit 757e2d021e
1 changed files with 38 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
tools/tiny-test-fw/Utility/CIAssignTest.py
View file

@ -218,6 +218,21 @@ class AssignTest(object):
test_count = int(test_count) test_count = int(test_count)
self.test_cases *= test_count self.test_cases *= test_count
@staticmethod
def _count_groups_by_keys(test_groups):
"""
Count the number of test groups by job match keys.
It's an important information to update CI config file.
"""
group_count = dict()
for group in test_groups:
key = ",".join(group.ci_job_match_keys)
try:
group_count[key] += 1
except KeyError:
group_count[key] = 1
return group_count
def assign_cases(self): def assign_cases(self):
""" """
separate test cases to groups and assign test cases to CI jobs. separate test cases to groups and assign test cases to CI jobs.
@ -230,6 +245,7 @@ class AssignTest(object):
self.test_cases = self._search_cases(self.test_case_path, case_filter) self.test_cases = self._search_cases(self.test_case_path, case_filter)
self._apply_bot_test_count() self._apply_bot_test_count()
test_groups = self._group_cases() test_groups = self._group_cases()
for group in test_groups: for group in test_groups:
for job in self.jobs: for job in self.jobs:
if job.match_group(group): if job.match_group(group):
@ -237,10 +253,29 @@ class AssignTest(object):
break break
else: else:
failed_to_assign.append(group) failed_to_assign.append(group)
# print debug info
# total requirement of current pipeline
required_group_count = self._count_groups_by_keys(test_groups)
console_log("Required job count by tags:")
for tags in required_group_count:
console_log("\t{}: {}".format(tags, required_group_count[tags]))
# number of unused jobs
not_used_jobs = [job for job in self.jobs if "case group" not in job]
if not_used_jobs:
console_log("{} jobs not used. Please check if you define too much jobs".format(len(not_used_jobs)), "O")
for job in not_used_jobs:
console_log("\t{}".format(job["name"]), "O")
# failures
if failed_to_assign: if failed_to_assign:
console_log("Too many test cases vs jobs to run. Please add the following jobs to tools/ci/config/target-test.yml with specific tags:", "R") console_log("Too many test cases vs jobs to run. "
for group in failed_to_assign: "Please increase parallel count in tools/ci/config/target-test.yml "
console_log("* Add job with: " + ",".join(group.ci_job_match_keys), "R") "for jobs with specific tags:", "R")
failed_group_count = self._count_groups_by_keys(failed_to_assign)
for tags in failed_group_count:
console_log("\t{}: {}".format(tags, failed_group_count[tags]), "R")
raise RuntimeError("Failed to assign test case to CI jobs") raise RuntimeError("Failed to assign test case to CI jobs")
def output_configs(self, output_path): def output_configs(self, output_path):