Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

test: collect unit-test performance items and save to report

Browse source
This commit is contained in:
He Yin Ling 2019-03-07 20:18:32 +08:00
parent 0462620a23
commit b75400294a
2 changed files with 18 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tools/tiny-test-fw/IDF/IDFDUT.py
View file

@ -65,14 +65,14 @@ class IDFRecvThread(DUT.RecvThread):
def __init__(self, read, dut): def __init__(self, read, dut):
super(IDFRecvThread, self).__init__(read, dut) super(IDFRecvThread, self).__init__(read, dut)
self.exceptions = _queue.Queue() self.exceptions = _queue.Queue()
self.performances = _queue.Queue() self.performance_items = _queue.Queue()
def collect_performance(self, comp_data): def collect_performance(self, comp_data):
matches = self.PERFORMANCE_PATTERN.findall(comp_data) matches = self.PERFORMANCE_PATTERN.findall(comp_data)
for match in matches: for match in matches:
Utility.console_log("[Performance][{}]: {}".format(match[0], match[1]), Utility.console_log("[Performance][{}]: {}".format(match[0], match[1]),
color="orange") color="orange")
self.performances.put((match[0], match[1])) self.performance_items.put((match[0], match[1]))
def detect_exception(self, comp_data): def detect_exception(self, comp_data):
for pattern in self.EXCEPTION_PATTERNS: for pattern in self.EXCEPTION_PATTERNS:
@ -157,7 +157,7 @@ class IDFDUT(DUT.SerialDUT):
super(IDFDUT, self).__init__(name, port, log_file, app, **kwargs) super(IDFDUT, self).__init__(name, port, log_file, app, **kwargs)
self.allow_dut_exception = allow_dut_exception self.allow_dut_exception = allow_dut_exception
self.exceptions = _queue.Queue() self.exceptions = _queue.Queue()
self.performances = _queue.Queue() self.performance_items = _queue.Queue()
@classmethod @classmethod
def get_mac(cls, app, port): def get_mac(cls, app, port):
@ -365,7 +365,7 @@ class IDFDUT(DUT.SerialDUT):
def stop_receive(self): def stop_receive(self):
if self.receive_thread: if self.receive_thread:
for name in ["performances", "exceptions"]: for name in ["performance_items", "exceptions"]:
source_queue = getattr(self.receive_thread, name) source_queue = getattr(self.receive_thread, name)
dest_queue = getattr(self, name) dest_queue = getattr(self, name)
self._queue_copy(source_queue, dest_queue) self._queue_copy(source_queue, dest_queue)
@ -382,7 +382,7 @@ class IDFDUT(DUT.SerialDUT):
:return: a list of performance items. :return: a list of performance items.
""" """
return self._get_from_queue("performances") return self._get_from_queue("performance_items")
def close(self): def close(self):
super(IDFDUT, self).close() super(IDFDUT, self).close()

13
tools/unit-test-app/unit_test.py
View file

@ -285,16 +285,19 @@ def run_unit_test_cases(env, extra_data):
Utility.console_log("Download finished, start running test cases", "O") Utility.console_log("Download finished, start running test cases", "O")
for one_case in case_config[ut_config]: for one_case in case_config[ut_config]:
performance_items = []
# create junit report test case # create junit report test case
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try: try:
run_one_normal_case(dut, one_case, junit_test_case) run_one_normal_case(dut, one_case, junit_test_case)
performance_items = dut.get_performance_items()
except TestCaseFailed: except TestCaseFailed:
failed_cases.append(one_case["name"]) failed_cases.append(one_case["name"])
except Exception as e: except Exception as e:
junit_test_case.add_failure_info("Unexpected exception: " + str(e)) junit_test_case.add_failure_info("Unexpected exception: " + str(e))
failed_cases.append(one_case["name"]) failed_cases.append(one_case["name"])
finally: finally:
TinyFW.JunitReport.update_performance(performance_items)
TinyFW.JunitReport.test_case_finish(junit_test_case) TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails # raise exception if any case fails
@ -445,6 +448,13 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, app_bin, junit
if not result: if not result:
junit_test_case.add_failure_info(output) junit_test_case.add_failure_info(output)
# collect performances from DUTs
performance_items = []
for dut_name in duts:
performance_items.extend(duts[dut_name].get_performance_items())
TinyFW.JunitReport.update_performance(performance_items)
return result return result
@ -628,15 +638,18 @@ def run_multiple_stage_cases(env, extra_data):
dut.start_app() dut.start_app()
for one_case in case_config[ut_config]: for one_case in case_config[ut_config]:
performance_items = []
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try: try:
run_one_multiple_stage_case(dut, one_case, junit_test_case) run_one_multiple_stage_case(dut, one_case, junit_test_case)
performance_items = dut.get_performance_items()
except TestCaseFailed: except TestCaseFailed:
failed_cases.append(one_case["name"]) failed_cases.append(one_case["name"])
except Exception as e: except Exception as e:
junit_test_case.add_failure_info("Unexpected exception: " + str(e)) junit_test_case.add_failure_info("Unexpected exception: " + str(e))
failed_cases.append(one_case["name"]) failed_cases.append(one_case["name"])
finally: finally:
TinyFW.JunitReport.update_performance(performance_items)
TinyFW.JunitReport.test_case_finish(junit_test_case) TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails # raise exception if any case fails