test: generate junit test report according to executed cases
This commit is contained in:
He Yin Ling
parent
ab2062aa45
commit
f298897fb5
2 changed files with 245 additions and 178 deletions
|
|
@ -81,7 +81,7 @@ def log_performance(item, value):
|
|||
Utility.console_log(performance_msg, "orange")
|
||||
# update to junit test report
|
||||
current_junit_case = TinyFW.JunitReport.get_current_test_case()
|
||||
current_junit_case.stdout += performance_msg
|
||||
current_junit_case.stdout += performance_msg + "\r\n"
|
||||
|
||||
|
||||
def check_performance(item, value):
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import re
|
|||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import threading
|
||||
|
||||
# if we want to run test case outside `tiny-test-fw` folder,
|
||||
|
|
@ -105,7 +104,112 @@ def format_test_case_config(test_case_data):
|
|||
return case_config
|
||||
|
||||
|
||||
@IDF.idf_unit_test(env_tag="UT_T1_1")
|
||||
def replace_app_bin(dut, name, new_app_bin):
|
||||
if new_app_bin is None:
|
||||
return
|
||||
search_pattern = '/{}.bin'.format(name)
|
||||
for i, config in enumerate(dut.download_config):
|
||||
if config.endswith(search_pattern):
|
||||
dut.download_config[i] = new_app_bin
|
||||
Utility.console_log("The replaced application binary is {}".format(new_app_bin), "O")
|
||||
break
|
||||
|
||||
|
||||
def reset_dut(dut):
|
||||
dut.reset()
|
||||
# esptool ``run`` cmd takes quite long time.
|
||||
# before reset finish, serial port is closed. therefore DUT could already bootup before serial port opened.
|
||||
# this could cause checking bootup print failed.
|
||||
# now use input cmd `-` and check test history to check if DUT is bootup.
|
||||
# we'll retry this step for a few times in case `dut.reset` returns during DUT bootup (when DUT can't process any command).
|
||||
for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
|
||||
dut.write("-")
|
||||
try:
|
||||
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
|
||||
break
|
||||
except ExpectTimeout:
|
||||
pass
|
||||
else:
|
||||
raise AssertationError("Reset {} ({}) failed!".format(dut.name, dut.port))
|
||||
|
||||
|
||||
def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
|
||||
|
||||
reset_dut(dut)
|
||||
|
||||
dut.start_capture_raw_data()
|
||||
# run test case
|
||||
dut.write("\"{}\"".format(one_case["name"]))
|
||||
dut.expect("Running " + one_case["name"] + "...")
|
||||
|
||||
exception_reset_list = []
|
||||
|
||||
# we want to set this flag in callbacks (inner functions)
|
||||
# use list here so we can use append to set this flag
|
||||
test_finish = list()
|
||||
|
||||
# expect callbacks
|
||||
def one_case_finish(result):
|
||||
""" one test finished, let expect loop break and log result """
|
||||
test_finish.append(True)
|
||||
output = dut.stop_capture_raw_data()
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
junit_test_case.add_failure_info(output)
|
||||
|
||||
def handle_exception_reset(data):
|
||||
"""
|
||||
just append data to exception list.
|
||||
exception list will be checked in ``handle_reset_finish``, once reset finished.
|
||||
"""
|
||||
exception_reset_list.append(data[0])
|
||||
|
||||
def handle_test_finish(data):
|
||||
""" test finished without reset """
|
||||
# in this scenario reset should not happen
|
||||
assert not exception_reset_list
|
||||
if int(data[1]):
|
||||
# case ignored
|
||||
Utility.console_log("Ignored: " + one_case["name"], color="orange")
|
||||
junit_test_case.add_skipped_info("ignored")
|
||||
one_case_finish(not int(data[0]))
|
||||
|
||||
def handle_reset_finish(data):
|
||||
""" reset happened and reboot finished """
|
||||
assert exception_reset_list # reboot but no exception/reset logged. should never happen
|
||||
result = False
|
||||
if len(one_case["reset"]) == len(exception_reset_list):
|
||||
for i, exception in enumerate(exception_reset_list):
|
||||
if one_case["reset"][i] not in exception:
|
||||
break
|
||||
else:
|
||||
result = True
|
||||
if not result:
|
||||
err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
|
||||
exception_reset_list)
|
||||
Utility.console_log(err_msg, color="orange")
|
||||
junit_test_case.add_error_info(err_msg)
|
||||
one_case_finish(result)
|
||||
|
||||
while not test_finish:
|
||||
try:
|
||||
dut.expect_any((RESET_PATTERN, handle_exception_reset),
|
||||
(EXCEPTION_PATTERN, handle_exception_reset),
|
||||
(ABORT_PATTERN, handle_exception_reset),
|
||||
(FINISH_PATTERN, handle_test_finish),
|
||||
(UT_APP_BOOT_UP_DONE, handle_reset_finish),
|
||||
timeout=one_case["timeout"])
|
||||
except ExpectTimeout:
|
||||
Utility.console_log("Timeout in expect", color="orange")
|
||||
junit_test_case.add_error_info("timeout")
|
||||
one_case_finish(False)
|
||||
break
|
||||
|
||||
|
||||
@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
|
||||
def run_unit_test_cases(env, extra_data):
|
||||
"""
|
||||
extra_data can be three types of value
|
||||
|
|
@ -133,82 +237,17 @@ def run_unit_test_cases(env, extra_data):
|
|||
Utility.console_log("Running unit test for config: " + ut_config, "O")
|
||||
dut = env.get_dut("unit-test-app", app_path=ut_config)
|
||||
dut.start_app()
|
||||
Utility.console_log("Download finished, start running test cases", "O")
|
||||
|
||||
for one_case in case_config[ut_config]:
|
||||
dut.reset()
|
||||
# esptool ``run`` cmd takes quite long time.
|
||||
# before reset finish, serial port is closed. therefore DUT could already bootup before serial port opened.
|
||||
# this could cause checking bootup print failed.
|
||||
# now we input cmd `-`, and check either bootup print or test history,
|
||||
# to determine if DUT is ready to test.
|
||||
dut.write("-", flush=False)
|
||||
dut.expect_any(UT_APP_BOOT_UP_DONE,
|
||||
"0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT)
|
||||
|
||||
# run test case
|
||||
dut.write("\"{}\"".format(one_case["name"]))
|
||||
dut.expect("Running " + one_case["name"] + "...")
|
||||
|
||||
exception_reset_list = []
|
||||
|
||||
# we want to set this flag in callbacks (inner functions)
|
||||
# use list here so we can use append to set this flag
|
||||
test_finish = list()
|
||||
|
||||
# expect callbacks
|
||||
def one_case_finish(result):
|
||||
""" one test finished, let expect loop break and log result """
|
||||
test_finish.append(True)
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
|
||||
def handle_exception_reset(data):
|
||||
"""
|
||||
just append data to exception list.
|
||||
exception list will be checked in ``handle_reset_finish``, once reset finished.
|
||||
"""
|
||||
exception_reset_list.append(data[0])
|
||||
|
||||
def handle_test_finish(data):
|
||||
""" test finished without reset """
|
||||
# in this scenario reset should not happen
|
||||
assert not exception_reset_list
|
||||
if int(data[1]):
|
||||
# case ignored
|
||||
Utility.console_log("Ignored: " + one_case["name"], color="orange")
|
||||
one_case_finish(not int(data[0]))
|
||||
|
||||
def handle_reset_finish(data):
|
||||
""" reset happened and reboot finished """
|
||||
assert exception_reset_list # reboot but no exception/reset logged. should never happen
|
||||
result = False
|
||||
if len(one_case["reset"]) == len(exception_reset_list):
|
||||
for i, exception in enumerate(exception_reset_list):
|
||||
if one_case["reset"][i] not in exception:
|
||||
break
|
||||
else:
|
||||
result = True
|
||||
if not result:
|
||||
Utility.console_log("""Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}"""
|
||||
.format(one_case["reset"], exception_reset_list),
|
||||
color="orange")
|
||||
one_case_finish(result)
|
||||
|
||||
while not test_finish:
|
||||
try:
|
||||
dut.expect_any((RESET_PATTERN, handle_exception_reset),
|
||||
(EXCEPTION_PATTERN, handle_exception_reset),
|
||||
(ABORT_PATTERN, handle_exception_reset),
|
||||
(FINISH_PATTERN, handle_test_finish),
|
||||
(UT_APP_BOOT_UP_DONE, handle_reset_finish),
|
||||
timeout=one_case["timeout"])
|
||||
except ExpectTimeout:
|
||||
Utility.console_log("Timeout in expect", color="orange")
|
||||
one_case_finish(False)
|
||||
break
|
||||
# create junit report test case
|
||||
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
|
||||
try:
|
||||
run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
# raise exception if any case fails
|
||||
if failed_cases:
|
||||
|
|
@ -233,11 +272,15 @@ class Handler(threading.Thread):
|
|||
self.child_case_index = child_case_index + 1
|
||||
self.finish = False
|
||||
self.result = False
|
||||
self.output = ""
|
||||
self.fail_name = None
|
||||
self.timeout = timeout
|
||||
threading.Thread.__init__(self, name="{} Handler".format(dut))
|
||||
|
||||
def run(self):
|
||||
|
||||
self.dut.start_capture_raw_data()
|
||||
|
||||
def get_child_case_name(data):
|
||||
self.child_case_name = data[0]
|
||||
time.sleep(1)
|
||||
|
|
@ -247,6 +290,8 @@ class Handler(threading.Thread):
|
|||
""" one test finished, let expect loop break and log result """
|
||||
self.finish = True
|
||||
self.result = result
|
||||
self.output = "[{}]\n\n{}\n".format(self.child_case_name,
|
||||
self.dut.stop_capture_raw_data())
|
||||
if not result:
|
||||
self.fail_name = self.child_case_name
|
||||
|
||||
|
|
@ -312,7 +357,7 @@ def get_dut(duts, env, name, ut_config):
|
|||
return dut
|
||||
|
||||
|
||||
def case_run(duts, ut_config, env, one_case, failed_cases):
|
||||
def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, junit_test_case):
|
||||
lock = threading.RLock()
|
||||
threads = []
|
||||
send_signal_list = []
|
||||
|
|
@ -326,19 +371,22 @@ def case_run(duts, ut_config, env, one_case, failed_cases):
|
|||
for thread in threads:
|
||||
thread.setDaemon(True)
|
||||
thread.start()
|
||||
output = "Multiple Device Failed\n"
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
result = result and thread.result
|
||||
output += thread.output
|
||||
if not thread.result:
|
||||
failed_device.append(thread.fail_name)
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
junit_test_case.add_failure_info(output)
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
|
||||
|
||||
@IDF.idf_unit_test(env_tag="UT_T2_1")
|
||||
@IDF.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True)
|
||||
def run_multiple_devices_cases(env, extra_data):
|
||||
"""
|
||||
extra_data can be two types of value
|
||||
|
|
@ -362,11 +410,17 @@ def run_multiple_devices_cases(env, extra_data):
|
|||
"""
|
||||
failed_cases = []
|
||||
case_config = format_test_case_config(extra_data)
|
||||
DUTS = {}
|
||||
duts = {}
|
||||
for ut_config in case_config:
|
||||
Utility.console_log("Running unit test for config: " + ut_config, "O")
|
||||
for one_case in case_config[ut_config]:
|
||||
case_run(DUTS, ut_config, env, one_case, failed_cases)
|
||||
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
|
||||
try:
|
||||
run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, junit_test_case)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
if failed_cases:
|
||||
Utility.console_log("Failed Cases:", color="red")
|
||||
|
|
@ -375,7 +429,109 @@ def run_multiple_devices_cases(env, extra_data):
|
|||
raise AssertionError("Unit Test Failed")
|
||||
|
||||
|
||||
@IDF.idf_unit_test(env_tag="UT_T1_1")
|
||||
def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
|
||||
reset_dut(dut)
|
||||
|
||||
dut.start_capture_raw_data()
|
||||
|
||||
exception_reset_list = []
|
||||
|
||||
for test_stage in range(one_case["child case num"]):
|
||||
# select multi stage test case name
|
||||
dut.write("\"{}\"".format(one_case["name"]))
|
||||
dut.expect("Running " + one_case["name"] + "...")
|
||||
# select test function for current stage
|
||||
dut.write(str(test_stage + 1))
|
||||
|
||||
# we want to set this flag in callbacks (inner functions)
|
||||
# use list here so we can use append to set this flag
|
||||
stage_finish = list()
|
||||
|
||||
def last_stage():
|
||||
return test_stage == one_case["child case num"] - 1
|
||||
|
||||
def check_reset():
|
||||
if one_case["reset"]:
|
||||
assert exception_reset_list # reboot but no exception/reset logged. should never happen
|
||||
result = False
|
||||
if len(one_case["reset"]) == len(exception_reset_list):
|
||||
for i, exception in enumerate(exception_reset_list):
|
||||
if one_case["reset"][i] not in exception:
|
||||
break
|
||||
else:
|
||||
result = True
|
||||
if not result:
|
||||
err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
|
||||
exception_reset_list)
|
||||
Utility.console_log(err_msg, color="orange")
|
||||
junit_test_case.add_error_info(err_msg)
|
||||
else:
|
||||
# we allow omit reset in multi stage cases
|
||||
result = True
|
||||
return result
|
||||
|
||||
# expect callbacks
|
||||
def one_case_finish(result):
|
||||
""" one test finished, let expect loop break and log result """
|
||||
# handle test finish
|
||||
result = result and check_reset()
|
||||
output = dut.stop_capture_raw_data()
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
junit_test_case.add_failure_info(output)
|
||||
stage_finish.append("break")
|
||||
|
||||
def handle_exception_reset(data):
|
||||
"""
|
||||
just append data to exception list.
|
||||
exception list will be checked in ``handle_reset_finish``, once reset finished.
|
||||
"""
|
||||
exception_reset_list.append(data[0])
|
||||
|
||||
def handle_test_finish(data):
|
||||
""" test finished without reset """
|
||||
# in this scenario reset should not happen
|
||||
if int(data[1]):
|
||||
# case ignored
|
||||
Utility.console_log("Ignored: " + one_case["name"], color="orange")
|
||||
junit_test_case.add_skipped_info("ignored")
|
||||
# only passed in last stage will be regarded as real pass
|
||||
if last_stage():
|
||||
one_case_finish(not int(data[0]))
|
||||
else:
|
||||
Utility.console_log("test finished before enter last stage", color="orange")
|
||||
one_case_finish(False)
|
||||
|
||||
def handle_next_stage(data):
|
||||
""" reboot finished. we goto next stage """
|
||||
if last_stage():
|
||||
# already last stage, should never goto next stage
|
||||
Utility.console_log("didn't finish at last stage", color="orange")
|
||||
one_case_finish(False)
|
||||
else:
|
||||
stage_finish.append("continue")
|
||||
|
||||
while not stage_finish:
|
||||
try:
|
||||
dut.expect_any((RESET_PATTERN, handle_exception_reset),
|
||||
(EXCEPTION_PATTERN, handle_exception_reset),
|
||||
(ABORT_PATTERN, handle_exception_reset),
|
||||
(FINISH_PATTERN, handle_test_finish),
|
||||
(UT_APP_BOOT_UP_DONE, handle_next_stage),
|
||||
timeout=one_case["timeout"])
|
||||
except ExpectTimeout:
|
||||
Utility.console_log("Timeout in expect", color="orange")
|
||||
one_case_finish(False)
|
||||
break
|
||||
if stage_finish[0] == "break":
|
||||
# test breaks on current stage
|
||||
break
|
||||
|
||||
|
||||
@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True)
|
||||
def run_multiple_stage_cases(env, extra_data):
|
||||
"""
|
||||
extra_data can be 2 types of value
|
||||
|
|
@ -400,102 +556,13 @@ def run_multiple_stage_cases(env, extra_data):
|
|||
dut.start_app()
|
||||
|
||||
for one_case in case_config[ut_config]:
|
||||
dut.reset()
|
||||
dut.write("-", flush=False)
|
||||
dut.expect_any(UT_APP_BOOT_UP_DONE,
|
||||
"0 Tests 0 Failures 0 Ignored")
|
||||
|
||||
exception_reset_list = []
|
||||
|
||||
for test_stage in range(one_case["child case num"]):
|
||||
# select multi stage test case name
|
||||
dut.write("\"{}\"".format(one_case["name"]))
|
||||
dut.expect("Running " + one_case["name"] + "...")
|
||||
# select test function for current stage
|
||||
dut.write(str(test_stage + 1))
|
||||
|
||||
# we want to set this flag in callbacks (inner functions)
|
||||
# use list here so we can use append to set this flag
|
||||
stage_finish = list()
|
||||
|
||||
def last_stage():
|
||||
return test_stage == one_case["child case num"] - 1
|
||||
|
||||
def check_reset():
|
||||
if one_case["reset"]:
|
||||
assert exception_reset_list # reboot but no exception/reset logged. should never happen
|
||||
result = False
|
||||
if len(one_case["reset"]) == len(exception_reset_list):
|
||||
for i, exception in enumerate(exception_reset_list):
|
||||
if one_case["reset"][i] not in exception:
|
||||
break
|
||||
else:
|
||||
result = True
|
||||
if not result:
|
||||
Utility.console_log("""Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}"""
|
||||
.format(one_case["reset"], exception_reset_list),
|
||||
color="orange")
|
||||
else:
|
||||
# we allow omit reset in multi stage cases
|
||||
result = True
|
||||
return result
|
||||
|
||||
# expect callbacks
|
||||
def one_case_finish(result):
|
||||
""" one test finished, let expect loop break and log result """
|
||||
# handle test finish
|
||||
result = result and check_reset()
|
||||
if result:
|
||||
Utility.console_log("Success: " + one_case["name"], color="green")
|
||||
else:
|
||||
failed_cases.append(one_case["name"])
|
||||
Utility.console_log("Failed: " + one_case["name"], color="red")
|
||||
stage_finish.append("break")
|
||||
|
||||
def handle_exception_reset(data):
|
||||
"""
|
||||
just append data to exception list.
|
||||
exception list will be checked in ``handle_reset_finish``, once reset finished.
|
||||
"""
|
||||
exception_reset_list.append(data[0])
|
||||
|
||||
def handle_test_finish(data):
|
||||
""" test finished without reset """
|
||||
# in this scenario reset should not happen
|
||||
if int(data[1]):
|
||||
# case ignored
|
||||
Utility.console_log("Ignored: " + one_case["name"], color="orange")
|
||||
# only passed in last stage will be regarded as real pass
|
||||
if last_stage():
|
||||
one_case_finish(not int(data[0]))
|
||||
else:
|
||||
Utility.console_log("test finished before enter last stage", color="orange")
|
||||
one_case_finish(False)
|
||||
|
||||
def handle_next_stage(data):
|
||||
""" reboot finished. we goto next stage """
|
||||
if last_stage():
|
||||
# already last stage, should never goto next stage
|
||||
Utility.console_log("didn't finish at last stage", color="orange")
|
||||
one_case_finish(False)
|
||||
else:
|
||||
stage_finish.append("continue")
|
||||
|
||||
while not stage_finish:
|
||||
try:
|
||||
dut.expect_any((RESET_PATTERN, handle_exception_reset),
|
||||
(EXCEPTION_PATTERN, handle_exception_reset),
|
||||
(ABORT_PATTERN, handle_exception_reset),
|
||||
(FINISH_PATTERN, handle_test_finish),
|
||||
(UT_APP_BOOT_UP_DONE, handle_next_stage),
|
||||
timeout=one_case["timeout"])
|
||||
except ExpectTimeout:
|
||||
Utility.console_log("Timeout in expect", color="orange")
|
||||
one_case_finish(False)
|
||||
break
|
||||
if stage_finish[0] == "break":
|
||||
# test breaks on current stage
|
||||
break
|
||||
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
|
||||
try:
|
||||
run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
except Exception as e:
|
||||
junit_test_case.add_error_info("Unexpected exception: " + str(e))
|
||||
TinyFW.JunitReport.test_case_finish(junit_test_case)
|
||||
|
||||
# raise exception if any case fails
|
||||
if failed_cases:
|
||||
|
|
|
|||
Loading…
Reference in a new issue