Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

unit-test: fix 2 issues in unit test script:

Browse source 
1. Gitlab will only regard failure (not error) as failed case. Use log
failure instead of log error, to display all error msg in test result
2. fix failed case not recognized by test script when failed during
reset DUT
This commit is contained in:
He Yin Ling 2018-12-20 17:00:12 +08:00 committed by bot
parent 7c59207351
commit 431653bba8
1 changed files with 24 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
tools/unit-test-app/unit_test.py
View file

@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2
DEFAULT_TIMEOUT = 20
DUT_STARTUP_CHECK_RETRY_COUNT = 5
TEST_HISTROY_CHECK_TIMEOUT = 1
TEST_HISTORY_CHECK_TIMEOUT = 1
class TestCaseFailed(AssertionError):
pass
def format_test_case_config(test_case_data):
@ -163,7 +167,7 @@ def reset_dut(dut):
for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
dut.write("-")
try:
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
break
except ExpectTimeout:
pass
@ -171,7 +175,7 @@ def reset_dut(dut):
raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
def run_one_normal_case(dut, one_case, junit_test_case):
reset_dut(dut)
@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
if result:
Utility.console_log("Success: " + one_case["name"], color="green")
else:
failed_cases.append(one_case["name"])
Utility.console_log("Failed: " + one_case["name"], color="red")
junit_test_case.add_failure_info(output)
raise TestCaseFailed()
def handle_exception_reset(data):
"""
@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
timeout=one_case["timeout"])
except ExpectTimeout:
Utility.console_log("Timeout in expect", color="orange")
junit_test_case.add_error_info("timeout")
junit_test_case.add_failure_info("timeout")
one_case_finish(False)
break
@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data):
# create junit report test case
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try:
run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
TinyFW.JunitReport.test_case_finish(junit_test_case)
run_one_normal_case(dut, one_case, junit_test_case)
except TestCaseFailed:
failed_cases.append(one_case["name"])
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails
@ -476,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data):
try:
run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
one_case.get('app_bin'), junit_test_case)
TinyFW.JunitReport.test_case_finish(junit_test_case)
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
if failed_cases:
@ -488,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data):
raise AssertionError("Unit Test Failed")
def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
def run_one_multiple_stage_case(dut, one_case, junit_test_case):
reset_dut(dut)
dut.start_capture_raw_data()
@ -523,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
exception_reset_list)
Utility.console_log(err_msg, color="orange")
junit_test_case.add_error_info(err_msg)
junit_test_case.add_failure_info(err_msg)
else:
# we allow omit reset in multi stage cases
result = True
@ -538,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
if result:
Utility.console_log("Success: " + one_case["name"], color="green")
else:
failed_cases.append(one_case["name"])
Utility.console_log("Failed: " + one_case["name"], color="red")
junit_test_case.add_failure_info(output)
raise TestCaseFailed()
stage_finish.append("break")
def handle_exception_reset(data):
@ -620,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data):
for one_case in case_config[ut_config]:
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try:
run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
TinyFW.JunitReport.test_case_finish(junit_test_case)
run_one_multiple_stage_case(dut, one_case, junit_test_case)
except TestCaseFailed:
failed_cases.append(one_case["name"])
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails