From 644571f78b5e7ae464b35ab293dfed274bda99e8 Mon Sep 17 00:00:00 2001 From: He Yin Ling Date: Fri, 13 Jul 2018 16:48:43 +0800 Subject: [PATCH] test: generate junit test report according to executed cases --- tools/tiny-test-fw/IDF/__init__.py | 2 +- tools/unit-test-app/unit_test.py | 384 ++++++++++++++++------------- 2 files changed, 220 insertions(+), 166 deletions(-) diff --git a/tools/tiny-test-fw/IDF/__init__.py b/tools/tiny-test-fw/IDF/__init__.py index c8751ebeb..0e342e844 100644 --- a/tools/tiny-test-fw/IDF/__init__.py +++ b/tools/tiny-test-fw/IDF/__init__.py @@ -81,7 +81,7 @@ def log_performance(item, value): Utility.console_log(performance_msg, "orange") # update to junit test report current_junit_case = TinyFW.JunitReport.get_current_test_case() - current_junit_case.stdout += performance_msg + current_junit_case.stdout += performance_msg + "\r\n" def check_performance(item, value): diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 884512a34..128b89504 100755 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -23,7 +23,6 @@ import os import sys import time import argparse - import threading # if we want to run test case outside `tiny-test-fw` folder, @@ -132,6 +131,7 @@ def format_test_case_config(test_case_data): return case_config + def replace_app_bin(dut, name, new_app_bin): if new_app_bin is None: return @@ -142,6 +142,7 @@ def replace_app_bin(dut, name, new_app_bin): Utility.console_log("The replaced application binary is {}".format(new_app_bin), "O") break + def reset_dut(dut): dut.reset() # esptool ``run`` cmd takes quite long time. @@ -160,7 +161,83 @@ def reset_dut(dut): raise AssertationError("Reset {} ({}) failed!".format(dut.name, dut.port)) -@IDF.idf_unit_test(env_tag="UT_T1_1") +def run_one_normal_case(dut, one_case, junit_test_case, failed_cases): + + reset_dut(dut) + + dut.start_capture_raw_data() + # run test case + dut.write("\"{}\"".format(one_case["name"])) + dut.expect("Running " + one_case["name"] + "...") + + exception_reset_list = [] + + # we want to set this flag in callbacks (inner functions) + # use list here so we can use append to set this flag + test_finish = list() + + # expect callbacks + def one_case_finish(result): + """ one test finished, let expect loop break and log result """ + test_finish.append(True) + output = dut.stop_capture_raw_data() + if result: + Utility.console_log("Success: " + one_case["name"], color="green") + else: + failed_cases.append(one_case["name"]) + Utility.console_log("Failed: " + one_case["name"], color="red") + junit_test_case.add_failure_info(output) + + def handle_exception_reset(data): + """ + just append data to exception list. + exception list will be checked in ``handle_reset_finish``, once reset finished. + """ + exception_reset_list.append(data[0]) + + def handle_test_finish(data): + """ test finished without reset """ + # in this scenario reset should not happen + assert not exception_reset_list + if int(data[1]): + # case ignored + Utility.console_log("Ignored: " + one_case["name"], color="orange") + junit_test_case.add_skipped_info("ignored") + one_case_finish(not int(data[0])) + + def handle_reset_finish(data): + """ reset happened and reboot finished """ + assert exception_reset_list # reboot but no exception/reset logged. should never happen + result = False + if len(one_case["reset"]) == len(exception_reset_list): + for i, exception in enumerate(exception_reset_list): + if one_case["reset"][i] not in exception: + break + else: + result = True + if not result: + err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"], + exception_reset_list) + Utility.console_log(err_msg, color="orange") + junit_test_case.add_error_info(err_msg) + one_case_finish(result) + + while not test_finish: + try: + dut.expect_any((RESET_PATTERN, handle_exception_reset), + (EXCEPTION_PATTERN, handle_exception_reset), + (ABORT_PATTERN, handle_exception_reset), + (FINISH_PATTERN, handle_test_finish), + (UT_APP_BOOT_UP_DONE, handle_reset_finish), + timeout=one_case["timeout"]) + except ExpectTimeout: + Utility.console_log("Timeout in expect", color="orange") + junit_test_case.add_error_info("timeout") + one_case_finish(False) + break + + +@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) def run_unit_test_cases(env, extra_data): """ extra_data can be three types of value @@ -190,74 +267,17 @@ def run_unit_test_cases(env, extra_data): if len(case_config[ut_config]) > 0: replace_app_bin(dut, "unit-test-app", case_config[ut_config][0].get('app_bin')) dut.start_app() + Utility.console_log("Download finished, start running test cases", "O") for one_case in case_config[ut_config]: - reset_dut(dut) - - # run test case - dut.write("\"{}\"".format(one_case["name"])) - dut.expect("Running " + one_case["name"] + "...") - - exception_reset_list = [] - - # we want to set this flag in callbacks (inner functions) - # use list here so we can use append to set this flag - test_finish = list() - - # expect callbacks - def one_case_finish(result): - """ one test finished, let expect loop break and log result """ - test_finish.append(True) - if result: - Utility.console_log("Success: " + one_case["name"], color="green") - else: - failed_cases.append(one_case["name"]) - Utility.console_log("Failed: " + one_case["name"], color="red") - - def handle_exception_reset(data): - """ - just append data to exception list. - exception list will be checked in ``handle_reset_finish``, once reset finished. - """ - exception_reset_list.append(data[0]) - - def handle_test_finish(data): - """ test finished without reset """ - # in this scenario reset should not happen - assert not exception_reset_list - if int(data[1]): - # case ignored - Utility.console_log("Ignored: " + one_case["name"], color="orange") - one_case_finish(not int(data[0])) - - def handle_reset_finish(data): - """ reset happened and reboot finished """ - assert exception_reset_list # reboot but no exception/reset logged. should never happen - result = False - if len(one_case["reset"]) == len(exception_reset_list): - for i, exception in enumerate(exception_reset_list): - if one_case["reset"][i] not in exception: - break - else: - result = True - if not result: - Utility.console_log("""Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}""" - .format(one_case["reset"], exception_reset_list), - color="orange") - one_case_finish(result) - - while not test_finish: - try: - dut.expect_any((RESET_PATTERN, handle_exception_reset), - (EXCEPTION_PATTERN, handle_exception_reset), - (ABORT_PATTERN, handle_exception_reset), - (FINISH_PATTERN, handle_test_finish), - (UT_APP_BOOT_UP_DONE, handle_reset_finish), - timeout=one_case["timeout"]) - except ExpectTimeout: - Utility.console_log("Timeout in expect", color="orange") - one_case_finish(False) - break + # create junit report test case + junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) + try: + run_one_normal_case(dut, one_case, junit_test_case, failed_cases) + TinyFW.JunitReport.test_case_finish(junit_test_case) + except Exception as e: + junit_test_case.add_error_info("Unexpected exception: " + str(e)) + TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails if failed_cases: @@ -283,6 +303,7 @@ class Handler(threading.Thread): self.child_case_index = child_case_index + 1 self.finish = False self.result = False + self.output = "" self.fail_name = None self.timeout = timeout self.force_stop = threading.Event() # it show the running status @@ -292,6 +313,9 @@ class Handler(threading.Thread): threading.Thread.__init__(self, name="{} Handler".format(dut)) def run(self): + + self.dut.start_capture_raw_data() + def get_child_case_name(data): self.child_case_name = data[0] time.sleep(1) @@ -301,6 +325,8 @@ class Handler(threading.Thread): """ one test finished, let expect loop break and log result """ self.finish = True self.result = result + self.output = "[{}]\n\n{}\n".format(self.child_case_name, + self.dut.stop_capture_raw_data()) if not result: self.fail_name = self.child_case_name @@ -370,7 +396,7 @@ def get_dut(duts, env, name, ut_config, app_bin=None): return dut -def case_run(duts, ut_config, env, one_case, failed_cases, app_bin): +def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, junit_test_case): lock = threading.RLock() threads = [] send_signal_list = [] @@ -384,9 +410,11 @@ def case_run(duts, ut_config, env, one_case, failed_cases, app_bin): for thread in threads: thread.setDaemon(True) thread.start() + output = "Multiple Device Failed\n" for thread in threads: thread.join() result = result and thread.result + output += thread.output if not thread.result: [thd.stop() for thd in threads] @@ -394,10 +422,11 @@ def case_run(duts, ut_config, env, one_case, failed_cases, app_bin): Utility.console_log("Success: " + one_case["name"], color="green") else: failed_cases.append(one_case["name"]) + junit_test_case.add_failure_info(output) Utility.console_log("Failed: " + one_case["name"], color="red") -@IDF.idf_unit_test(env_tag="UT_T2_1") +@IDF.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True) def run_multiple_devices_cases(env, extra_data): """ extra_data can be two types of value @@ -421,11 +450,17 @@ def run_multiple_devices_cases(env, extra_data): """ failed_cases = [] case_config = format_test_case_config(extra_data) - DUTS = {} + duts = {} for ut_config in case_config: Utility.console_log("Running unit test for config: " + ut_config, "O") for one_case in case_config[ut_config]: - case_run(DUTS, ut_config, env, one_case, failed_cases, one_case.get('app_bin')) + junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) + try: + run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, junit_test_case) + TinyFW.JunitReport.test_case_finish(junit_test_case) + except Exception as e: + junit_test_case.add_error_info("Unexpected exception: " + str(e)) + TinyFW.JunitReport.test_case_finish(junit_test_case) if failed_cases: Utility.console_log("Failed Cases:", color="red") @@ -434,7 +469,109 @@ def run_multiple_devices_cases(env, extra_data): raise AssertionError("Unit Test Failed") -@IDF.idf_unit_test(env_tag="UT_T1_1") +def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case): + reset_dut(dut) + + dut.start_capture_raw_data() + + exception_reset_list = [] + + for test_stage in range(one_case["child case num"]): + # select multi stage test case name + dut.write("\"{}\"".format(one_case["name"])) + dut.expect("Running " + one_case["name"] + "...") + # select test function for current stage + dut.write(str(test_stage + 1)) + + # we want to set this flag in callbacks (inner functions) + # use list here so we can use append to set this flag + stage_finish = list() + + def last_stage(): + return test_stage == one_case["child case num"] - 1 + + def check_reset(): + if one_case["reset"]: + assert exception_reset_list # reboot but no exception/reset logged. should never happen + result = False + if len(one_case["reset"]) == len(exception_reset_list): + for i, exception in enumerate(exception_reset_list): + if one_case["reset"][i] not in exception: + break + else: + result = True + if not result: + err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"], + exception_reset_list) + Utility.console_log(err_msg, color="orange") + junit_test_case.add_error_info(err_msg) + else: + # we allow omit reset in multi stage cases + result = True + return result + + # expect callbacks + def one_case_finish(result): + """ one test finished, let expect loop break and log result """ + # handle test finish + result = result and check_reset() + output = dut.stop_capture_raw_data() + if result: + Utility.console_log("Success: " + one_case["name"], color="green") + else: + failed_cases.append(one_case["name"]) + Utility.console_log("Failed: " + one_case["name"], color="red") + junit_test_case.add_failure_info(output) + stage_finish.append("break") + + def handle_exception_reset(data): + """ + just append data to exception list. + exception list will be checked in ``handle_reset_finish``, once reset finished. + """ + exception_reset_list.append(data[0]) + + def handle_test_finish(data): + """ test finished without reset """ + # in this scenario reset should not happen + if int(data[1]): + # case ignored + Utility.console_log("Ignored: " + one_case["name"], color="orange") + junit_test_case.add_skipped_info("ignored") + # only passed in last stage will be regarded as real pass + if last_stage(): + one_case_finish(not int(data[0])) + else: + Utility.console_log("test finished before enter last stage", color="orange") + one_case_finish(False) + + def handle_next_stage(data): + """ reboot finished. we goto next stage """ + if last_stage(): + # already last stage, should never goto next stage + Utility.console_log("didn't finish at last stage", color="orange") + one_case_finish(False) + else: + stage_finish.append("continue") + + while not stage_finish: + try: + dut.expect_any((RESET_PATTERN, handle_exception_reset), + (EXCEPTION_PATTERN, handle_exception_reset), + (ABORT_PATTERN, handle_exception_reset), + (FINISH_PATTERN, handle_test_finish), + (UT_APP_BOOT_UP_DONE, handle_next_stage), + timeout=one_case["timeout"]) + except ExpectTimeout: + Utility.console_log("Timeout in expect", color="orange") + one_case_finish(False) + break + if stage_finish[0] == "break": + # test breaks on current stage + break + + +@IDF.idf_unit_test(env_tag="UT_T1_1", junit_report_by_case=True) def run_multiple_stage_cases(env, extra_data): """ extra_data can be 2 types of value @@ -461,98 +598,13 @@ def run_multiple_stage_cases(env, extra_data): dut.start_app() for one_case in case_config[ut_config]: - reset_dut(dut) - exception_reset_list = [] - - for test_stage in range(one_case["child case num"]): - # select multi stage test case name - dut.write("\"{}\"".format(one_case["name"])) - dut.expect("Running " + one_case["name"] + "...") - # select test function for current stage - dut.write(str(test_stage + 1)) - - # we want to set this flag in callbacks (inner functions) - # use list here so we can use append to set this flag - stage_finish = list() - - def last_stage(): - return test_stage == one_case["child case num"] - 1 - - def check_reset(): - if one_case["reset"]: - assert exception_reset_list # reboot but no exception/reset logged. should never happen - result = False - if len(one_case["reset"]) == len(exception_reset_list): - for i, exception in enumerate(exception_reset_list): - if one_case["reset"][i] not in exception: - break - else: - result = True - if not result: - Utility.console_log("""Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}""" - .format(one_case["reset"], exception_reset_list), - color="orange") - else: - # we allow omit reset in multi stage cases - result = True - return result - - # expect callbacks - def one_case_finish(result): - """ one test finished, let expect loop break and log result """ - # handle test finish - result = result and check_reset() - if result: - Utility.console_log("Success: " + one_case["name"], color="green") - else: - failed_cases.append(one_case["name"]) - Utility.console_log("Failed: " + one_case["name"], color="red") - stage_finish.append("break") - - def handle_exception_reset(data): - """ - just append data to exception list. - exception list will be checked in ``handle_reset_finish``, once reset finished. - """ - exception_reset_list.append(data[0]) - - def handle_test_finish(data): - """ test finished without reset """ - # in this scenario reset should not happen - if int(data[1]): - # case ignored - Utility.console_log("Ignored: " + one_case["name"], color="orange") - # only passed in last stage will be regarded as real pass - if last_stage(): - one_case_finish(not int(data[0])) - else: - Utility.console_log("test finished before enter last stage", color="orange") - one_case_finish(False) - - def handle_next_stage(data): - """ reboot finished. we goto next stage """ - if last_stage(): - # already last stage, should never goto next stage - Utility.console_log("didn't finish at last stage", color="orange") - one_case_finish(False) - else: - stage_finish.append("continue") - - while not stage_finish: - try: - dut.expect_any((RESET_PATTERN, handle_exception_reset), - (EXCEPTION_PATTERN, handle_exception_reset), - (ABORT_PATTERN, handle_exception_reset), - (FINISH_PATTERN, handle_test_finish), - (UT_APP_BOOT_UP_DONE, handle_next_stage), - timeout=one_case["timeout"]) - except ExpectTimeout: - Utility.console_log("Timeout in expect", color="orange") - one_case_finish(False) - break - if stage_finish[0] == "break": - # test breaks on current stage - break + junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"])) + try: + run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case) + TinyFW.JunitReport.test_case_finish(junit_test_case) + except Exception as e: + junit_test_case.add_error_info("Unexpected exception: " + str(e)) + TinyFW.JunitReport.test_case_finish(junit_test_case) # raise exception if any case fails if failed_cases: @@ -561,6 +613,7 @@ def run_multiple_stage_cases(env, extra_data): Utility.console_log("\t" + _case_name, color="red") raise AssertionError("Unit Test Failed") + def detect_update_unit_test_info(env, extra_data, app_bin): case_config = format_test_case_config(extra_data) @@ -624,6 +677,7 @@ def detect_update_unit_test_info(env, extra_data, app_bin): # These options are the same for all configs, therefore there is no need to continue break + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument(