From e8525396ddfc8a61bf8951ad44b778c9dbe0a15f Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 21 May 2018 14:50:27 +1000 Subject: [PATCH 1/2] test: Allow per-unit-test timeouts Work around for "(WL) write/read speed test" taking >30s in some configs --- components/fatfs/test/test_fatfs_sdmmc.c | 2 +- components/fatfs/test/test_fatfs_spiflash.c | 2 +- tools/tiny-test-fw/CIAssignUnitTest.py | 5 +++++ tools/unit-test-app/tools/TagDefinition.yml | 3 +++ tools/unit-test-app/tools/UnitTestParser.py | 6 +++--- tools/unit-test-app/unit_test.py | 15 ++++++++------- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/components/fatfs/test/test_fatfs_sdmmc.c b/components/fatfs/test/test_fatfs_sdmmc.c index 6dc1ec3fc..c81302207 100644 --- a/components/fatfs/test/test_fatfs_sdmmc.c +++ b/components/fatfs/test/test_fatfs_sdmmc.c @@ -154,7 +154,7 @@ TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDM static void speed_test(void* buf, size_t buf_size, size_t file_size, bool write); -TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE]") +TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60]") { size_t heap_size; HEAP_SIZE_CAPTURE(heap_size); diff --git a/components/fatfs/test/test_fatfs_spiflash.c b/components/fatfs/test/test_fatfs_spiflash.c index 55b3b6878..70f73f0e5 100644 --- a/components/fatfs/test/test_fatfs_spiflash.c +++ b/components/fatfs/test/test_fatfs_spiflash.c @@ -145,7 +145,7 @@ TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling]") test_teardown(); } -TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling]") +TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60]") { /* Erase partition before running the test to get consistent results */ const esp_partition_t* part = get_test_data_partition(); diff --git a/tools/tiny-test-fw/CIAssignUnitTest.py b/tools/tiny-test-fw/CIAssignUnitTest.py index 03768dbf8..3c3151458 100644 --- a/tools/tiny-test-fw/CIAssignUnitTest.py +++ b/tools/tiny-test-fw/CIAssignUnitTest.py @@ -38,12 +38,17 @@ class Group(CIAssignTest.Group): return test_app[3:] def _create_extra_data(self, test_function): + """ + For unit test case, we need to copy some attributes of test cases into config file. + So unit test function knows how to run the case. + """ case_data = [] for case in self.case_list: one_case_data = { "config": self._get_ut_config(self._get_case_attr(case, "Test App")), "name": self._get_case_attr(case, "summary"), "reset": self._get_case_attr(case, "reset"), + "timeout": self._get_case_attr(case, "timeout"), } if test_function in ["run_multiple_devices_cases", "run_multiple_stage_cases"]: diff --git a/tools/unit-test-app/tools/TagDefinition.yml b/tools/unit-test-app/tools/TagDefinition.yml index 293208d7b..f84d9a702 100644 --- a/tools/unit-test-app/tools/TagDefinition.yml +++ b/tools/unit-test-app/tools/TagDefinition.yml @@ -15,3 +15,6 @@ multi_device: multi_stage: default: "Yes" omitted: "No" +timeout: + default: 30 + omitted: 30 diff --git a/tools/unit-test-app/tools/UnitTestParser.py b/tools/unit-test-app/tools/UnitTestParser.py index db9a94119..688e4ce37 100644 --- a/tools/unit-test-app/tools/UnitTestParser.py +++ b/tools/unit-test-app/tools/UnitTestParser.py @@ -8,7 +8,6 @@ import hashlib from copy import deepcopy import CreateSectionTable - TEST_CASE_PATTERN = { "initial condition": "UTINIT1", "SDK": "ESP32_IDF", @@ -21,7 +20,7 @@ TEST_CASE_PATTERN = { "test environment": "UT_T1_1", "reset": "", "expected result": "1. set succeed", - "cmd set": "test_unit_test_case", + "cmd set": "test_unit_test_case" } CONFIG_FILE_PATTERN = { @@ -191,7 +190,8 @@ class Parser(object): "sub module": self.module_map[prop["module"]]['sub module'], "summary": name, "multi_device": prop["multi_device"], - "multi_stage": prop["multi_stage"]}) + "multi_stage": prop["multi_stage"], + "timeout": int(prop["timeout"])}) return test_case def dump_test_cases(self, test_cases): diff --git a/tools/unit-test-app/unit_test.py b/tools/unit-test-app/unit_test.py index 594a71bec..e049e7505 100644 --- a/tools/unit-test-app/unit_test.py +++ b/tools/unit-test-app/unit_test.py @@ -27,7 +27,8 @@ RESET_PATTERN = re.compile(r"(ets [\w]{3}\s+[\d]{1,2} [\d]{4} [\d]{2}:[\d]{2}:[\ EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))") ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)") FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored") -UT_TIMEOUT = 30 + +STARTUP_TIMEOUT=10 def format_test_case_config(test_case_data): @@ -142,7 +143,7 @@ def run_unit_test_cases(env, extra_data): # to determine if DUT is ready to test. dut.write("-", flush=False) dut.expect_any(UT_APP_BOOT_UP_DONE, - "0 Tests 0 Failures 0 Ignored", timeout=UT_TIMEOUT) + "0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT) # run test case dut.write("\"{}\"".format(one_case["name"])) @@ -203,7 +204,7 @@ def run_unit_test_cases(env, extra_data): (ABORT_PATTERN, handle_exception_reset), (FINISH_PATTERN, handle_test_finish), (UT_APP_BOOT_UP_DONE, handle_reset_finish), - timeout=UT_TIMEOUT) + timeout=one_case["timeout"]) except ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") one_case_finish(False) @@ -223,7 +224,7 @@ class Handler(threading.Thread): SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!') FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored") - def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout=30): + def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout): self.dut = dut self.sent_signal_list = sent_signal_list self.lock = lock @@ -288,7 +289,7 @@ class Handler(threading.Thread): (self.WAIT_SIGNAL_PATTERN, device_wait_action), # wait signal pattern (self.SEND_SIGNAL_PATTERN, device_send_action), # send signal pattern (self.FINISH_PATTERN, handle_device_test_finish), # test finish pattern - timeout=UT_TIMEOUT) + timeout=self.timeout) except ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") one_device_case_finish(False) @@ -321,7 +322,7 @@ def case_run(duts, ut_config, env, one_case, failed_cases): for i in range(case_num): dut = get_dut(duts, env, "dut%d" % i, ut_config) threads.append(Handler(dut, send_signal_list, lock, - parent_case, i)) + parent_case, i, one_case["timeout"])) for thread in threads: thread.setDaemon(True) thread.start() @@ -487,7 +488,7 @@ def run_multiple_stage_cases(env, extra_data): (ABORT_PATTERN, handle_exception_reset), (FINISH_PATTERN, handle_test_finish), (UT_APP_BOOT_UP_DONE, handle_next_stage), - timeout=UT_TIMEOUT) + timeout=one_case["timeout"]) except ExpectTimeout: Utility.console_log("Timeout in expect", color="orange") one_case_finish(False) From 156dd4841d5b0a6f4b03305f768a26b3a89217e0 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Mon, 21 May 2018 14:59:55 +1000 Subject: [PATCH 2/2] Temporarily disable SD CD/WP mode tests Tracked in TW22648 --- components/sdmmc/test/test_sd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/sdmmc/test/test_sd.c b/components/sdmmc/test/test_sd.c index 1808a0cdd..d16dcd0e3 100644 --- a/components/sdmmc/test/test_sd.c +++ b/components/sdmmc/test/test_sd.c @@ -266,7 +266,7 @@ static void test_cd_input(int gpio_cd_num, const sdmmc_host_t* config) free(card); } -TEST_CASE("CD input works in SD mode", "[sd][test_env=UT_T1_SDMODE]") +TEST_CASE("CD input works in SD mode", "[sd][test_env=UT_T1_SDMODE][ignore]") { sdmmc_host_t config = SDMMC_HOST_DEFAULT(); sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT(); @@ -327,7 +327,7 @@ static void test_wp_input(int gpio_wp_num, const sdmmc_host_t* config) free(card); } -TEST_CASE("WP input works in SD mode", "[sd][test_env=UT_T1_SDMODE]") +TEST_CASE("WP input works in SD mode", "[sd][test_env=UT_T1_SDMODE][ignore]") { sdmmc_host_t config = SDMMC_HOST_DEFAULT(); sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();