Merge branch 'bugfix/unit_test_timeout' into 'master'

Unit tests: Add per-test timeout option, disable two SD tests

See merge request idf/esp-idf!2423

components/fatfs/test/test_fatfs_sdmmc.c

@@ -154,7 +154,7 @@ TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDM
 
 static void speed_test(void* buf, size_t buf_size, size_t file_size, bool write);
 
-TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE]")
+TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60]")
 {
     size_t heap_size;
     HEAP_SIZE_CAPTURE(heap_size);

components/fatfs/test/test_fatfs_spiflash.c

@@ -145,7 +145,7 @@ TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling]")
     test_teardown();
 }
 
-TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling]")
+TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60]")
 {
     /* Erase partition before running the test to get consistent results */
     const esp_partition_t* part = get_test_data_partition();

components/sdmmc/test/test_sd.c

@@ -266,7 +266,7 @@ static void test_cd_input(int gpio_cd_num, const sdmmc_host_t* config)
     free(card);
 }
 
-TEST_CASE("CD input works in SD mode", "[sd][test_env=UT_T1_SDMODE]")
+TEST_CASE("CD input works in SD mode", "[sd][test_env=UT_T1_SDMODE][ignore]")
 {
     sdmmc_host_t config = SDMMC_HOST_DEFAULT();
     sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();
@@ -327,7 +327,7 @@ static void test_wp_input(int gpio_wp_num, const sdmmc_host_t* config)
     free(card);
 }
 
-TEST_CASE("WP input works in SD mode", "[sd][test_env=UT_T1_SDMODE]")
+TEST_CASE("WP input works in SD mode", "[sd][test_env=UT_T1_SDMODE][ignore]")
 {
     sdmmc_host_t config = SDMMC_HOST_DEFAULT();
     sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();

tools/tiny-test-fw/CIAssignUnitTest.py

@@ -38,12 +38,17 @@ class Group(CIAssignTest.Group):
         return test_app[3:]
 
     def _create_extra_data(self, test_function):
+        """
+        For unit test case, we need to copy some attributes of test cases into config file.
+        So unit test function knows how to run the case.
+        """
         case_data = []
         for case in self.case_list:
             one_case_data = {
                 "config": self._get_ut_config(self._get_case_attr(case, "Test App")),
                 "name": self._get_case_attr(case, "summary"),
                 "reset": self._get_case_attr(case, "reset"),
+                "timeout": self._get_case_attr(case, "timeout"),
             }
 
             if test_function in ["run_multiple_devices_cases", "run_multiple_stage_cases"]:

tools/unit-test-app/tools/TagDefinition.yml

@@ -15,3 +15,6 @@ multi_device:
 multi_stage:
   default: "Yes"
   omitted: "No"
+timeout:
+  default: 30
+  omitted: 30

tools/unit-test-app/tools/UnitTestParser.py

@@ -8,7 +8,6 @@ import hashlib
 from copy import deepcopy
 import CreateSectionTable
 
-
 TEST_CASE_PATTERN = {
     "initial condition": "UTINIT1",
     "SDK": "ESP32_IDF",
@@ -21,7 +20,7 @@ TEST_CASE_PATTERN = {
     "test environment": "UT_T1_1",
     "reset": "",
     "expected result": "1. set succeed",
-    "cmd set": "test_unit_test_case",
+    "cmd set": "test_unit_test_case"
 }
 
 CONFIG_FILE_PATTERN = {
@@ -191,7 +190,8 @@ class Parser(object):
                           "sub module": self.module_map[prop["module"]]['sub module'],
                           "summary": name,
                           "multi_device": prop["multi_device"],
-                          "multi_stage": prop["multi_stage"]})
+                          "multi_stage": prop["multi_stage"],
+                          "timeout": int(prop["timeout"])})
         return test_case
 
     def dump_test_cases(self, test_cases):

tools/unit-test-app/unit_test.py

@@ -27,7 +27,8 @@ RESET_PATTERN = re.compile(r"(ets [\w]{3}\s+[\d]{1,2} [\d]{4} [\d]{2}:[\d]{2}:[\
 EXCEPTION_PATTERN = re.compile(r"(Guru Meditation Error: Core\s+\d panic'ed \([\w].*?\))")
 ABORT_PATTERN = re.compile(r"(abort\(\) was called at PC 0x[a-eA-E\d]{8} on core \d)")
 FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
-UT_TIMEOUT = 30
+
+STARTUP_TIMEOUT=10
 
 
 def format_test_case_config(test_case_data):
@@ -142,7 +143,7 @@ def run_unit_test_cases(env, extra_data):
             # to determine if DUT is ready to test.
             dut.write("-", flush=False)
             dut.expect_any(UT_APP_BOOT_UP_DONE,
-                           "0 Tests 0 Failures 0 Ignored", timeout=UT_TIMEOUT)
+                           "0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT)
 
             # run test case
             dut.write("\"{}\"".format(one_case["name"]))
@@ -203,7 +204,7 @@ def run_unit_test_cases(env, extra_data):
                                    (ABORT_PATTERN, handle_exception_reset),
                                    (FINISH_PATTERN, handle_test_finish),
                                    (UT_APP_BOOT_UP_DONE, handle_reset_finish),
-                                   timeout=UT_TIMEOUT)
+                                   timeout=one_case["timeout"])
                 except ExpectTimeout:
                     Utility.console_log("Timeout in expect", color="orange")
                     one_case_finish(False)
@@ -223,7 +224,7 @@ class Handler(threading.Thread):
     SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!')
     FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
 
-    def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout=30):
+    def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout):
         self.dut = dut
         self.sent_signal_list = sent_signal_list
         self.lock = lock
@@ -288,7 +289,7 @@ class Handler(threading.Thread):
                                     (self.WAIT_SIGNAL_PATTERN, device_wait_action),  # wait signal pattern
                                     (self.SEND_SIGNAL_PATTERN, device_send_action),  # send signal pattern
                                     (self.FINISH_PATTERN, handle_device_test_finish),  # test finish pattern
-                                    timeout=UT_TIMEOUT)
+                                    timeout=self.timeout)
             except ExpectTimeout:
                 Utility.console_log("Timeout in expect", color="orange")
                 one_device_case_finish(False)
@@ -321,7 +322,7 @@ def case_run(duts, ut_config, env, one_case, failed_cases):
     for i in range(case_num):
         dut = get_dut(duts, env, "dut%d" % i, ut_config)
         threads.append(Handler(dut, send_signal_list, lock,
-                               parent_case, i))
+                               parent_case, i, one_case["timeout"]))
     for thread in threads:
         thread.setDaemon(True)
         thread.start()
@@ -487,7 +488,7 @@ def run_multiple_stage_cases(env, extra_data):
                                        (ABORT_PATTERN, handle_exception_reset),
                                        (FINISH_PATTERN, handle_test_finish),
                                        (UT_APP_BOOT_UP_DONE, handle_next_stage),
-                                       timeout=UT_TIMEOUT)
+                                       timeout=one_case["timeout"])
                     except ExpectTimeout:
                         Utility.console_log("Timeout in expect", color="orange")
                         one_case_finish(False)