Merge branch 'test/fix_unit_test_job_pass_even_no_test_executed' into 'master'

test: fix CI UT job pass even ut app bootup fail

See merge request idf/esp-idf!4208

components/driver/test/test_spi_param.c

@@ -465,9 +465,9 @@ static const ptest_func_t slave_test_func = {
     .def_param = spitest_def_param,
 };
 
-#define TEST_SPI_MASTER_SLAVE(name, param_group) \
+#define TEST_SPI_MASTER_SLAVE(name, param_group, extra_tag) \
     PARAM_GROUP_DECLARE(name, param_group) \
-    TEST_MASTER_SLAVE(name, param_group, "[spi_ms][test_env=Example_SPI_Multi_device][timeout=120]", &master_test_func, &slave_test_func)
+    TEST_MASTER_SLAVE(name, param_group, "[spi_ms][test_env=Example_SPI_Multi_device][timeout=120]"#extra_tag, &master_test_func, &slave_test_func)
 
 /************ Master Code ***********************************************/
 static void test_master_init(void** arg)
@@ -789,7 +789,7 @@ static spitest_param_set_t timing_conf[] = {
         .slave_tv_ns = TV_WITH_ESP_SLAVE_GPIO,
     },
 };
-TEST_SPI_MASTER_SLAVE(TIMING, timing_conf)
+TEST_SPI_MASTER_SLAVE(TIMING, timing_conf, "")
 
 /************ Mode Test ***********************************************/
 #define FREQ_LIMIT_MODE SPI_MASTER_FREQ_16M
@@ -976,4 +976,4 @@ spitest_param_set_t mode_conf[] = {
       .slave_dma_chan = 1,
     },
 };
-TEST_SPI_MASTER_SLAVE(MODE, mode_conf)
+TEST_SPI_MASTER_SLAVE(MODE, mode_conf, "[ignore]")

tools/unit-test-app/partition_table_unit_test_app.csv

@@ -5,7 +5,7 @@
 nvs,        data, nvs,      0x9000,  0x4000
 otadata,    data, ota,      0xd000,  0x2000
 phy_init,   data, phy,      0xf000,  0x1000
-factory,    0,    0,        0x10000, 0x240000
+factory,    0,    0,        0x10000, 0x250000
 # these OTA partitions are used for tests, but can't fit real OTA apps in them
 # (done this way to reduce total flash usage.)
 ota_0,      0,    ota_0,    ,        64K

tools/unit-test-app/unit_test.py

@@ -293,6 +293,7 @@ def run_unit_test_cases(env, extra_data):
                 failed_cases.append(one_case["name"])
             except Exception as e:
                 junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+                failed_cases.append(one_case["name"])
             finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
@@ -420,7 +421,7 @@ def get_dut(duts, env, name, ut_config, app_bin=None):
     return dut
 
 
-def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases, app_bin, junit_test_case):
+def run_one_multiple_devices_case(duts, ut_config, env, one_case, app_bin, junit_test_case):
     lock = threading.RLock()
     threads = []
     send_signal_list = []
@@ -442,12 +443,9 @@ def run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
         if not thread.result:
             [thd.stop() for thd in threads]
 
-    if result:
+    if not result:
-        Utility.console_log("Success: " + one_case["name"], color="green")
-    else:
-        failed_cases.append(one_case["name"])
         junit_test_case.add_failure_info(output)
-        Utility.console_log("Failed: " + one_case["name"], color="red")
+    return result
 
 
 @IDF.idf_unit_test(env_tag="UT_T2_1", junit_report_by_case=True)
@@ -478,13 +476,19 @@ def run_multiple_devices_cases(env, extra_data):
     for ut_config in case_config:
         Utility.console_log("Running unit test for config: " + ut_config, "O")
         for one_case in case_config[ut_config]:
+            result = False
             junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
             try:
-                run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
+                result = run_one_multiple_devices_case(duts, ut_config, env, one_case,
-                                              one_case.get('app_bin'), junit_test_case)
+                                                       one_case.get('app_bin'), junit_test_case)
             except Exception as e:
                 junit_test_case.add_failure_info("Unexpected exception: " + str(e))
             finally:
+                if result:
+                    Utility.console_log("Success: " + one_case["name"], color="green")
+                else:
+                    failed_cases.append(one_case["name"])
+                    Utility.console_log("Failed: " + one_case["name"], color="red")
                 TinyFW.JunitReport.test_case_finish(junit_test_case)
 
     if failed_cases:
@@ -631,6 +635,7 @@ def run_multiple_stage_cases(env, extra_data):
                 failed_cases.append(one_case["name"])
             except Exception as e:
                 junit_test_case.add_failure_info("Unexpected exception: " + str(e))
+                failed_cases.append(one_case["name"])
             finally:
                 TinyFW.JunitReport.test_case_finish(junit_test_case)