Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'test/unit_test_multi_devices_case_pass_param_by_signals' into 'master'

Browse source 
unit-test-app: support passing parameter with signals

See merge request idf/esp-idf!3960
This commit is contained in:
Ivan Grokhotkov 2019-01-11 15:23:19 +08:00
commit a3e3d00cf0
6 changed files with 313 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

133
components/esp32/test/test_wifi.c
View file

@ -1,6 +1,7 @@
/*
Tests for the Wi-Fi
*/
#include "string.h"
#include "esp_system.h"
#include "unity.h"
#include "esp_system.h"
@ -11,12 +12,22 @@
#include "nvs_flash.h"
#include "test_utils.h"
#include "freertos/task.h"
#include "freertos/event_groups.h"
static const char* TAG = "test_wifi";
#define DEFAULT_SSID "TEST_SSID"
#define DEFAULT_PWD "TEST_PASS"
#define GOT_IP_EVENT 0x00000001
#define DISCONNECT_EVENT 0x00000002
#define EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT 0x00000001
static uint32_t wifi_event_handler_flag;
static EventGroupHandle_t wifi_events;
static esp_err_t event_handler(void *ctx, system_event_t *event)
{
printf("ev_handle_called.\n");
@ -30,10 +41,18 @@ static esp_err_t event_handler(void *ctx, system_event_t *event)
ESP_LOGI(TAG, "SYSTEM_EVENT_STA_GOT_IP");
ESP_LOGI(TAG, "got ip:%s\n",
ip4addr_ntoa(&event->event_info.got_ip.ip_info.ip));
if (wifi_events) {
xEventGroupSetBits(wifi_events, GOT_IP_EVENT);
}
break;
case SYSTEM_EVENT_STA_DISCONNECTED:
ESP_LOGI(TAG, "SYSTEM_EVENT_STA_DISCONNECTED");
TEST_ESP_OK(esp_wifi_connect());
if (! (EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT & wifi_event_handler_flag) ) {
TEST_ESP_OK(esp_wifi_connect());
}
if (wifi_events) {
xEventGroupSetBits(wifi_events, DISCONNECT_EVENT);
}
break;
default:
break;
@ -129,8 +148,8 @@ static void start_wifi_as_softap(void)
cfg.nvs_enable = false;
wifi_config_t w_config = {
.ap.ssid = "default_ssid",
.ap.password = "default_password",
.ap.ssid = DEFAULT_SSID,
.ap.password = DEFAULT_PWD,
.ap.ssid_len = 0,
.ap.channel = 1,
.ap.authmode = WIFI_AUTH_WPA2_PSK,
@ -139,21 +158,60 @@ static void start_wifi_as_softap(void)
.ap.beacon_interval = 100,
};
TEST_ESP_OK(esp_event_loop_init(event_handler, NULL));
// can't deinit event loop, need to reset leak check
unity_reset_leak_checks();
if (wifi_events == NULL) {
wifi_events = xEventGroupCreate();
}
TEST_ESP_OK(esp_wifi_init(&cfg));
TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_AP));
TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_AP, &w_config));
TEST_ESP_OK(esp_wifi_start());
}
static void start_wifi_as_sta(void)
{
wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
cfg.nvs_enable = false;
// do not auto connect
wifi_event_handler_flag |= EVENT_HANDLER_FLAG_DO_NOT_AUTO_RECONNECT;
TEST_ESP_OK(esp_event_loop_init(event_handler, NULL));
// can't deinit event loop, need to reset leak check
unity_reset_leak_checks();
if (wifi_events == NULL) {
wifi_events = xEventGroupCreate();
} else {
xEventGroupClearBits(wifi_events, 0x00ffffff);
}
TEST_ESP_OK(esp_wifi_init(&cfg));
TEST_ESP_OK(esp_wifi_set_mode(WIFI_MODE_STA));
TEST_ESP_OK(esp_wifi_start());
}
static void stop_wifi(void)
{
printf("stop wifi\n");
TEST_ESP_OK(esp_wifi_stop());
TEST_ESP_OK(esp_wifi_deinit());
if (wifi_events) {
vEventGroupDelete(wifi_events);
wifi_events = NULL;
}
vTaskDelay(1000/portTICK_PERIOD_MS);
}
static void receive_ds2ds_packet(void)
{
test_case_uses_tcpip();
start_wifi_as_softap();
unity_wait_for_signal("sender ready");
unity_send_signal("receiver ready");
@ -161,7 +219,6 @@ static void receive_ds2ds_packet(void)
// wait for sender to send packets
vTaskDelay(1000/portTICK_PERIOD_MS);
stop_wifi();
}
static const char ds2ds_pdu[] = {
@ -174,6 +231,7 @@ static const char ds2ds_pdu[] = {
static void send_ds2ds_packet(void)
{
test_case_uses_tcpip();
start_wifi_as_softap();
unity_send_signal("sender ready");
unity_wait_for_signal("receiver ready");
@ -187,3 +245,70 @@ static void send_ds2ds_packet(void)
}
TEST_CASE_MULTIPLE_DEVICES("receive ds2ds packet without exception", "[wifi][test_env=UT_T2_1]", receive_ds2ds_packet, send_ds2ds_packet);
static void wifi_connect_by_bssid(uint8_t *bssid)
{
EventBits_t bits;
wifi_config_t w_config = {
.sta.ssid = DEFAULT_SSID,
.sta.password = DEFAULT_PWD,
.sta.bssid_set = true,
};
memcpy(w_config.sta.bssid, bssid, 6);
TEST_ESP_OK(esp_wifi_set_config(WIFI_IF_STA, &w_config));
TEST_ESP_OK(esp_wifi_connect());
bits = xEventGroupWaitBits(wifi_events, GOT_IP_EVENT, 1, 0, 5000/portTICK_RATE_MS);
TEST_ASSERT(bits == GOT_IP_EVENT);
}
static void test_wifi_connection_sta(void)
{
char mac_str[19];
uint8_t mac[6];
EventBits_t bits;
test_case_uses_tcpip();
start_wifi_as_sta();
unity_wait_for_signal_param("SoftAP mac", mac_str, 19);
TEST_ASSERT_TRUE(unity_util_convert_mac_from_string(mac_str, mac));
wifi_connect_by_bssid(mac);
unity_send_signal("STA connected");
bits = xEventGroupWaitBits(wifi_events, DISCONNECT_EVENT, 1, 0, 60000 / portTICK_RATE_MS);
// disconnect event not triggered
printf("wait finish\n");
TEST_ASSERT(bits == 0);
stop_wifi();
}
static void test_wifi_connection_softap(void)
{
char mac_str[19] = {0};
uint8_t mac[6];
test_case_uses_tcpip();
start_wifi_as_softap();
TEST_ESP_OK(esp_wifi_get_mac(ESP_IF_WIFI_AP, mac));
sprintf(mac_str, MACSTR, MAC2STR(mac));
unity_send_signal_param("SoftAP mac", mac_str);
unity_wait_for_signal("STA connected");
vTaskDelay(60000 / portTICK_PERIOD_MS);
stop_wifi();
}
TEST_CASE_MULTIPLE_DEVICES("test wifi retain connection for 60s", "[wifi][test_env=UT_T2_1][timeout=90]", test_wifi_connection_sta, test_wifi_connection_softap);

26
docs/en/api-guides/unit-tests.rst
View file

@ -75,13 +75,27 @@ As the secnario in the above example, slave should get GPIO level after master s
DUT1 (master) console::
Waiting for signal: [output high level]!
Please press "Enter" key to once any board send this signal.
Please press "Enter" key once any board send this signal.
DUT2 (slave) console::
Send signal: [output high level]!
Once the signal is set from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level.
Once the signal is sent from DUT2, you need to press "Enter" on DUT1, then DUT1 unblocks from ``unity_wait_for_signal`` and starts to change GPIO level.
Signals can also be used to pass parameters between multiple devices. For example, DUT1 want to know the MAC address of DUT2, so it can connect to DUT2.
In this case, ``unity_wait_for_signal_param`` and ``unity_send_signal_param`` can be used:
DUT1 console::
Waiting for signal: [dut2 mac address]!
Please input parameter value from any board send this signal and press "Enter" key.
DUT2 console::
Send signal: [dut2 mac address][10:20:30:40:50:60]!
Once the signal is sent from DUT2, you need to input ``10:20:30:40:50:60`` on DUT1 and press "Enter". Then DUT1 will get the MAC address string of DUT2 and unblocks from ``unity_wait_for_signal_param``, start to connect to DUT2.
Add multiple stages test cases
@ -165,13 +179,13 @@ Normal case will print the case name and description. Master slave cases will al
Test cases can be run by inputting one of the following:
- Test case name in quotation marks to run a single test case
- Test case name in quotation marks (for example, ``"esp_ota_begin() verifies arguments"``) to run a single test case.
- Test case index to run a single test case
- Test case index (for example, ``1``) to run a single test case.
- Module name in square brackets to run all test cases for a specific module
- Module name in square brackets (for example, ``[cxx]``) to run all test cases for a specific module.
- An asterisk to run all test cases
- An asterisk (``*``) to run all test cases
``[multi_device]`` and ``[multi_stage]`` tags tell the test runner whether a test case is a multiple devices or multiple stages test case.
These tags are automatically added by ```TEST_CASE_MULTIPLE_STAGES`` and ``TEST_CASE_MULTIPLE_DEVICES`` macros.

23
docs/zh_CN/api-guides/unit-tests.rst
View file

@ -109,6 +109,21 @@ DUT2（slave）终端：
一旦 DUT2 发送了该信号，您需要在 DUT2 的终端输入回车，然后 DUT1 会从
``unity_wait_for_signal`` 函数中解除阻塞，并开始更改 GPIO 的电平。
信号也可以用来在不同 DUT 之间传递参数。例如，DUT1 希望能够拿到 DUT2 的 MAC 地址，来进行蓝牙连接。
这时，我们可以使用 ``unity_wait_for_signal_param`` 以及 ``unity_send_signal_param``。
DUT1 终端::
Waiting for signal: [dut2 mac address]!
Please input parameter value from any board send this signal and press "Enter" key.
DUT2 终端::
Send signal: [dut2 mac address][10:20:30:40:50:60]!
一旦 DUT2 发送信号，您需要在 DUT1 输入 ``10:20:30:40:50:60`` 及回车，然后 DUT1 会从 ``unity_wait_for_signal_param`` 中获取到蓝牙地址的字符串，并解除阻塞开始蓝牙连接。
添加多阶段测试用例
------------------
@ -203,13 +218,13 @@ DUT2（slave）终端：
可以输入以下任意一项来运行测试用例：
- 引号中的测试用例的名字，运行单个测试用例。
- 引号中的测试用例的名字（例如 ``"esp_ota_begin() verifies arguments"``），运行单个测试用例。
- 测试用例的序号，运行单个测试用例。
- 测试用例的序号（例如 ``1``），运行单个测试用例。
- 方括号中的模块名字，运行指定模块所有的测试用例。
- 方括号中的模块名字（例如 ``[cxx]``），运行指定模块所有的测试用例。
- 星号，运行所有测试用例。
- 星号 (``*``)，运行所有测试用例。
``[multi_device]`` 和 ``[multi_stage]``
标签告诉测试运行者该用例是多设备测试还是多阶段测试。这些标签由

60
tools/unit-test-app/components/test_utils/include/test_utils.h
View file

@ -107,9 +107,8 @@ void unity_reset_leak_checks(void);
*/
void test_case_uses_tcpip(void);
/**
* @brief wait for signals.
* @brief wait for signals with parameters.
*
* for multiple devices test cases, DUT might need to wait for other DUTs before continue testing.
* As all DUTs are independent, need user (or test script) interaction to make test synchronized.
@ -136,13 +135,64 @@ void test_case_uses_tcpip(void);
*
* Then we press Enter key on DUT1's console, DUT1 starts to read input and then test success.
*
* Another example, we have 2 DUTs in multiple devices test, and DUT1 need to get DUT2's mac address to perform BT connection.
* DUT1 should call `unity_wait_for_signal_param("dut2 mac address", mac, 19);` to wait for DUT2's mac address.
* DUT2 should call `unity_send_signal_param("dut2 mac address", "10:20:30:40:50:60");` to send to DUT1 its mac address.
* According to the console logs:
*
* DUT1 console:
*
* ```
* Waiting for signal: [dut2 mac address]!
* Please input parameter value from any board send this signal and press "Enter" key.
* ```
*
* DUT2 console:
*
* ```
* Send signal: [dut2 mac address][10:20:30:40:50:60]!
* ```
*
* @param signal_name signal name which DUT expected to wait before proceed testing
* @param parameter_buf buffer to receive parameter
* @param buf_len length of parameter_buf.
* Currently we have a limitation that it will write 1 extra byte at the end of string.
* We need to use a buffer with 2 bytes longer than actual string length.
*/
void unity_wait_for_signal(const char* signal_name);
void unity_wait_for_signal_param(const char* signal_name, char *parameter_buf, uint8_t buf_len);
/**
* @brief DUT send signal.
* @brief wait for signals.
*
* @param signal_name signal name which DUT expected to wait before proceed testing
*/
static inline void unity_wait_for_signal(const char* signal_name)
{
unity_wait_for_signal_param(signal_name, NULL, 0);
}
/**
* @brief DUT send signal and pass parameter to other devices.
*
* @param signal_name signal name which DUT send once it finished preparing.
* @param parameter a string to let remote device to receive.
*/
void unity_send_signal_param(const char* signal_name, const char *parameter);
/**
* @brief DUT send signal with parameter.
*
* @param signal_name signal name which DUT send once it finished preparing.
*/
void unity_send_signal(const char* signal_name);
static inline void unity_send_signal(const char* signal_name)
{
unity_send_signal_param(signal_name, NULL);
}
/**
* @brief convert mac string to mac address
*
* @param mac_str MAC address string with format "xx:xx:xx:xx:xx:xx"
* @param[out] mac_addr store converted MAC address
*/
bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr);

70
tools/unit-test-app/components/test_utils/test_utils.c
View file

@ -31,20 +31,6 @@ const esp_partition_t *get_test_data_partition()
return result;
}
// wait user to send "Enter" key
static void wait_user_control()
{
char sign[5] = {0};
while(strlen(sign) == 0)
{
/* Flush anything already in the RX buffer */
while(uart_rx_one_char((uint8_t *) sign) == OK) {
}
/* Read line */
UartRxString((uint8_t*) sign, sizeof(sign) - 1);
}
}
void test_case_uses_tcpip()
{
// Can be called more than once, does nothing on subsequent calls
@ -71,16 +57,60 @@ void test_case_uses_tcpip()
unity_reset_leak_checks();
}
// wait user to send "Enter" key or input parameter
static void wait_user_control(char* parameter_buf, uint8_t buf_len)
{
char *buffer = parameter_buf;
char sign[5];
uint8_t buffer_len = buf_len - 1;
if (parameter_buf == NULL) {
buffer = sign;
buffer_len = sizeof(sign) - 1;
}
// workaround that unity_gets (UartRxString) will not set '\0' correctly
bzero(buffer, buffer_len);
unity_gets(buffer, buffer_len);
}
// signal functions, used for sync between unity DUTs for multiple devices cases
void unity_wait_for_signal(const char* signal_name)
void unity_wait_for_signal_param(const char* signal_name, char* parameter_buf, uint8_t buf_len)
{
printf("Waiting for signal: [%s]!\n"
"Please press \"Enter\" key to once any board send this signal.\n", signal_name);
wait_user_control();
printf("Waiting for signal: [%s]!\n", signal_name);
if (parameter_buf == NULL) {
printf("Please press \"Enter\" key once any board send this signal.\n");
} else {
printf("Please input parameter value from any board send this signal and press \"Enter\" key.\n");
}
wait_user_control(parameter_buf, buf_len);
}
void unity_send_signal(const char* signal_name)
void unity_send_signal_param(const char* signal_name, const char *parameter)
{
printf("Send signal: [%s]!\n", signal_name);
if (parameter == NULL) {
printf("Send signal: [%s]!\n", signal_name);
} else {
printf("Send signal: [%s][%s]!\n", signal_name, parameter);
}
}
bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr)
{
uint8_t loop = 0;
uint8_t tmp = 0;
const char *start;
char *stop;
for (loop = 0; loop < 6; loop++) {
start = mac_str + loop * 3;
tmp = strtol(start, &stop, 16);
if (stop - start == 2 && (*stop == ':' || (*stop == 0 && loop == 5))) {
mac_addr[loop] = tmp;
} else {
return false;
}
}
return true;
}

64
tools/unit-test-app/unit_test.py
View file

@ -64,7 +64,11 @@ MULTI_DEVICE_ID = 2
DEFAULT_TIMEOUT = 20
DUT_STARTUP_CHECK_RETRY_COUNT = 5
TEST_HISTROY_CHECK_TIMEOUT = 1
TEST_HISTORY_CHECK_TIMEOUT = 1
class TestCaseFailed(AssertionError):
pass
def format_test_case_config(test_case_data):
@ -163,7 +167,7 @@ def reset_dut(dut):
for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
dut.write("-")
try:
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTORY_CHECK_TIMEOUT)
break
except ExpectTimeout:
pass
@ -171,7 +175,7 @@ def reset_dut(dut):
raise AssertionError("Reset {} ({}) failed!".format(dut.name, dut.port))
def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
def run_one_normal_case(dut, one_case, junit_test_case):
reset_dut(dut)
@ -194,9 +198,9 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
if result:
Utility.console_log("Success: " + one_case["name"], color="green")
else:
failed_cases.append(one_case["name"])
Utility.console_log("Failed: " + one_case["name"], color="red")
junit_test_case.add_failure_info(output)
raise TestCaseFailed()
def handle_exception_reset(data):
"""
@ -242,7 +246,7 @@ def run_one_normal_case(dut, one_case, junit_test_case, failed_cases):
timeout=one_case["timeout"])
except ExpectTimeout:
Utility.console_log("Timeout in expect", color="orange")
junit_test_case.add_error_info("timeout")
junit_test_case.add_failure_info("timeout")
one_case_finish(False)
break
@ -284,10 +288,12 @@ def run_unit_test_cases(env, extra_data):
# create junit report test case
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try:
run_one_normal_case(dut, one_case, junit_test_case, failed_cases)
TinyFW.JunitReport.test_case_finish(junit_test_case)
run_one_normal_case(dut, one_case, junit_test_case)
except TestCaseFailed:
failed_cases.append(one_case["name"])
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails
@ -300,8 +306,8 @@ def run_unit_test_cases(env, extra_data):
class Handler(threading.Thread):
WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)\]!')
SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[(.+)\]!')
WAIT_SIGNAL_PATTERN = re.compile(r'Waiting for signal: \[(.+)]!')
SEND_SIGNAL_PATTERN = re.compile(r'Send signal: \[([^]]+)](\[([^]]+)])?!')
FINISH_PATTERN = re.compile(r"1 Tests (\d) Failures (\d) Ignored")
def __init__(self, dut, sent_signal_list, lock, parent_case_name, child_case_index, timeout):
@ -348,15 +354,23 @@ class Handler(threading.Thread):
Utility.console_log("Timeout in device for function: %s" % self.child_case_name, color="orange")
break
with self.lock:
if expected_signal in self.sent_signal_list:
self.dut.write(" ")
self.sent_signal_list.remove(expected_signal)
break
time.sleep(0.01)
for sent_signal in self.sent_signal_list:
if expected_signal == sent_signal["name"]:
self.dut.write(sent_signal["parameter"])
self.sent_signal_list.remove(sent_signal)
break
else:
time.sleep(0.01)
continue
break
def device_send_action(data):
with self.lock:
self.sent_signal_list.append(data[0].encode('utf-8'))
self.sent_signal_list.append({
"name": data[0].encode('utf-8'),
"parameter": "" if data[2] is None else data[2].encode('utf-8')
# no parameter means we only write EOL to DUT
})
def handle_device_test_finish(data):
""" test finished without reset """
@ -468,9 +482,9 @@ def run_multiple_devices_cases(env, extra_data):
try:
run_one_multiple_devices_case(duts, ut_config, env, one_case, failed_cases,
one_case.get('app_bin'), junit_test_case)
TinyFW.JunitReport.test_case_finish(junit_test_case)
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
if failed_cases:
@ -480,7 +494,7 @@ def run_multiple_devices_cases(env, extra_data):
raise AssertionError("Unit Test Failed")
def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
def run_one_multiple_stage_case(dut, one_case, junit_test_case):
reset_dut(dut)
dut.start_capture_raw_data()
@ -515,7 +529,7 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
err_msg = "Reset Check Failed: \r\n\tExpected: {}\r\n\tGet: {}".format(one_case["reset"],
exception_reset_list)
Utility.console_log(err_msg, color="orange")
junit_test_case.add_error_info(err_msg)
junit_test_case.add_failure_info(err_msg)
else:
# we allow omit reset in multi stage cases
result = True
@ -530,9 +544,9 @@ def run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case):
if result:
Utility.console_log("Success: " + one_case["name"], color="green")
else:
failed_cases.append(one_case["name"])
Utility.console_log("Failed: " + one_case["name"], color="red")
junit_test_case.add_failure_info(output)
raise TestCaseFailed()
stage_finish.append("break")
def handle_exception_reset(data):
@ -612,10 +626,12 @@ def run_multiple_stage_cases(env, extra_data):
for one_case in case_config[ut_config]:
junit_test_case = TinyFW.JunitReport.create_test_case("[{}] {}".format(ut_config, one_case["name"]))
try:
run_one_multiple_stage_case(dut, one_case, failed_cases, junit_test_case)
TinyFW.JunitReport.test_case_finish(junit_test_case)
run_one_multiple_stage_case(dut, one_case, junit_test_case)
except TestCaseFailed:
failed_cases.append(one_case["name"])
except Exception as e:
junit_test_case.add_error_info("Unexpected exception: " + str(e))
junit_test_case.add_failure_info("Unexpected exception: " + str(e))
finally:
TinyFW.JunitReport.test_case_finish(junit_test_case)
# raise exception if any case fails