From 74d768fe6ddbdcb794b200980aec72f5f6a4f3e2 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 4 Dec 2019 14:21:29 +0100 Subject: [PATCH] ci: updated mqtt weekend test for qemu support Added default sdkconfig for qemu build for the mqtt publish example, Added environment configuration for running the same test on target or in qemu Updated missing example tests per latest ttfw refactoring --- .../lwip/weekend_test/net_suite_test.py | 26 ++++---------- .../mqtt/weekend_test/mqtt_publish_test.py | 34 +++++++------------ .../weekend_test/test_weekend_mqtt_qemu.yml | 7 ++++ examples/get-started/blink/example_test.py | 17 +++------- .../mqtt/publish_test/sdkconfig.qemu | 19 +++++++++++ .../python_packages/ttfw_idf/requirements.txt | 3 ++ 6 files changed, 52 insertions(+), 54 deletions(-) create mode 100644 components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml create mode 100644 examples/protocols/mqtt/publish_test/sdkconfig.qemu create mode 100644 tools/ci/python_packages/ttfw_idf/requirements.txt diff --git a/components/lwip/weekend_test/net_suite_test.py b/components/lwip/weekend_test/net_suite_test.py index 59e763b44..41d95c8d6 100644 --- a/components/lwip/weekend_test/net_suite_test.py +++ b/components/lwip/weekend_test/net_suite_test.py @@ -1,27 +1,13 @@ import re import os -import sys import socket from threading import Thread, Event import subprocess import time from shutil import copyfile -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT -import Utility +from tiny_test_fw import Utility, DUT +import ttfw_idf stop_sock_listener = Event() stop_io_listener = Event() @@ -73,7 +59,7 @@ def sock_listener(dut1): sock = None -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def lwip_test_suite(env, extra_data): global stop_io_listener global stop_sock_listener @@ -84,12 +70,12 @@ def lwip_test_suite(env, extra_data): 3. Execute ttcn3 test suite 4. Collect result from ttcn3 """ - dut1 = env.get_dut("net_suite", "examples/system/network_tests", dut_class=ESP32DUT) + dut1 = env.get_dut("net_suite", "examples/system/network_tests", dut_class=ttfw_idf.ESP32DUT) # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "net_suite.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("net_suite", "{}KB".format(bin_size // 1024)) - IDF.check_performance("net_suite", bin_size // 1024) + ttfw_idf.log_performance("net_suite", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("net_suite", bin_size // 1024) dut1.start_app() thread1 = Thread(target=sock_listener, args=(dut1, )) thread2 = Thread(target=io_listener, args=(dut1, )) diff --git a/components/mqtt/weekend_test/mqtt_publish_test.py b/components/mqtt/weekend_test/mqtt_publish_test.py index 55e04074c..32ae3389f 100644 --- a/components/mqtt/weekend_test/mqtt_publish_test.py +++ b/components/mqtt/weekend_test/mqtt_publish_test.py @@ -11,20 +11,8 @@ import time import string import random -try: - import IDF - from IDF.IDFDUT import ESP32DUT -except ImportError: - # this is a test case write with tiny-test-fw. - # to run test cases outside tiny-test-fw, - # we need to set environment variable `TEST_FW_PATH`, - # then get and insert `TEST_FW_PATH` to sys path before import FW module - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - import IDF - -import DUT +from tiny_test_fw import DUT +import ttfw_idf event_client_connected = Event() @@ -53,6 +41,8 @@ def mqtt_client_task(client): def get_host_port_from_dut(dut1, config_option): value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option]) + if value is None: + return None, None return value.group(1), int(value.group(2)) @@ -124,7 +114,7 @@ def test_single_config(dut, transport, qos, repeat, published): event_stop_client.clear() -@IDF.idf_example_test(env_tag="Example_WIFI") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_weekend_mqtt_publish(env, extra_data): # Using broker url dictionary for different transport global broker_host @@ -138,13 +128,12 @@ def test_weekend_mqtt_publish(env, extra_data): 3. Test evaluates python client received correct qos0 message 4. Test ESP32 client received correct qos0 message """ - dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test", dut_class=ESP32DUT) + dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test") # check and log bin size binary_file = os.path.join(dut1.app.binary_path, "mqtt_publish.bin") bin_size = os.path.getsize(binary_file) - IDF.log_performance("mqtt_publish_bin_size", "{}KB" - .format(bin_size // 1024)) - IDF.check_performance("mqtt_publish_size", bin_size // 1024) + ttfw_idf.log_performance("mqtt_publish_bin_size", "{}KB".format(bin_size // 1024)) + ttfw_idf.check_performance("mqtt_publish_size", bin_size // 1024) # Look for host:port in sdkconfig try: # python client subscribes to the topic to which esp client publishes and vice versa @@ -159,13 +148,16 @@ def test_weekend_mqtt_publish(env, extra_data): raise dut1.start_app() try: - ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30) + ip_address = dut1.expect(re.compile(r" IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)"), timeout=30) print("Connected to AP with IP: {}".format(ip_address)) except DUT.ExpectTimeout: print('ENV_TEST_FAILURE: Cannot connect to AP') raise for qos in [0, 1, 2]: for transport in ["tcp", "ssl", "ws", "wss"]: + if broker_host[transport] is None: + print('Skipping transport: {}...'.format(transport)) + continue # simple test with empty message test_single_config(dut1, transport, qos, 0, 5) # decide on broker what level of test will pass (local broker works the best) @@ -189,4 +181,4 @@ def test_weekend_mqtt_publish(env, extra_data): if __name__ == '__main__': - test_weekend_mqtt_publish() + test_weekend_mqtt_publish(dut=ttfw_idf.ESP32QEMUDUT if sys.argv[1:] == ['qemu'] else ttfw_idf.ESP32DUT) diff --git a/components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml b/components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml new file mode 100644 index 000000000..e4a72ecd0 --- /dev/null +++ b/components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml @@ -0,0 +1,7 @@ +CaseConfig: +- name: test_weekend_mqtt_publish + overwrite: + dut: + class: ESP32QEMUDUT + package: ttfw_idf + diff --git a/examples/get-started/blink/example_test.py b/examples/get-started/blink/example_test.py index b00b033fc..10815cf08 100644 --- a/examples/get-started/blink/example_test.py +++ b/examples/get-started/blink/example_test.py @@ -7,17 +7,8 @@ import re import os import hashlib -try: - import IDF -except ImportError: - # This environment variable is expected on the host machine - test_fw_path = os.getenv("TEST_FW_PATH") - if test_fw_path and test_fw_path not in sys.path: - sys.path.insert(0, test_fw_path) - - import IDF -from IDF.IDFDUT import ESP32DUT, ESP32QEMUDUT -import Utility +from tiny_test_fw import Utility +import ttfw_idf def verify_elf_sha256_embedding(dut): @@ -37,9 +28,9 @@ def verify_elf_sha256_embedding(dut): raise ValueError('ELF file SHA256 mismatch') -@IDF.idf_example_test(env_tag="Example_QEMU") +@ttfw_idf.idf_example_test(env_tag="Example_WIFI") def test_examples_blink(env, extra_data): - dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ESP32QEMUDUT) + dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ttfw_idf.ESP32DUT) binary_file = os.path.join(dut.app.binary_path, "blink.bin") bin_size = os.path.getsize(binary_file) ttfw_idf.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024)) diff --git a/examples/protocols/mqtt/publish_test/sdkconfig.qemu b/examples/protocols/mqtt/publish_test/sdkconfig.qemu new file mode 100644 index 000000000..e0d289d74 --- /dev/null +++ b/examples/protocols/mqtt/publish_test/sdkconfig.qemu @@ -0,0 +1,19 @@ +CONFIG_IDF_TARGET_ESP32=y +CONFIG_EXAMPLE_USE_OPENETH=y +CONFIG_ETH_USE_OPENETH=y +CONFIG_ETH_OPENETH_DMA_RX_BUFFER_NUM=4 +CONFIG_ETH_OPENETH_DMA_TX_BUFFER_NUM=1 +CONFIG_EXAMPLE_CONNECT_ETHERNET=y +CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y +CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y +CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y +CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN=16384 +CONFIG_MBEDTLS_SSL_OUT_CONTENT_LEN=16384 +CONFIG_EXAMPLE_BROKER_SSL_URI="mqtts://${EXAMPLE_MQTT_BROKER_SSL}" +CONFIG_EXAMPLE_BROKER_TCP_URI="mqtt://${EXAMPLE_MQTT_BROKER_TCP}" +CONFIG_EXAMPLE_BROKER_WS_URI="ws://${EXAMPLE_MQTT_BROKER_WS}/ws" +CONFIG_EXAMPLE_BROKER_WSS_URI="wss://${EXAMPLE_MQTT_BROKER_WSS}/ws" +CONFIG_EXAMPLE_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}" +CONFIG_MBEDTLS_HARDWARE_AES=n +CONFIG_MBEDTLS_HARDWARE_MPI=n +CONFIG_MBEDTLS_HARDWARE_SHA=n diff --git a/tools/ci/python_packages/ttfw_idf/requirements.txt b/tools/ci/python_packages/ttfw_idf/requirements.txt new file mode 100644 index 000000000..112ee0bc8 --- /dev/null +++ b/tools/ci/python_packages/ttfw_idf/requirements.txt @@ -0,0 +1,3 @@ +-r ../tiny_test_fw/requirements.txt +pexpect +python-gitlab