ci: updated mqtt weekend test for qemu support

Added default sdkconfig for qemu build for the mqtt publish example,
Added environment configuration for running the same test on target
or in qemu
Updated missing example tests per latest ttfw refactoring

components/lwip/weekend_test/net_suite_test.py

@@ -1,27 +1,13 @@
 import re
 import os
-import sys
 import socket
 from threading import Thread, Event
 import subprocess
 import time
 from shutil import copyfile
 
-try:
+from tiny_test_fw import Utility, DUT
-    import IDF
+import ttfw_idf
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
-import Utility
 
 stop_sock_listener = Event()
 stop_io_listener = Event()
@@ -73,7 +59,7 @@ def sock_listener(dut1):
         sock = None
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def lwip_test_suite(env, extra_data):
     global stop_io_listener
     global stop_sock_listener
@@ -84,12 +70,12 @@ def lwip_test_suite(env, extra_data):
       3. Execute ttcn3 test suite
       4. Collect result from ttcn3
     """
-    dut1 = env.get_dut("net_suite", "examples/system/network_tests", dut_class=ESP32DUT)
+    dut1 = env.get_dut("net_suite", "examples/system/network_tests", dut_class=ttfw_idf.ESP32DUT)
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "net_suite.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("net_suite", "{}KB".format(bin_size // 1024))
+    ttfw_idf.log_performance("net_suite", "{}KB".format(bin_size // 1024))
-    IDF.check_performance("net_suite", bin_size // 1024)
+    ttfw_idf.check_performance("net_suite", bin_size // 1024)
     dut1.start_app()
     thread1 = Thread(target=sock_listener, args=(dut1, ))
     thread2 = Thread(target=io_listener, args=(dut1, ))

components/mqtt/weekend_test/mqtt_publish_test.py

@@ -11,20 +11,8 @@ import time
 import string
 import random
 
-try:
+from tiny_test_fw import DUT
-    import IDF
+import ttfw_idf
-    from IDF.IDFDUT import ESP32DUT
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
 
 
 event_client_connected = Event()
@@ -53,6 +41,8 @@ def mqtt_client_task(client):
 
 def get_host_port_from_dut(dut1, config_option):
     value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option])
+    if value is None:
+        return None, None
     return value.group(1), int(value.group(2))
 
 
@@ -124,7 +114,7 @@ def test_single_config(dut, transport, qos, repeat, published):
     event_stop_client.clear()
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_weekend_mqtt_publish(env, extra_data):
     # Using broker url dictionary for different transport
     global broker_host
@@ -138,13 +128,12 @@ def test_weekend_mqtt_publish(env, extra_data):
       3. Test evaluates python client received correct qos0 message
       4. Test ESP32 client received correct qos0 message
     """
-    dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test", dut_class=ESP32DUT)
+    dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test")
     # check and log bin size
     binary_file = os.path.join(dut1.app.binary_path, "mqtt_publish.bin")
     bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_publish_bin_size", "{}KB"
+    ttfw_idf.log_performance("mqtt_publish_bin_size", "{}KB".format(bin_size // 1024))
-                        .format(bin_size // 1024))
+    ttfw_idf.check_performance("mqtt_publish_size", bin_size // 1024)
-    IDF.check_performance("mqtt_publish_size", bin_size // 1024)
     # Look for host:port in sdkconfig
     try:
         # python client subscribes to the topic to which esp client publishes and vice versa
@@ -159,13 +148,16 @@ def test_weekend_mqtt_publish(env, extra_data):
         raise
     dut1.start_app()
     try:
-        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        ip_address = dut1.expect(re.compile(r" IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)"), timeout=30)
         print("Connected to AP with IP: {}".format(ip_address))
     except DUT.ExpectTimeout:
         print('ENV_TEST_FAILURE: Cannot connect to AP')
         raise
     for qos in [0, 1, 2]:
         for transport in ["tcp", "ssl", "ws", "wss"]:
+            if broker_host[transport] is None:
+                print('Skipping transport: {}...'.format(transport))
+                continue
             # simple test with empty message
             test_single_config(dut1, transport, qos, 0, 5)
             # decide on broker what level of test will pass (local broker works the best)
@@ -189,4 +181,4 @@ def test_weekend_mqtt_publish(env, extra_data):
 
 
 if __name__ == '__main__':
-    test_weekend_mqtt_publish()
+    test_weekend_mqtt_publish(dut=ttfw_idf.ESP32QEMUDUT if sys.argv[1:] == ['qemu'] else ttfw_idf.ESP32DUT)

components/mqtt/weekend_test/test_weekend_mqtt_qemu.yml

@@ -0,0 +1,7 @@
+CaseConfig:
+- name: test_weekend_mqtt_publish
+  overwrite:
+    dut:
+      class: ESP32QEMUDUT
+      package: ttfw_idf
+

examples/get-started/blink/example_test.py

@@ -7,17 +7,8 @@ import re
 import os
 import hashlib
 
-try:
+from tiny_test_fw import Utility
-    import IDF
+import ttfw_idf
-except ImportError:
-    # This environment variable is expected on the host machine
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-
-    import IDF
-from IDF.IDFDUT import ESP32DUT, ESP32QEMUDUT
-import Utility
 
 
 def verify_elf_sha256_embedding(dut):
@@ -37,9 +28,9 @@ def verify_elf_sha256_embedding(dut):
         raise ValueError('ELF file SHA256 mismatch')
 
 
-@IDF.idf_example_test(env_tag="Example_QEMU")
+@ttfw_idf.idf_example_test(env_tag="Example_WIFI")
 def test_examples_blink(env, extra_data):
-    dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ESP32QEMUDUT)
+    dut = env.get_dut("blink", "examples/get-started/blink", dut_class=ttfw_idf.ESP32DUT)
     binary_file = os.path.join(dut.app.binary_path, "blink.bin")
     bin_size = os.path.getsize(binary_file)
     ttfw_idf.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024))

examples/protocols/mqtt/publish_test/sdkconfig.qemu

@@ -0,0 +1,19 @@
+CONFIG_IDF_TARGET_ESP32=y
+CONFIG_EXAMPLE_USE_OPENETH=y
+CONFIG_ETH_USE_OPENETH=y
+CONFIG_ETH_OPENETH_DMA_RX_BUFFER_NUM=4
+CONFIG_ETH_OPENETH_DMA_TX_BUFFER_NUM=1
+CONFIG_EXAMPLE_CONNECT_ETHERNET=y
+CONFIG_LOG_DEFAULT_LEVEL_DEBUG=y
+CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y
+CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y
+CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN=16384
+CONFIG_MBEDTLS_SSL_OUT_CONTENT_LEN=16384
+CONFIG_EXAMPLE_BROKER_SSL_URI="mqtts://${EXAMPLE_MQTT_BROKER_SSL}"
+CONFIG_EXAMPLE_BROKER_TCP_URI="mqtt://${EXAMPLE_MQTT_BROKER_TCP}"
+CONFIG_EXAMPLE_BROKER_WS_URI="ws://${EXAMPLE_MQTT_BROKER_WS}/ws"
+CONFIG_EXAMPLE_BROKER_WSS_URI="wss://${EXAMPLE_MQTT_BROKER_WSS}/ws"
+CONFIG_EXAMPLE_BROKER_CERTIFICATE_OVERRIDE="${EXAMPLE_MQTT_BROKER_CERTIFICATE}"
+CONFIG_MBEDTLS_HARDWARE_AES=n
+CONFIG_MBEDTLS_HARDWARE_MPI=n
+CONFIG_MBEDTLS_HARDWARE_SHA=n

tools/ci/python_packages/ttfw_idf/requirements.txt

@@ -0,0 +1,3 @@
+-r ../tiny_test_fw/requirements.txt
+pexpect
+python-gitlab