Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add multi-target support for performance tests

Browse source
This commit is contained in:
Roland Dobai 2020-02-28 16:12:11 +01:00
parent 37997407df
commit 15884eccf2
36 changed files with 173 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
components/driver/test/test_spi_master.c
View file

@ -1031,7 +1031,7 @@ TEST_CASE("spi_speed","[spi]")
for (int i = 0; i < TEST_TIMES; i++) {
ESP_LOGI(TAG, "%.2lf", GET_US_BY_CCOUNT(t_flight_sorted[i]));
}
TEST_TARGET_PERFORMANCE_LESS_THAN(SPI_PER_TRANS_NO_POLLING, "%d us", (int)GET_US_BY_CCOUNT(t_flight_sorted[(TEST_TIMES+1)/2]));
TEST_PERFORMANCE_LESS_THAN(SPI_PER_TRANS_NO_POLLING, "%d us", (int)GET_US_BY_CCOUNT(t_flight_sorted[(TEST_TIMES+1)/2]));
//acquire the bus to send polling transactions faster
ret = spi_device_acquire_bus(spi, portMAX_DELAY);
@ -1064,7 +1064,7 @@ TEST_CASE("spi_speed","[spi]")
for (int i = 0; i < TEST_TIMES; i++) {
ESP_LOGI(TAG, "%.2lf", GET_US_BY_CCOUNT(t_flight_sorted[i]));
}
TEST_TARGET_PERFORMANCE_LESS_THAN(SPI_PER_TRANS_NO_POLLING_NO_DMA, "%d us", (int)GET_US_BY_CCOUNT(t_flight_sorted[(TEST_TIMES+1)/2]));
TEST_PERFORMANCE_LESS_THAN(SPI_PER_TRANS_NO_POLLING_NO_DMA, "%d us", (int)GET_US_BY_CCOUNT(t_flight_sorted[(TEST_TIMES+1)/2]));
//acquire the bus to send polling transactions faster
ret = spi_device_acquire_bus(spi, portMAX_DELAY);

4
components/esp32/test/test_fp.c
View file

@ -232,7 +232,7 @@ TEST_CASE("floating point division performance", "[fp]")
printf("%d divisions from %f = %f\n", COUNTS, MAXFLOAT, f);
printf("Per division = %d cycles\n", cycles);
TEST_PERFORMANCE_LESS_THAN(ESP32_CYCLES_PER_DIV, "%d cycles", cycles);
TEST_PERFORMANCE_LESS_THAN(CYCLES_PER_DIV, "%d cycles", cycles);
}
/* Note: not static, to avoid optimisation of const result */
@ -265,6 +265,6 @@ TEST_CASE("floating point square root performance", "[fp]")
printf("%d square roots from %f = %f\n", COUNTS, MAXFLOAT, f);
printf("Per sqrt = %d cycles\n", cycles);
TEST_PERFORMANCE_LESS_THAN(ESP32_CYCLES_PER_SQRT, "%d cycles", cycles);
TEST_PERFORMANCE_LESS_THAN(CYCLES_PER_SQRT, "%d cycles", cycles);
}

4
components/idf_test/CMakeLists.txt
View file

@ -1 +1,3 @@
idf_component_register(INCLUDE_DIRS include)
idf_build_get_property(target IDF_TARGET)
idf_component_register(INCLUDE_DIRS "include" "include/${target}")

2
components/idf_test/component.mk
View file

@ -3,3 +3,5 @@
#
# (Uses default behaviour of compiling all source files in directory, adding 'include' to include path.)
# make supports esp32 only
COMPONENT_ADD_INCLUDEDIRS := include include/esp32

22
components/idf_test/include/esp32/idf_performance_target.h Normal file
View file

@ -0,0 +1,22 @@
#pragma once
// AES-CBC hardware throughput (accounts for worst-case performance with PSRAM workaround)
#define IDF_PERFORMANCE_MIN_AES_CBC_THROUGHPUT_MBSEC 8.2
// SHA256 hardware throughput at 240MHz, threshold set lower than worst case
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 9.0
// esp_sha() time to process 32KB of input data from RAM
#define IDF_PERFORMANCE_MAX_TIME_SHA1_32KB 5000
#define IDF_PERFORMANCE_MAX_TIME_SHA512_32KB 4500
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PUBLIC_OP 19000
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PRIVATE_OP 180000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PUBLIC_OP 65000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PRIVATE_OP 850000
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING 30
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA 27
// floating point instructions per divide and per sqrt (configured for worst-case with PSRAM workaround)
#define IDF_PERFORMANCE_MAX_CYCLES_PER_DIV 70
#define IDF_PERFORMANCE_MAX_CYCLES_PER_SQRT 140

17
components/idf_test/include/esp32s2/idf_performance_target.h Normal file
View file

@ -0,0 +1,17 @@
#pragma once
#define IDF_PERFORMANCE_MIN_AES_CBC_THROUGHPUT_MBSEC 14.4
// SHA256 hardware throughput at 240MHz, threshold set lower than worst case
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 19.8
// esp_sha() time to process 32KB of input data from RAM
#define IDF_PERFORMANCE_MAX_TIME_SHA1_32KB 1000
#define IDF_PERFORMANCE_MAX_TIME_SHA512_32KB 900
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PUBLIC_OP 14000
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PRIVATE_OP 100000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PUBLIC_OP 60000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PRIVATE_OP 600000
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING 32
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA 30

102
components/idf_test/include/idf_performance.h
View file

@ -1,80 +1,96 @@
#pragma once
/* declare the performance here */
/* put target-specific macros into include/target/idf_performance_target.h */
#include "idf_performance_target.h"
/* Define default values in this file with #ifndef if the value could been overwritten in the target-specific headers
* above. Forgetting this will produce compile-time warnings.
*/
#ifndef IDF_PERFORMANCE_MAX_HTTPS_REQUEST_BIN_SIZE
#define IDF_PERFORMANCE_MAX_HTTPS_REQUEST_BIN_SIZE 900
#endif
#ifndef IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP
#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP 200
#endif
#ifndef IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_PSRAM
#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_PSRAM 300
#endif
#ifndef IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_UNICORE
#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_UNICORE 130
#endif
#ifndef IDF_PERFORMANCE_MAX_ESP_TIMER_GET_TIME_PER_CALL
#define IDF_PERFORMANCE_MAX_ESP_TIMER_GET_TIME_PER_CALL 1000
#endif
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_ESP32 30
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA_ESP32 27
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_ESP32S2 32
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA_ESP32S2 30
#ifndef IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING 15
#endif
#ifndef IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING_NO_DMA
#define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING_NO_DMA 15
#endif
/* Due to code size & linker layout differences interacting with cache, VFS
microbenchmark currently runs slower with PSRAM enabled. */
#ifndef IDF_PERFORMANCE_MAX_VFS_OPEN_WRITE_CLOSE_TIME
#define IDF_PERFORMANCE_MAX_VFS_OPEN_WRITE_CLOSE_TIME 20000
#endif
#ifndef IDF_PERFORMANCE_MAX_VFS_OPEN_WRITE_CLOSE_TIME_PSRAM
#define IDF_PERFORMANCE_MAX_VFS_OPEN_WRITE_CLOSE_TIME_PSRAM 25000
#endif
// throughput performance by iperf
#ifndef IDF_PERFORMANCE_MIN_TCP_RX_THROUGHPUT
#define IDF_PERFORMANCE_MIN_TCP_RX_THROUGHPUT 45
#endif
#ifndef IDF_PERFORMANCE_MIN_TCP_TX_THROUGHPUT
#define IDF_PERFORMANCE_MIN_TCP_TX_THROUGHPUT 40
#endif
#ifndef IDF_PERFORMANCE_MIN_UDP_RX_THROUGHPUT
#define IDF_PERFORMANCE_MIN_UDP_RX_THROUGHPUT 64
#endif
#ifndef IDF_PERFORMANCE_MIN_UDP_TX_THROUGHPUT
#define IDF_PERFORMANCE_MIN_UDP_TX_THROUGHPUT 50
#endif
// events dispatched per second by event loop library
#ifndef IDF_PERFORMANCE_MIN_EVENT_DISPATCH
#define IDF_PERFORMANCE_MIN_EVENT_DISPATCH 25000
#endif
#ifndef IDF_PERFORMANCE_MIN_EVENT_DISPATCH_PSRAM
#define IDF_PERFORMANCE_MIN_EVENT_DISPATCH_PSRAM 21000
#endif
// floating point instructions per divide and per sqrt (configured for worst-case with PSRAM workaround)
#define IDF_PERFORMANCE_MAX_ESP32_CYCLES_PER_DIV 70
#define IDF_PERFORMANCE_MAX_ESP32_CYCLES_PER_SQRT 140
#ifndef IDF_PERFORMANCE_MAX_SPILL_REG_CYCLES
#define IDF_PERFORMANCE_MAX_SPILL_REG_CYCLES 150
#endif
#ifndef IDF_PERFORMANCE_MAX_ISR_ENTER_CYCLES
#define IDF_PERFORMANCE_MAX_ISR_ENTER_CYCLES 290
#endif
#ifndef IDF_PERFORMANCE_MAX_ISR_EXIT_CYCLES
#define IDF_PERFORMANCE_MAX_ISR_EXIT_CYCLES 565
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_4BIT
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_4BIT 12500
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_4BIT
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_4BIT 12500
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_1BIT
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_1BIT 4000
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_1BIT
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_1BIT 4000
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_SPI
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_TOHOST_SPI 1000
#endif
#ifndef IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_SPI
#define IDF_PERFORMANCE_MIN_SDIO_THROUGHPUT_MBSEC_FRHOST_SPI 1000
#ifdef CONFIG_IDF_TARGET_ESP32
// AES-CBC hardware throughput (accounts for worst-case performance with PSRAM workaround)
#define IDF_PERFORMANCE_MIN_AES_CBC_THROUGHPUT_MBSEC 8.2
// SHA256 hardware throughput at 240MHz, threshold set lower than worst case
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 9.0
// esp_sha() time to process 32KB of input data from RAM
#define IDF_PERFORMANCE_MAX_TIME_SHA1_32KB 5000
#define IDF_PERFORMANCE_MAX_TIME_SHA512_32KB 4500
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PUBLIC_OP 19000
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PRIVATE_OP 180000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PUBLIC_OP 65000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PRIVATE_OP 850000
#elif defined CONFIG_IDF_TARGET_ESP32S2
#define IDF_PERFORMANCE_MIN_AES_CBC_THROUGHPUT_MBSEC 14.4
// SHA256 hardware throughput at 240MHz, threshold set lower than worst case
#define IDF_PERFORMANCE_MIN_SHA256_THROUGHPUT_MBSEC 19.8
// esp_sha() time to process 32KB of input data from RAM
#define IDF_PERFORMANCE_MAX_TIME_SHA1_32KB 1000
#define IDF_PERFORMANCE_MAX_TIME_SHA512_32KB 900
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PUBLIC_OP 14000
#define IDF_PERFORMANCE_MAX_RSA_2048KEY_PRIVATE_OP 100000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PUBLIC_OP 60000
#define IDF_PERFORMANCE_MAX_RSA_4096KEY_PRIVATE_OP 600000
#endif //CONFIG_IDF_TARGET_ESP32S2
#endif
//time to perform the task selection plus context switch (from task)
#ifndef IDF_PERFORMANCE_MAX_SCHEDULING_TIME
#define IDF_PERFORMANCE_MAX_SCHEDULING_TIME 1500
#endif

2
components/lwip/weekend_test/net_suite_test.py
View file

@ -75,7 +75,7 @@ def lwip_test_suite(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "net_suite.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("net_suite", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("net_suite", bin_size // 1024)
ttfw_idf.check_performance("net_suite", bin_size // 1024, dut1.TARGET)
dut1.start_app()
thread1 = Thread(target=sock_listener, args=(dut1, ))
thread2 = Thread(target=io_listener, args=(dut1, ))

2
examples/bluetooth/nimble/blehr/blehr_test.py
View file

@ -112,7 +112,7 @@ def test_example_app_ble_hr(env, extra_data):
binary_file = os.path.join(dut.app.binary_path, "blehr.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("blehr_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("blehr_bin_size", bin_size // 1024)
ttfw_idf.check_performance("blehr_bin_size", bin_size // 1024, dut.TARGET)
# Upload binary and start testing
Utility.console_log("Starting blehr simple example test app")

2
examples/bluetooth/nimble/bleprph/bleprph_test.py
View file

@ -135,7 +135,7 @@ def test_example_app_ble_peripheral(env, extra_data):
binary_file = os.path.join(dut.app.binary_path, "bleprph.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("bleprph_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("bleprph_bin_size", bin_size // 1024)
ttfw_idf.check_performance("bleprph_bin_size", bin_size // 1024, dut.TARGET)
# Upload binary and start testing
Utility.console_log("Starting bleprph simple example test app")

2
examples/get-started/blink/example_test.py
View file

@ -34,7 +34,7 @@ def test_examples_blink(env, extra_data):
binary_file = os.path.join(dut.app.binary_path, "blink.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("blink_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("blink_bin_size", bin_size // 1024)
ttfw_idf.check_performance("blink_bin_size", bin_size // 1024, dut.TARGET)
dut.start_app()

2
examples/protocols/asio/chat_client/asio_chat_client_test.py
View file

@ -63,7 +63,7 @@ def test_examples_protocol_asio_chat_client(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "asio_chat_client.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("asio_chat_client_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("asio_chat_client_size", bin_size // 1024)
ttfw_idf.check_performance("asio_chat_client_size", bin_size // 1024, dut1.TARGET)
# 1. start a tcp server on the host
host_ip = get_my_ip()
thread1 = Thread(target=chat_server_sketch, args=(host_ip,))

2
examples/protocols/asio/chat_server/asio_chat_server_test.py
View file

@ -20,7 +20,7 @@ def test_examples_protocol_asio_chat_server(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "asio_chat_server.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("asio_chat_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("asio_chat_server_size", bin_size // 1024)
ttfw_idf.check_performance("asio_chat_server_size", bin_size // 1024, dut1.TARGET)
# 1. start test
dut1.start_app()
# 2. get the server IP address

2
examples/protocols/asio/tcp_echo_server/asio_tcp_server_test.py
View file

@ -21,7 +21,7 @@ def test_examples_protocol_asio_tcp_server(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "asio_tcp_echo_server.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("asio_tcp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("asio_tcp_echo_server_size", bin_size // 1024)
ttfw_idf.check_performance("asio_tcp_echo_server_size", bin_size // 1024, dut1.TARGET)
# 1. start test
dut1.start_app()
# 2. get the server IP address

2
examples/protocols/asio/udp_echo_server/asio_udp_server_test.py
View file

@ -21,7 +21,7 @@ def test_examples_protocol_asio_udp_server(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "asio_udp_echo_server.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("asio_udp_echo_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("asio_udp_echo_server_size", bin_size // 1024)
ttfw_idf.check_performance("asio_udp_echo_server_size", bin_size // 1024, dut1.TARGET)
# 1. start test
dut1.start_app()
# 2. get the server IP address

2
examples/protocols/esp_http_client/esp_http_client_test.py
View file

@ -16,7 +16,7 @@ def test_examples_protocol_esp_http_client(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "esp-http-client-example.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("esp_http_client_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("esp_http_client_bin_size", bin_size // 1024)
ttfw_idf.check_performance("esp_http_client_bin_size", bin_size // 1024, dut1.TARGET)
# start test
dut1.start_app()
dut1.expect("Connected to AP, begin http example", timeout=30)

2
examples/protocols/http_server/advanced_tests/http_server_advanced_test.py
View file

@ -43,7 +43,7 @@ def test_examples_protocol_http_server_advanced(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "tests.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
Utility.console_log("Starting http_server advanced test app")

2
examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py
View file

@ -42,7 +42,7 @@ def test_examples_protocol_http_server_persistence(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "persistent_sockets.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
Utility.console_log("Starting http_server persistance test app")

2
examples/protocols/http_server/simple/http_server_simple_test.py
View file

@ -42,7 +42,7 @@ def test_examples_protocol_http_server_simple(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "simple.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("http_server_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024)
ttfw_idf.check_performance("http_server_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
Utility.console_log("Starting http_server simple test app")

2
examples/protocols/https_request/example_test.py
View file

@ -17,7 +17,7 @@ def test_examples_protocol_https_request(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "https_request.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("https_request_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("https_request_bin_size", bin_size // 1024)
ttfw_idf.check_performance("https_request_bin_size", bin_size // 1024, dut1.TARGET)
# start test
dut1.start_app()
dut1.expect("Connection established...", timeout=30)

2
examples/protocols/mdns/mdns_example_test.py
View file

@ -103,7 +103,7 @@ def test_examples_protocol_mdns(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "mdns-test.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mdns-test_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("mdns-test_bin_size", bin_size // 1024)
ttfw_idf.check_performance("mdns-test_bin_size", bin_size // 1024, dut1.TARGET)
# 1. start mdns application
dut1.start_app()
# 2. get the dut host name (and IP address)

2
examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py
View file

@ -78,7 +78,7 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mqtt_ssl_bin_size", "{}KB"
.format(bin_size // 1024))
ttfw_idf.check_performance("mqtt_ssl_size", bin_size // 1024)
ttfw_idf.check_performance("mqtt_ssl_size", bin_size // 1024, dut1.TARGET)
# Look for host:port in sdkconfig
try:
value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

2
examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py
View file

@ -68,7 +68,7 @@ def test_examples_protocol_mqtt_qos1(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "mqtt_tcp.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mqtt_tcp_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("mqtt_tcp_size", bin_size // 1024)
ttfw_idf.check_performance("mqtt_tcp_size", bin_size // 1024, dut1.TARGET)
# 1. start mqtt broker sketch
host_ip = get_my_ip()
thread1 = Thread(target=mqqt_server_sketch, args=(host_ip,1883))

2
examples/protocols/mqtt/ws/mqtt_ws_example_test.py
View file

@ -55,7 +55,7 @@ def test_examples_protocol_mqtt_ws(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("mqtt_websocket_size", bin_size // 1024)
ttfw_idf.check_performance("mqtt_websocket_size", bin_size // 1024, dut1.TARGET)
# Look for host:port in sdkconfig
try:
value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

2
examples/protocols/mqtt/wss/mqtt_wss_example_test.py
View file

@ -57,7 +57,7 @@ def test_examples_protocol_mqtt_wss(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "mqtt_websocket_secure.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("mqtt_websocket_secure_size", bin_size // 1024)
ttfw_idf.check_performance("mqtt_websocket_secure_size", bin_size // 1024, dut1.TARGET)
# Look for host:port in sdkconfig
try:
value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()["CONFIG_BROKER_URI"])

2
examples/protocols/websocket/example_test.py
View file

@ -220,7 +220,7 @@ def test_examples_protocol_websocket(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "websocket-example.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("websocket_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("websocket_bin_size", bin_size // 1024)
ttfw_idf.check_performance("websocket_bin_size", bin_size // 1024, dut1.TARGET)
try:
if "CONFIG_WEBSOCKET_URI_FROM_STDIN" in dut1.app.get_sdkconfig():

2
examples/provisioning/ble_prov/ble_prov_test.py
View file

@ -35,7 +35,7 @@ def test_examples_provisioning_ble(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "ble_prov.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("ble_prov_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("ble_prov_bin_size", bin_size // 1024)
ttfw_idf.check_performance("ble_prov_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
dut1.start_app()

2
examples/provisioning/manager/wifi_prov_mgr_test.py
View file

@ -35,7 +35,7 @@ def test_examples_wifi_prov_mgr(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "wifi_prov_mgr.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("wifi_prov_mgr_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024)
ttfw_idf.check_performance("wifi_prov_mgr_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
dut1.start_app()

2
examples/provisioning/softap_prov/softap_prov_test.py
View file

@ -36,7 +36,7 @@ def test_examples_provisioning_softap(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "softap_prov.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("softap_prov_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("softap_prov_bin_size", bin_size // 1024)
ttfw_idf.check_performance("softap_prov_bin_size", bin_size // 1024, dut1.TARGET)
# Upload binary and start testing
dut1.start_app()

12
examples/system/ota/advanced_https_ota/example_test.py
View file

@ -160,7 +160,7 @@ def test_examples_protocol_advanced_https_ota_example(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -215,7 +215,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_bin(env, extra_d
binary_file = os.path.join(dut1.app.binary_path, truncated_bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -266,7 +266,7 @@ def test_examples_protocol_advanced_https_ota_example_truncated_header(env, extr
binary_file = os.path.join(dut1.app.binary_path, truncated_bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -316,7 +316,7 @@ def test_examples_protocol_advanced_https_ota_example_random(env, extra_data):
fo.close()
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -355,7 +355,7 @@ def test_examples_protocol_advanced_https_ota_example_chunked(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
chunked_server = start_chunked_server(dut1.app.binary_path, 8070)
@ -398,7 +398,7 @@ def test_examples_protocol_advanced_https_ota_example_redirect_url(env, extra_da
binary_file = os.path.join(dut1.app.binary_path, bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("advanced_https_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("advanced_https_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):

10
examples/system/ota/native_ota_example/example_test.py
View file

@ -132,7 +132,7 @@ def test_examples_protocol_native_ota_example(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -187,7 +187,7 @@ def test_examples_protocol_native_ota_example_truncated_bin(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, truncated_bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -238,7 +238,7 @@ def test_examples_protocol_native_ota_example_truncated_header(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, truncated_bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -288,7 +288,7 @@ def test_examples_protocol_native_ota_example_random(env, extra_data):
fo.close()
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
if (get_server_status(host_ip, server_port) is False):
@ -327,7 +327,7 @@ def test_examples_protocol_native_ota_example_chunked(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, bin_name)
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("native_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("native_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
chunked_server = start_chunked_server(dut1.app.binary_path, 8070)

2
examples/system/ota/simple_ota_example/example_test.py
View file

@ -108,7 +108,7 @@ def test_examples_protocol_simple_ota_example(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "simple_ota.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("simple_ota_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("simple_ota_bin_size", bin_size // 1024)
ttfw_idf.check_performance("simple_ota_bin_size", bin_size // 1024, dut1.TARGET)
# start test
host_ip = get_my_ip()
thread1 = Thread(target=start_https_server, args=(dut1.app.binary_path, host_ip, 8000))

2
examples/wifi/iperf/iperf_test.py
View file

@ -610,7 +610,7 @@ def test_wifi_throughput_basic(env, extra_data):
# do check after logging, otherwise test will exit immediately if check fail, some performance can't be logged.
for throughput_type in test_result:
ttfw_idf.check_performance("{}_throughput".format(throughput_type),
test_result[throughput_type].get_best_throughput())
test_result[throughput_type].get_best_throughput(), dut.TARGET)
env.close_dut("iperf")

49
tools/ci/python_packages/ttfw_idf/__init__.py
View file

@ -137,31 +137,42 @@ def log_performance(item, value):
current_junit_case.stdout += performance_msg + "\r\n"
def check_performance(item, value):
def check_performance(item, value, target):
"""
check if idf performance meet pass standard
:param item: performance item name
:param value: performance item value
:param target: target chip
:raise: AssertionError: if check fails
"""
ret = True
standard_value = 0
idf_path = IDFApp.get_sdk_path()
performance_file = os.path.join(idf_path, "components", "idf_test", "include", "idf_performance.h")
if os.path.exists(performance_file):
with open(performance_file, "r") as f:
def _find_perf_item(path):
with open(path, 'r') as f:
data = f.read()
match = re.search(r"#define\s+IDF_PERFORMANCE_(MIN|MAX)_{}\s+([\d.]+)".format(item.upper()), data)
if match:
op = match.group(1)
standard_value = float(match.group(2))
if op == "MAX":
ret = value <= standard_value
else:
ret = value >= standard_value
if not ret:
raise AssertionError("[Performance] {} value is {}, doesn't meet pass standard {}"
.format(item, value, standard_value))
match = re.search(r'#define\s+IDF_PERFORMANCE_(MIN|MAX)_{}\s+([\d.]+)'.format(item.upper()), data)
return match.group(1), float(match.group(2))
def _check_perf(op, standard_value):
if op == 'MAX':
ret = value <= standard_value
else:
ret = value >= standard_value
if not ret:
raise AssertionError("[Performance] {} value is {}, doesn't meet pass standard {}"
.format(item, value, standard_value))
path_prefix = os.path.join(IDFApp.get_sdk_path(), 'components', 'idf_test', 'include')
performance_files = (os.path.join(path_prefix, target, 'idf_performance_target.h'),
os.path.join(path_prefix, 'idf_performance.h'))
for performance_file in performance_files:
try:
op, value = _find_perf_item(performance_file)
except (IOError, AttributeError):
# performance file doesn't exist or match is not found in it
continue
_check_perf(op, value)
# if no exception was thrown then the performance is met and no need to continue
break

2
tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py
View file

@ -143,7 +143,7 @@ def test_examples_protocol_mqtt_publish_connect(env, extra_data):
binary_file = os.path.join(dut1.app.binary_path, "mqtt_publish_connect_test.bin")
bin_size = os.path.getsize(binary_file)
ttfw_idf.log_performance("mqtt_publish_connect_test_bin_size", "{}KB".format(bin_size // 1024))
ttfw_idf.check_performance("mqtt_publish_connect_test_bin_size_vin_size", bin_size // 1024)
ttfw_idf.check_performance("mqtt_publish_connect_test_bin_size_vin_size", bin_size // 1024, dut1.TARGET)
# Look for test case symbolic names
cases = {}
try:

15
tools/unit-test-app/components/test_utils/include/test_utils.h
View file

@ -45,21 +45,6 @@
TEST_ASSERT(value > PERFORMANCE_CON(IDF_PERFORMANCE_MIN_, name)); \
} while(0)
//Add more targets here, and corresponding performance requirements for that target in idf_performance.h
#ifdef CONFIG_IDF_TARGET_ESP32
#define PERFORMANCE_TARGET_SUFFIX _ESP32
#elif CONFIG_IDF_TARGET_ESP32S2
#define PERFORMANCE_TARGET_SUFFIX _ESP32S2
#else
#error target surfix not defined!
#endif
#define TEST_TARGET_PERFORMANCE_LESS_THAN(name, value_fmt, value) TEST_PERFORMANCE_LESS_THAN(PERFORMANCE_CON(name, PERFORMANCE_TARGET_SUFFIX), value_fmt, value)
#define TEST_TARGET_PERFORMANCE_GREATER_THAN(name, value_fmt, value) TEST_PERFORMANCE_GREATER_THAN(PERFORMANCE_CON(name, PERFORMANCE_TARGET_SUFFIX), value_fmt, value)
/* @brief macro to print IDF performance
* @param mode : performance item name. a string pointer.
* @param value_fmt: print format and unit of the value, for example: "%02fms", "%dKB"