Crash_Override/OVMS3-idf
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support a [leaks] tag for UT

Browse source
This commit is contained in:
Konstantin Kondrashov 2019-04-12 09:31:23 +08:00 committed by Angus Gratton
parent 3ec7bec6a7
commit 5021129e71
15 changed files with 255 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.gitlab-ci.yml
View file

@ -1528,6 +1528,20 @@ UT_004_17:
- UT_T1_1
- psram
UT_004_18:
<<: *unit_test_template
tags:
- ESP32_IDF
- UT_T1_1
- psram
UT_004_19:
<<: *unit_test_template
tags:
- ESP32_IDF
- UT_T1_1
- psram
UT_005_01:
<<: *unit_test_template
tags:

2
components/driver/test/test_uart.c
View file

@ -136,7 +136,7 @@ TEST_CASE("test uart get baud-rate","[uart]")
ESP_LOGI(UART_TAG, "get baud-rate test passed ....\n");
}
TEST_CASE("test uart tx data with break","[uart]")
TEST_CASE("test uart tx data with break","[uart][leaks=2192]")
{
const int buf_len = 200;
const int send_len = 128;

2
components/esp_event/test/test_event.c
View file

@ -868,7 +868,7 @@ TEST_CASE("performance test - dedicated task", "[event]")
performance_test(true);
}
TEST_CASE("performance test - no dedicated task", "[event]")
TEST_CASE("performance test - no dedicated task", "[event][leaks=2736]")
{
performance_test(false);
}

2
components/esp_wifi/test/test_phy_rtc.c
View file

@ -77,7 +77,7 @@ static IRAM_ATTR void test_phy_rtc_cache_task(void *arg)
vTaskDelete(NULL);
}
TEST_CASE("Test PHY/RTC functions called when cache is disabled", "[phy_rtc][cache_disabled]")
TEST_CASE("Test PHY/RTC functions called when cache is disabled", "[phy_rtc][cache_disabled][leaks=1216]")
{
semphr_done = xSemaphoreCreateCounting(1, 0);

2
components/fatfs/test/test_fatfs_rawflash.c
View file

@ -69,7 +69,7 @@ TEST_CASE("(raw) can read file", "[fatfs]")
test_teardown();
}
TEST_CASE("(raw) can open maximum number of files", "[fatfs]")
TEST_CASE("(raw) can open maximum number of files", "[fatfs][leaks=2028]")
{
size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
test_setup(max_files);

4
components/fatfs/test/test_fatfs_sdmmc.c
View file

@ -158,7 +158,7 @@ TEST_CASE("(SD) opendir, readdir, rewinddir, seekdir work as expected", "[fatfs]
test_teardown();
}
TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDMODE]")
TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDMODE][leaks=1264]")
{
test_setup();
test_fatfs_concurrent("/sdcard/f");
@ -167,7 +167,7 @@ TEST_CASE("(SD) multiple tasks can use same volume", "[fatfs][test_env=UT_T1_SDM
static void speed_test(void* buf, size_t buf_size, size_t file_size, bool write);
TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60]")
TEST_CASE("(SD) write/read speed test", "[fatfs][sd][test_env=UT_T1_SDMODE][timeout=60][leaks=1080]")
{
size_t heap_size;
HEAP_SIZE_CAPTURE(heap_size);

6
components/fatfs/test/test_fatfs_spiflash.c
View file

@ -70,7 +70,7 @@ TEST_CASE("(WL) can read file", "[fatfs][wear_levelling]")
test_teardown();
}
TEST_CASE("(WL) can open maximum number of files", "[fatfs][wear_levelling]")
TEST_CASE("(WL) can open maximum number of files", "[fatfs][wear_levelling][leaks=2460]")
{
size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
esp_vfs_fat_sdmmc_mount_config_t mount_config = {
@ -152,14 +152,14 @@ TEST_CASE("(WL) opendir, readdir, rewinddir, seekdir work as expected", "[fatfs]
test_teardown();
}
TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling]")
TEST_CASE("(WL) multiple tasks can use same volume", "[fatfs][wear_levelling][leaks=1340]")
{
test_setup();
test_fatfs_concurrent("/spiflash/f");
test_teardown();
}
TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60]")
TEST_CASE("(WL) write/read speed test", "[fatfs][wear_levelling][timeout=60][leaks=1156]")
{
/* Erase partition before running the test to get consistent results */
const esp_partition_t* part = get_test_data_partition();

60
components/heap/test/test_leak.c Normal file
View file

@ -0,0 +1,60 @@
/*
Tests for a leak tag
*/
#include <stdio.h>
#include "unity.h"
#include "esp_heap_caps_init.h"
#include "esp_system.h"
#include <stdlib.h>
static char* check_calloc(int size)
{
char *arr = calloc(size, sizeof(char));
TEST_ASSERT_NOT_NULL(arr);
return arr;
}
TEST_CASE("Check for leaks (no leak)", "[heap]")
{
char *arr = check_calloc(7000);
free(arr);
}
TEST_CASE("Check for leaks (leak)", "[heap][ignore]")
{
check_calloc(7000);
}
TEST_CASE("Not check for leaks", "[heap][leaks]")
{
check_calloc(7000);
}
TEST_CASE("Set a leak level = 7016", "[heap][leaks=7016]")
{
check_calloc(7000);
}
static void test_fn(void)
{
check_calloc(7000);
}
TEST_CASE_MULTIPLE_STAGES("Not check for leaks in MULTIPLE_STAGES mode", "[heap][leaks]", test_fn, test_fn, test_fn);
TEST_CASE_MULTIPLE_STAGES("Check for leaks in MULTIPLE_STAGES mode (leak)", "[heap][ignore]", test_fn, test_fn, test_fn);
static void test_fn2(void)
{
check_calloc(7000);
esp_restart();
}
static void test_fn3(void)
{
check_calloc(7000);
}
TEST_CASE_MULTIPLE_STAGES("Check for leaks in MULTIPLE_STAGES mode (manual reset)", "[heap][leaks][reset=SW_CPU_RESET, SW_CPU_RESET]", test_fn2, test_fn2, test_fn3);

4
components/spiffs/test/test_spiffs.c
View file

@ -531,7 +531,7 @@ TEST_CASE("can read file", "[spiffs]")
test_teardown();
}
TEST_CASE("can open maximum number of files", "[spiffs]")
TEST_CASE("can open maximum number of files", "[spiffs][leaks=2244]")
{
size_t max_files = FOPEN_MAX - 3; /* account for stdin, stdout, stderr */
esp_vfs_spiffs_conf_t conf = {
@ -602,7 +602,7 @@ TEST_CASE("readdir with large number of files", "[spiffs][timeout=30]")
test_teardown();
}
TEST_CASE("multiple tasks can use same volume", "[spiffs]")
TEST_CASE("multiple tasks can use same volume", "[spiffs][leaks=1128]")
{
test_setup();
test_spiffs_concurrent("/spiffs/f");

49
components/unity/unity_runner.c
View file

@ -18,6 +18,7 @@
#include <ctype.h>
#include <stdio.h>
#include "unity.h"
#include "esp_system.h"
/* similar to UNITY_PRINT_EOL */
#define UNITY_PRINT_TAB() UNITY_OUTPUT_CHAR('\t')
@ -61,7 +62,29 @@ static void print_multiple_function_test_menu(const test_desc_t *test_ms)
}
}
static void multiple_function_option(const test_desc_t *test_ms)
/*
* This function looks like UnityDefaultTestRun function only without UNITY_CLR_DETAILS.
* void UnityDefaultTestRun(UnityTestFunction Func, const char* FuncName, const int FuncLineNum)
* was moved from `components/unity/unity/src/unity.c` to here.
*/
static void unity_default_test_run(UnityTestFunction Func, const char* FuncName, const int FuncLineNum)
{
Unity.CurrentTestName = FuncName;
Unity.CurrentTestLineNumber = (UNITY_LINE_TYPE)FuncLineNum;
Unity.NumberOfTests++;
if (TEST_PROTECT())
{
setUp();
Func();
}
if (TEST_PROTECT())
{
tearDown();
}
UnityConcludeTest();
}
static int multiple_function_option(const test_desc_t *test_ms)
{
int selection;
char cmdline[256] = {0};
@ -76,12 +99,13 @@ static void multiple_function_option(const test_desc_t *test_ms)
}
selection = atoi((const char *) cmdline) - 1;
if (selection >= 0 && selection < test_ms->test_fn_count) {
UnityDefaultTestRun(test_ms->fn[selection], test_ms->name, test_ms->line);
unity_default_test_run(test_ms->fn[selection], test_ms->name, test_ms->line);
} else {
UnityPrint("Invalid selection, your should input number 1-");
UnityPrintNumber(test_ms->test_fn_count);
UNITY_PRINT_EOL();
}
return selection;
}
static void unity_run_single_test(const test_desc_t *test)
@ -95,10 +119,27 @@ static void unity_run_single_test(const test_desc_t *test)
Unity.TestFile = test->file;
Unity.CurrentDetail1 = test->desc;
bool reset_after_test = strstr(Unity.CurrentDetail1, "[leaks") != NULL;
bool multi_device = strstr(Unity.CurrentDetail1, "[multi_device]") != NULL;
if (test->test_fn_count == 1) {
UnityDefaultTestRun(test->fn[0], test->name, test->line);
unity_default_test_run(test->fn[0], test->name, test->line);
} else {
multiple_function_option(test);
int selection = multiple_function_option(test);
if (reset_after_test && multi_device == false) {
if (selection != (test->test_fn_count - 1)) {
// to do a reset for all stages except the last stage.
esp_restart();
}
}
}
if (reset_after_test) {
// print a result of test before to do reset for the last stage.
UNITY_END();
UnityPrint("Enter next test, or 'enter' to see menu");
UNITY_PRINT_EOL();
UNITY_OUTPUT_FLUSH();
esp_restart();
}
}

9
tools/unit-test-app/README.md
View file

@ -56,6 +56,15 @@ When we add new test case, it will construct a structure to save case data durin
2. the rest tags should be [type=value]. Tags could have default value and omitted value. For example, reset tag default value is "POWERON_RESET", omitted value is "" (do not reset) :
* "[reset]" equal to [reset=POWERON_RESET]
* if reset tag doesn't exist, then it equals to [reset=""]
3. the `[leaks]` tag is used to disable the leak checking. A specific maximum memory leakage can be set as follows: `[leaks=500]`. This allows no more than 500 bytes of heap to be leaked. Also there is a special function to set the critical level of leakage not through a tag, just directly in the test code ``test_utils_set_critical_leak_level()``.
The priority of using leakage level is as follows:
1. Setting by tag `[leaks=500]`.
2. Setting by ``test_utils_set_critical_leak_level()`` function.
3. Setting by default leakage in Kconfig ``CONFIG_UNITY_CRITICAL_LEAK_LEVEL_GENERAL``.
Tests marked as `[leaks]` or `[leaks=xxx]` reset the device after completion (or after each stage in multistage tests).
`TagDefinition.yml` defines how we should parse the description. In `TagDefinition.yml`, we declare the tags we are interested in, their default value and omitted value. Parser will parse the properities of test cases according to this file, and add them as test case attributes.

12
tools/unit-test-app/components/test_utils/Kconfig
View file

@ -12,4 +12,16 @@ menu "IDF unit test"
int "Stack size of Unity test task, in bytes"
default 8192
config UNITY_WARN_LEAK_LEVEL_GENERAL
int "Leak warning level"
default 255
config UNITY_CRITICAL_LEAK_LEVEL_GENERAL
int "Critical leak"
default 1024
config UNITY_CRITICAL_LEAK_LEVEL_LWIP
int "Critical leak for UT which use LWIP component"
default 4095
endmenu

40
tools/unit-test-app/components/test_utils/include/test_utils.h
View file

@ -196,3 +196,43 @@ static inline void unity_send_signal(const char* signal_name)
* @param[out] mac_addr store converted MAC address
*/
bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr);
/**
* @brief Leak for components
*/
typedef enum {
COMP_LEAK_GENERAL = 0, /**< Leak by default */
COMP_LEAK_LWIP, /**< Leak for LWIP */
COMP_LEAK_NVS, /**< Leak for NVS */
COMP_LEAK_ALL, /**< Use for getting the summary leak level */
} esp_comp_leak_t;
/**
* @brief Type of leak
*/
typedef enum {
TYPE_LEAK_WARNING = 0, /**< Warning level of leak */
TYPE_LEAK_CRITICAL, /**< Critical level of leak */
TYPE_LEAK_MAX, /**< Max number of leak levels */
} esp_type_leak_t;
/**
* @brief Set a leak level for the required type and component.
*
* @param[in] leak_level Level of leak
* @param[in] type Type of leak
* @param[in] component Name of component
*
* return ESP_OK: Successful.
* ESP_ERR_INVALID_ARG: Invalid argument.
*/
esp_err_t test_utils_set_leak_level(size_t leak_level, esp_type_leak_t type, esp_comp_leak_t component);
/**
* @brief Get a leak level for the required type and component.
*
* @param[in] type Type of leak.
* @param[in] component Name of component. If COMP_LEAK_ALL, then the level will be summarized for all components.
* return Leak level
*/
size_t test_utils_get_leak_level(esp_type_leak_t type, esp_comp_leak_t component);

50
tools/unit-test-app/components/test_utils/test_runner.c
View file

@ -13,6 +13,7 @@
// limitations under the License.
#include <stdio.h>
#include "string.h"
#include "esp_heap_caps.h"
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
@ -26,14 +27,8 @@
static size_t before_free_8bit;
static size_t before_free_32bit;
/* Each unit test is allowed to "leak" this many bytes.
TODO: Make this value editable by the test.
Will always need to be some value here, as fragmentation can reduce free space even when no leak is occurring.
*/
const size_t WARN_LEAK_THRESHOLD = 256;
const size_t CRITICAL_LEAK_THRESHOLD = 4096;
static size_t warn_leak_threshold;
static size_t critical_leak_threshold;
static void unity_task(void *pvParameters)
{
@ -77,6 +72,8 @@ void setUp(void)
get_test_data_partition(); /* allocate persistent partition table structures */
unity_reset_leak_checks();
test_utils_set_leak_level(CONFIG_UNITY_CRITICAL_LEAK_LEVEL_GENERAL, TYPE_LEAK_CRITICAL, COMP_LEAK_GENERAL);
test_utils_set_leak_level(CONFIG_UNITY_WARN_LEAK_LEVEL_GENERAL, TYPE_LEAK_WARNING, COMP_LEAK_GENERAL);
}
static void check_leak(size_t before_free, size_t after_free, const char *type)
@ -85,16 +82,37 @@ static void check_leak(size_t before_free, size_t after_free, const char *type)
return;
}
size_t leaked = before_free - after_free;
if (leaked < WARN_LEAK_THRESHOLD) {
if (leaked <= warn_leak_threshold) {
return;
}
printf("MALLOC_CAP_%s %s leak: Before %u bytes free, After %u bytes free (delta %u)\n",
type,
leaked < CRITICAL_LEAK_THRESHOLD ? "potential" : "critical",
leaked <= critical_leak_threshold ? "potential" : "critical",
before_free, after_free, leaked);
fflush(stdout);
TEST_ASSERT_MESSAGE(leaked < CRITICAL_LEAK_THRESHOLD, "The test leaked too much memory");
TEST_ASSERT_MESSAGE(leaked <= critical_leak_threshold, "The test leaked too much memory");
}
static bool leak_check_required()
{
warn_leak_threshold = test_utils_get_leak_level(TYPE_LEAK_WARNING, COMP_LEAK_ALL);
critical_leak_threshold = test_utils_get_leak_level(TYPE_LEAK_CRITICAL, COMP_LEAK_ALL);
if (Unity.CurrentDetail1 != NULL) {
const char *leaks = "[leaks";
const int len_leaks = strlen(leaks);
const char *sub_leaks = strstr(Unity.CurrentDetail1, leaks);
if (sub_leaks != NULL) {
if (sub_leaks[len_leaks] == ']') {
return true;
} else if (sub_leaks[len_leaks] == '=') {
critical_leak_threshold = strtol(&sub_leaks[len_leaks + 1], NULL, 10);
warn_leak_threshold = critical_leak_threshold;
return false;
}
}
}
return false;
}
/* tearDown runs after every test */
@ -103,6 +121,8 @@ void tearDown(void)
/* some FreeRTOS stuff is cleaned up by idle task */
vTaskDelay(5);
size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
/* We want the teardown to have this file in the printout if TEST_ASSERT fails */
const char *real_testfile = Unity.TestFile;
Unity.TestFile = __FILE__;
@ -115,11 +135,11 @@ void tearDown(void)
heap_trace_stop();
heap_trace_dump();
#endif
size_t after_free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
size_t after_free_32bit = heap_caps_get_free_size(MALLOC_CAP_32BIT);
check_leak(before_free_8bit, after_free_8bit, "8BIT");
check_leak(before_free_32bit, after_free_32bit, "32BIT");
if (leak_check_required() == false) {
check_leak(before_free_8bit, after_free_8bit, "8BIT");
check_leak(before_free_32bit, after_free_32bit, "32BIT");
}
Unity.TestFile = real_testfile; // go back to the real filename
}

29
tools/unit-test-app/components/test_utils/test_utils.c
View file

@ -55,6 +55,8 @@ void test_case_uses_tcpip()
// Reset the leak checker as LWIP allocates a lot of memory on first run
unity_reset_leak_checks();
test_utils_set_leak_level(0, TYPE_LEAK_CRITICAL, COMP_LEAK_GENERAL);
test_utils_set_leak_level(CONFIG_UNITY_CRITICAL_LEAK_LEVEL_LWIP, TYPE_LEAK_CRITICAL, COMP_LEAK_LWIP);
}
// wait user to send "Enter" key or input parameter
@ -114,3 +116,30 @@ bool unity_util_convert_mac_from_string(const char* mac_str, uint8_t *mac_addr)
return true;
}
static size_t test_unity_leak_level[TYPE_LEAK_MAX][COMP_LEAK_ALL] = { 0 };
esp_err_t test_utils_set_leak_level(size_t leak_level, esp_type_leak_t type_of_leak, esp_comp_leak_t component)
{
if (type_of_leak >= TYPE_LEAK_MAX || component >= COMP_LEAK_ALL) {
return ESP_ERR_INVALID_ARG;
}
test_unity_leak_level[type_of_leak][component] = leak_level;
return ESP_OK;
}
size_t test_utils_get_leak_level(esp_type_leak_t type_of_leak, esp_comp_leak_t component)
{
size_t leak_level = 0;
if (type_of_leak >= TYPE_LEAK_MAX || component > COMP_LEAK_ALL) {
leak_level = 0;
} else {
if (component == COMP_LEAK_ALL) {
for (int comp = 0; comp < COMP_LEAK_ALL; ++comp) {
leak_level += test_unity_leak_level[type_of_leak][comp];
}
} else {
leak_level = test_unity_leak_level[type_of_leak][component];
}
}
return leak_level;
}