diff --git a/tools/test_apps/system/panic/CMakeLists.txt b/tools/test_apps/system/panic/CMakeLists.txt new file mode 100644 index 000000000..756602542 --- /dev/null +++ b/tools/test_apps/system/panic/CMakeLists.txt @@ -0,0 +1,6 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.5) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(test_panic) diff --git a/tools/test_apps/system/panic/app_test.py b/tools/test_apps/system/panic/app_test.py new file mode 100644 index 000000000..e4c82e80d --- /dev/null +++ b/tools/test_apps/system/panic/app_test.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +import re +from test_panic_util.test_panic_util import panic_test, get_dut, run_all + + +@panic_test() +def test_panic_task_wdt(env, extra_data): + with get_dut(env, "panic", "test_task_wdt", qemu_wdt_enable=True) as dut: + dut.expect("Task watchdog got triggered. The following tasks did not reset the watchdog in time:") + dut.expect("CPU 0: main") + dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0")) + dut.expect_none("register dump:") + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_int_wdt(env, extra_data): + with get_dut(env, "panic", "test_int_wdt", qemu_wdt_enable=True) as dut: + dut.expect_gme("Interrupt wdt timeout on CPU0") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect_reg_dump(1) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_cache_error(env, extra_data): + with get_dut(env, "panic", "test_cache_error") as dut: + dut.expect("Re-enable cpu cache.") + dut.expect_gme("Cache disabled but cached memory region accessed") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_int_wdt_cache_disabled(env, extra_data): + with get_dut(env, "panic", "test_int_wdt_cache_disabled", qemu_wdt_enable=True) as dut: + dut.expect("Re-enable cpu cache.") + dut.expect_gme("Interrupt wdt timeout on CPU0") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect_reg_dump(1) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_abort(env, extra_data): + with get_dut(env, "panic", "test_abort") as dut: + dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0")) + dut.expect_none("register dump:") + dut.expect("Backtrace:") + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_storeprohibited(env, extra_data): + with get_dut(env, "panic", "test_storeprohibited") as dut: + dut.expect_gme("StoreProhibited") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_stack_overflow(env, extra_data): + with get_dut(env, "panic", "test_stack_overflow") as dut: + dut.expect_gme("Unhandled debug exception") + dut.expect("Stack canary watchpoint triggered (main)") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_illegal_instruction(env, extra_data): + with get_dut(env, "panic", "test_illegal_instruction") as dut: + dut.expect_gme("IllegalInstruction") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_panic_instr_fetch_prohibited(env, extra_data): + with get_dut(env, "panic", "test_instr_fetch_prohibited") as dut: + dut.expect_gme("InstrFetchProhibited") + dut.expect_reg_dump(0) + dut.expect("Backtrace:") + # At the moment the backtrace is corrupted, need to jump over the first PC in case of InstrFetchProhibited. + # Fix this and change expect to expect_none. + dut.expect("CORRUPTED") + dut.expect_elf_sha256() + dut.expect_none("Guru Meditation") + dut.expect("Rebooting...") + + +@panic_test() +def test_coredump_uart_abort(env, extra_data): + with get_dut(env, "coredump_uart", "test_abort") as dut: + dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0")) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump") + dut.expect(dut.COREDUMP_UART_END) + dut.expect("Rebooting...") + dut.process_coredump_uart() + # TODO: check the contents of core dump output + + +@panic_test() +def test_coredump_flash_abort(env, extra_data): + with get_dut(env, "coredump_flash", "test_abort") as dut: + dut.expect(re.compile(r"abort\(\) was called at PC [0-9xa-f]+ on core 0")) + dut.expect("Backtrace:") + dut.expect_elf_sha256() + dut.expect_none("CORRUPTED", "Guru Meditation", "Re-entered core dump") + dut.expect("Rebooting...") + dut.process_coredump_flash() + # TODO: check the contents of core dump output + + +if __name__ == '__main__': + run_all(__file__) diff --git a/tools/test_apps/system/panic/main/CMakeLists.txt b/tools/test_apps/system/panic/main/CMakeLists.txt new file mode 100644 index 000000000..5b5faac69 --- /dev/null +++ b/tools/test_apps/system/panic/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRCS "test_panic_main.c" + INCLUDE_DIRS ".") \ No newline at end of file diff --git a/tools/test_apps/system/panic/main/test_panic_main.c b/tools/test_apps/system/panic/main/test_panic_main.c new file mode 100644 index 000000000..f72afaeba --- /dev/null +++ b/tools/test_apps/system/panic/main/test_panic_main.c @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "esp_partition.h" +#include "esp_flash.h" +#include "esp_system.h" + +/* utility functions */ +static void die(const char* msg) __attribute__ ((noreturn)); +static const char* get_test_name(); + +/* functions which cause an exception/panic in different ways */ +static void test_abort(); +static void test_int_wdt(); +static void test_task_wdt(); +static void test_storeprohibited(); +static void test_cache_error(); +static void test_int_wdt_cache_disabled(); +static void test_stack_overflow(); +static void test_illegal_instruction(); +static void test_instr_fetch_prohibited(); + + +void app_main(void) +{ + /* Needed to allow the tick hook to set correct INT WDT timeouts */ + vTaskDelay(2); + + /* Test script sends to command over UART. Read it and determine how to proceed. */ + const char* test_name = get_test_name(); + if (test_name == NULL) { + /* Nothing to do */ + return; + } + printf("Got test name: %s\n", test_name); + + #define HANDLE_TEST(name_) \ + if (strcmp(test_name, #name_) == 0) { \ + name_(); \ + die("Test function has returned"); \ + } + + HANDLE_TEST(test_abort); + HANDLE_TEST(test_int_wdt); + HANDLE_TEST(test_task_wdt); + HANDLE_TEST(test_storeprohibited); + HANDLE_TEST(test_cache_error); + HANDLE_TEST(test_int_wdt_cache_disabled); + HANDLE_TEST(test_stack_overflow); + HANDLE_TEST(test_illegal_instruction); + HANDLE_TEST(test_instr_fetch_prohibited); + + #undef HANDLE_TEST + + die("Unknown test name"); +} + +/* implementations of the test functions */ + +static void test_abort() +{ + abort(); +} + +static void test_int_wdt() +{ + portDISABLE_INTERRUPTS(); + while (true) { + ; + } +} + +static void test_task_wdt() +{ + while (true) { + ; + } +} + +static void test_storeprohibited() +{ + *(int*) 0x1 = 0; +} + +static IRAM_ATTR void test_cache_error() +{ + esp_flash_default_chip->os_func->start(esp_flash_default_chip->os_func_data); + die("this should not be printed"); +} + +static void IRAM_ATTR test_int_wdt_cache_disabled() +{ + esp_flash_default_chip->os_func->start(esp_flash_default_chip->os_func_data); + portDISABLE_INTERRUPTS(); + while (true) { + ; + } +} + +static void test_stack_overflow() +{ + volatile uint8_t stuff[CONFIG_ESP_MAIN_TASK_STACK_SIZE * 2]; + for (int i = 0; i < sizeof(stuff); ++i) { + stuff[i] = rand(); + } +} + +static void test_illegal_instruction() +{ + __asm__ __volatile__("ill"); +} + +static void test_instr_fetch_prohibited() +{ + typedef void (*fptr_t)(void); + volatile fptr_t fptr = (fptr_t) 0x4; + fptr(); +} + +/* implementations of the utility functions */ + +#define BOOT_CMD_MAX_LEN (128) + +static const char* get_test_name() +{ + static char test_name_str[BOOT_CMD_MAX_LEN] = {0}; + + printf("Enter test name: "); + fflush(stdout); + + /* Not using blocking fgets(stdin) here, as QEMU doesn't yet implement RX timeout interrupt, + * which is required for the UART driver and blocking stdio to work. + */ + int c = EOF; + char *p = test_name_str; + const char *end = test_name_str + sizeof(test_name_str) - 1; + while (p < end) { + c = getchar(); + if (c == EOF) { + vTaskDelay(pdMS_TO_TICKS(10)); + } else if (c == '\r') { + continue; + } else if (c == '\n') { + *p = '\0'; + break; + } else { + *p = c; + ++p; + } + } + + return test_name_str; +} + +extern void esp_restart_noos(void) __attribute__ ((noreturn)); + +static void die(const char* msg) +{ + printf("Test error: %s\n\n", msg); + fflush(stdout); + fsync(fileno(stdout)); + /* Don't use abort here as it would enter the panic handler */ + esp_restart_noos(); +} diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash new file mode 100644 index 000000000..3b19e0ab9 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash @@ -0,0 +1 @@ +CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=y diff --git a/tools/test_apps/system/panic/sdkconfig.ci.coredump_uart b/tools/test_apps/system/panic/sdkconfig.ci.coredump_uart new file mode 100644 index 000000000..ac111d161 --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.ci.coredump_uart @@ -0,0 +1 @@ +CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=y diff --git a/tools/test_apps/system/panic/sdkconfig.ci.panic b/tools/test_apps/system/panic/sdkconfig.ci.panic new file mode 100644 index 000000000..e69de29bb diff --git a/tools/test_apps/system/panic/sdkconfig.defaults b/tools/test_apps/system/panic/sdkconfig.defaults new file mode 100644 index 000000000..20d6175bb --- /dev/null +++ b/tools/test_apps/system/panic/sdkconfig.defaults @@ -0,0 +1,12 @@ +# Flash DOUT mode (QEMU limitation) +CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y + +# Less noisy output +CONFIG_BOOTLOADER_LOG_LEVEL_WARN=y +CONFIG_LOG_DEFAULT_LEVEL_WARN=y + +# To check for stack overflows +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y + +# To panic on task WDT +CONFIG_ESP_TASK_WDT_PANIC=y diff --git a/tools/test_apps/system/panic/test_panic_util/__init__.py b/tools/test_apps/system/panic/test_panic_util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/test_apps/system/panic/test_panic_util/test_panic_util.py b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py new file mode 100644 index 000000000..39f0cfda0 --- /dev/null +++ b/tools/test_apps/system/panic/test_panic_util/test_panic_util.py @@ -0,0 +1,176 @@ +import os +import sys +import re +import subprocess +import ttfw_idf +from tiny_test_fw import Utility, TinyFW, DUT +from tiny_test_fw.Utility import SearchCases, CaseConfig + + +# hard-coded to the path one level above - only intended to be used from the panic test app +TEST_PATH = os.path.relpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."), os.getenv("IDF_PATH")) +TEST_SUITE = "Panic" + + +def ok(data): + """ Helper function used with dut.expect_any """ + pass + + +def unexpected(data): + """ Helper function used with dut.expect_any """ + raise AssertionError("Unexpected: {}".format(data)) + + +class PanicTestApp(ttfw_idf.TestApp): + pass + + +class PanicTestMixin(object): + """ Provides custom functionality for the panic test DUT """ + BOOT_CMD_ADDR = 0x9000 + BOOT_CMD_SIZE = 0x1000 + DEFAULT_EXPECT_TIMEOUT = 10 + COREDUMP_UART_START = "================= CORE DUMP START =================" + COREDUMP_UART_END = "================= CORE DUMP END =================" + + def start_test(self, test_name): + """ Starts the app and sends it the test name """ + self.test_name = test_name + # Start the app and verify that it has started up correctly + self.start_capture_raw_data() + self.start_app() + self.expect("Enter test name: ") + Utility.console_log("Setting boot command: " + test_name) + self.write(test_name) + self.expect("Got test name: " + test_name) + + def expect_none(self, *patterns, **timeout_args): + """ like dut.expect_all, but with an inverse logic """ + found_data = [] + if "timeout" not in timeout_args: + timeout_args["timeout"] = 1 + + def found(data): + raise AssertionError("Unexpected: {}".format(data)) + found_data.append(data) + try: + expect_items = [(pattern, found) for pattern in patterns] + self.expect_any(*expect_items, **timeout_args) + raise AssertionError("Unexpected: {}".format(found_data)) + except DUT.ExpectTimeout: + return True + + def expect_gme(self, reason): + """ Expect method for Guru Meditation Errors """ + self.expect(r"Guru Meditation Error: Core 0 panic'ed (%s)" % reason) + + def expect_reg_dump(self, core=0): + """ Expect method for the register dump """ + self.expect(re.compile(r"Core\s+%d register dump:" % core)) + + def expect_elf_sha256(self): + """ Expect method for ELF SHA256 line """ + elf_sha256 = self.app.get_elf_sha256() + sdkconfig = self.app.get_sdkconfig() + elf_sha256_len = int(sdkconfig.get("CONFIG_APP_RETRIEVE_LEN_ELF_SHA", "16")) + self.expect("ELF file SHA256: " + elf_sha256[0:elf_sha256_len]) + + def __enter__(self): + self._raw_data = None + return self + + def __exit__(self, type, value, traceback): + log_folder = self.app.get_log_folder(TEST_SUITE) + with open(os.path.join(log_folder, "log_" + self.test_name + ".txt"), "w") as log_file: + Utility.console_log("Writing output of {} to {}".format(self.test_name, log_file.name)) + log_file.write(self.get_raw_data()) + + self.close() + + def get_raw_data(self): + if not self._raw_data: + self._raw_data = self.stop_capture_raw_data() + return self._raw_data + + def _call_espcoredump(self, extra_args, coredump_file_name, output_file_name): + # no "with" here, since we need the file to be open for later inspection by the test case + self.coredump_output = open(output_file_name, "w") + espcoredump_script = os.path.join(os.environ["IDF_PATH"], "components", "espcoredump", "espcoredump.py") + espcoredump_args = [ + sys.executable, + espcoredump_script, + "info_corefile", + "--core", coredump_file_name, + ] + espcoredump_args += extra_args + espcoredump_args.append(self.app.elf_file) + Utility.console_log("Running " + " ".join(espcoredump_args)) + Utility.console_log("espcoredump output is written to " + self.coredump_output.name) + + subprocess.check_call(espcoredump_args, stdout=self.coredump_output) + self.coredump_output.flush() + self.coredump_output.seek(0) + + def process_coredump_uart(self): + """ Extract the core dump from UART output of the test, run espcoredump on it """ + log_folder = self.app.get_log_folder(TEST_SUITE) + data = self.get_raw_data() + coredump_start = data.find(self.COREDUMP_UART_START) + coredump_end = data.find(self.COREDUMP_UART_END) + coredump_base64 = data[coredump_start + len(self.COREDUMP_UART_START):coredump_end] + with open(os.path.join(log_folder, "coredump_data_" + self.test_name + ".b64"), "w") as coredump_file: + Utility.console_log("Writing UART base64 core dump to " + coredump_file.name) + coredump_file.write(coredump_base64) + + output_file_name = os.path.join(log_folder, "coredump_uart_result_" + self.test_name + ".txt") + self._call_espcoredump(["--core-format", "b64"], coredump_file.name, output_file_name) + + def process_coredump_flash(self): + """ Extract the core dump from flash, run espcoredump on it """ + log_folder = self.app.get_log_folder(TEST_SUITE) + coredump_file_name = os.path.join(log_folder, "coredump_data_" + self.test_name + ".bin") + Utility.console_log("Writing flash binary core dump to " + coredump_file_name) + self.dump_flush(coredump_file_name, partition="coredump") + + output_file_name = os.path.join(log_folder, "coredump_flash_result_" + self.test_name + ".txt") + self._call_espcoredump(["--core-format", "raw"], coredump_file_name, output_file_name) + + +class ESP32PanicTestDUT(ttfw_idf.ESP32DUT, PanicTestMixin): + def get_gdb_remote(self): + return self.port + + +def panic_test(**kwargs): + """ Decorator for the panic tests, sets correct App and DUT classes """ + return ttfw_idf.idf_custom_test(app=PanicTestApp, dut=ESP32PanicTestDUT, env_tag="test_jtag_arm", **kwargs) + + +def get_dut(env, app_config_name, test_name, qemu_wdt_enable=False): + dut = env.get_dut("panic", TEST_PATH, app_config_name=app_config_name, allow_dut_exception=True) + dut.qemu_wdt_enable = qemu_wdt_enable + """ Wrapper for getting the DUT and starting the test """ + dut.start_test(test_name) + return dut + + +def run_all(filename): + """ Helper function to run all test cases defined in a file; to be called from __main__. """ + TinyFW.set_default_config(env_config_file=None, test_suite_name=TEST_SUITE) + test_methods = SearchCases.Search.search_test_cases(filename) + test_methods = filter(lambda m: not m.case_info["ignore"], test_methods) + test_cases = CaseConfig.Parser.apply_config(test_methods, None) + tests_failed = [] + for case in test_cases: + result = case.run() + if not result: + tests_failed.append(case) + + if tests_failed: + print("The following tests have failed:") + for case in tests_failed: + print(" - " + case.test_method.__name__) + raise SystemExit(1) + + print("Tests pass")