From 624828ce83adce851dea85e2e6e03868b9154aaa Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 11 Jan 2018 21:43:58 +0800 Subject: [PATCH] Dump the frame of the other core when interrupt watchdog happens --- components/esp32/cpu_start.c | 8 +- components/esp32/include/esp_int_wdt.h | 20 +++- components/esp32/int_wdt.c | 11 ++- components/esp32/panic.c | 94 ++++++++++++++++--- .../freertos/include/freertos/portable.h | 6 ++ components/freertos/port.c | 8 ++ 6 files changed, 124 insertions(+), 23 deletions(-) diff --git a/components/esp32/cpu_start.c b/components/esp32/cpu_start.c index 671bbd525..b66201c4f 100644 --- a/components/esp32/cpu_start.c +++ b/components/esp32/cpu_start.c @@ -1,4 +1,4 @@ -// Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD +// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -328,6 +328,8 @@ void start_cpu0_default(void) do_global_ctors(); #if CONFIG_INT_WDT esp_int_wdt_init(); + //Initialize the interrupt watch dog for CPU0. + esp_int_wdt_cpu_init(); #endif esp_cache_err_int_init(); esp_crosscore_int_init(); @@ -377,6 +379,10 @@ void start_cpu1_default(void) #if CONFIG_ESP32_APPTRACE_ENABLE esp_err_t err = esp_apptrace_init(); assert(err == ESP_OK && "Failed to init apptrace module on APP CPU!"); +#endif +#if CONFIG_INT_WDT + //Initialize the interrupt watch dog for CPU1. + esp_int_wdt_cpu_init(); #endif //Take care putting stuff here: if asked, FreeRTOS will happily tell you the scheduler //has started, but it isn't active *on this CPU* yet. diff --git a/components/esp32/include/esp_int_wdt.h b/components/esp32/include/esp_int_wdt.h index b32d0219f..f581d939a 100644 --- a/components/esp32/include/esp_int_wdt.h +++ b/components/esp32/include/esp_int_wdt.h @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ interrupts for too long, or code within interrupt handlers taking too long. It does this by setting up a watchdog which gets fed from the FreeRTOS task switch interrupt. When this watchdog times out, initially it will call a high-level interrupt routine that will panic FreeRTOS in order to allow -for forensic examination of the state of the CPU. When this interrupt +for forensic examination of the state of the both CPUs. When this interrupt handler is not called and the watchdog times out a second time, it will reset the SoC. @@ -38,12 +38,22 @@ This uses the TIMERG1 WDT. /** - * @brief Initialize the interrupt watchdog. This is called in the init code if - * the interrupt watchdog is enabled in menuconfig. + * @brief Initialize the non-CPU-specific parts of interrupt watchdog. + * This is called in the init code if the interrupt watchdog + * is enabled in menuconfig. * */ void esp_int_wdt_init(); +/** + * @brief Enable the interrupt watchdog on the current CPU. This is called + * in the init code by both CPUs if the interrupt watchdog is enabled + * in menuconfig. + * + */ +void esp_int_wdt_cpu_init(); + + /** * @} @@ -54,4 +64,4 @@ void esp_int_wdt_init(); } #endif -#endif \ No newline at end of file +#endif diff --git a/components/esp32/int_wdt.c b/components/esp32/int_wdt.c index debd675c4..efef92511 100644 --- a/components/esp32/int_wdt.c +++ b/components/esp32/int_wdt.c @@ -1,4 +1,4 @@ -// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// Copyright 2015-2018 Espressif Systems (Shanghai) PTE LTD // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -88,10 +88,11 @@ void esp_int_wdt_init() { TIMERG1.wdt_wprotect=0; TIMERG1.int_clr_timers.wdt=1; timer_group_intr_enable(TIMER_GROUP_1, TIMG_WDT_INT_ENA_M); - esp_register_freertos_tick_hook_for_cpu(tick_hook, 0); -#ifndef CONFIG_FREERTOS_UNICORE - esp_register_freertos_tick_hook_for_cpu(tick_hook, 1); -#endif +} + +void esp_int_wdt_cpu_init() +{ + esp_register_freertos_tick_hook_for_cpu(tick_hook, xPortGetCoreID()); ESP_INTR_DISABLE(WDT_INT_NUM); intr_matrix_set(xPortGetCoreID(), ETS_TG1_WDT_LEVEL_INTR_SOURCE, WDT_INT_NUM); //We do not register a handler for the interrupt because it is interrupt level 4 which diff --git a/components/esp32/panic.c b/components/esp32/panic.c index 9b347c4e5..83bc657f1 100644 --- a/components/esp32/panic.c +++ b/components/esp32/panic.c @@ -186,6 +186,12 @@ static void setFirstBreakpoint(uint32_t pc) ::"r"(pc):"a3", "a4"); } +//When interrupt watchdog happen in one core, both cores will be interrupted. +//The core which doesn't trigger the interrupt watchdog will save the frame and return. +//The core which triggers the interrupt watchdog will use the saved frame, and dump frames for both cores. +#if !CONFIG_FREERTOS_UNICORE +static volatile XtExcFrame * other_core_frame = NULL; +#endif //!CONFIG_FREERTOS_UNICORE void panicHandler(XtExcFrame *frame) { @@ -206,11 +212,26 @@ void panicHandler(XtExcFrame *frame) if (frame->exccause <= PANIC_RSN_MAX) { reason = reasons[frame->exccause]; } + +#if !CONFIG_FREERTOS_UNICORE + //Save frame for other core. + if ((frame->exccause == PANIC_RSN_INTWDT_CPU0 && core_id == 1) || (frame->exccause == PANIC_RSN_INTWDT_CPU1 && core_id == 0)) { + other_core_frame = frame; + while (1); + } + + //The core which triggers the interrupt watchdog will delay 1 us, so the other core can save its frame. + if (frame->exccause == PANIC_RSN_INTWDT_CPU0 || frame->exccause == PANIC_RSN_INTWDT_CPU1) { + ets_delay_us(1); + } + if (frame->exccause == PANIC_RSN_CACHEERR && esp_cache_err_get_cpuid() != core_id) { // Cache error interrupt will be handled by the panic handler // on the other CPU. - return; + while (1); } +#endif //!CONFIG_FREERTOS_UNICORE + haltOtherCore(); esp_dport_access_int_abort(); panicPutStr("Guru Meditation Error: Core "); @@ -428,10 +449,9 @@ static void doBacktrace(XtExcFrame *frame) } /* - We arrive here after a panic or unhandled exception, when no OCD is detected. Dump the registers to the - serial port and either jump to the gdb stub, halt the CPU or reboot. -*/ -static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame) + * Dump registers and do backtrace. + */ +static void commonErrorHandler_dump(XtExcFrame *frame, int core_id) { int *regs = (int *)frame; int x, y; @@ -441,17 +461,13 @@ static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame) "A14 ", "A15 ", "SAR ", "EXCCAUSE", "EXCVADDR", "LBEG ", "LEND ", "LCOUNT " }; - // start panic WDT to restart system if we hang in this handler - esp_panic_wdt_start(); - - //Feed the watchdogs, so they will give us time to print out debug info - reconfigureAllWdts(); - /* only dump registers for 'real' crashes, if crashing via abort() the register window is no longer useful. */ if (!abort_called) { - panicPutStr("Register dump:\r\n"); + panicPutStr("Core"); + panicPutDec(core_id); + panicPutStr(" register dump:\r\n"); for (x = 0; x < 24; x += 4) { for (y = 0; y < 4; y++) { @@ -464,11 +480,65 @@ static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame) } panicPutStr("\r\n"); } + + if (xPortInterruptedFromISRContext() +#if !CONFIG_FREERTOS_UNICORE + && other_core_frame != frame +#endif //!CONFIG_FREERTOS_UNICORE + ) { + //If the core which triggers the interrupt watchdog was in ISR context, dump the epc registers. + uint32_t __value; + panicPutStr("Core"); + panicPutDec(core_id); + panicPutStr(" was running in ISR context:\r\n"); + + __asm__("rsr.epc1 %0" : "=a"(__value)); + panicPutStr("EPC1 : 0x"); + panicPutHex(__value); + + __asm__("rsr.epc2 %0" : "=a"(__value)); + panicPutStr(" EPC2 : 0x"); + panicPutHex(__value); + + __asm__("rsr.epc3 %0" : "=a"(__value)); + panicPutStr(" EPC3 : 0x"); + panicPutHex(__value); + + __asm__("rsr.epc4 %0" : "=a"(__value)); + panicPutStr(" EPC4 : 0x"); + panicPutHex(__value); + + panicPutStr("\r\n"); + } + } /* With windowed ABI backtracing is easy, let's do it. */ doBacktrace(frame); +} + +/* + We arrive here after a panic or unhandled exception, when no OCD is detected. Dump the registers to the + serial port and either jump to the gdb stub, halt the CPU or reboot. +*/ +static __attribute__((noreturn)) void commonErrorHandler(XtExcFrame *frame) +{ + + int core_id = xPortGetCoreID(); + // start panic WDT to restart system if we hang in this handler + esp_panic_wdt_start(); + + //Feed the watchdogs, so they will give us time to print out debug info + reconfigureAllWdts(); + + commonErrorHandler_dump(frame, core_id); +#if !CONFIG_FREERTOS_UNICORE + if (other_core_frame != NULL) { + commonErrorHandler_dump((XtExcFrame *)other_core_frame, (core_id ? 0 : 1)); + } +#endif //!CONFIG_FREERTOS_UNICORE + #if CONFIG_ESP32_APPTRACE_ENABLE disableAllWdts(); #if CONFIG_SYSVIEW_ENABLE diff --git a/components/freertos/include/freertos/portable.h b/components/freertos/include/freertos/portable.h index b0fe405f7..f999a1f9e 100644 --- a/components/freertos/include/freertos/portable.h +++ b/components/freertos/include/freertos/portable.h @@ -183,6 +183,12 @@ void vPortSetStackWatchpoint( void* pxStackStart ); */ BaseType_t xPortInIsrContext(); +/* + * This function will be called in High prio ISRs. Returns true if the current core was in ISR context + * before calling into high prio ISR context. + */ +BaseType_t xPortInterruptedFromISRContext(); + /* * The structures and methods of manipulating the MPU are contained within the * port layer. diff --git a/components/freertos/port.c b/components/freertos/port.c index 734e3d66c..5e2c3c8f8 100644 --- a/components/freertos/port.c +++ b/components/freertos/port.c @@ -288,6 +288,14 @@ BaseType_t xPortInIsrContext() return ret; } +/* + * This function will be called in High prio ISRs. Returns true if the current core was in ISR context + * before calling into high prio ISR context. + */ +BaseType_t IRAM_ATTR xPortInterruptedFromISRContext() +{ + return (port_interruptNesting[xPortGetCoreID()] != 0); +} void vPortAssertIfInISR() {