From 73592d9bc4348fd6c4eff6ce135bb73363b438d5 Mon Sep 17 00:00:00 2001 From: Felipe Neves Date: Wed, 22 Jan 2020 06:20:34 +0800 Subject: [PATCH] spin_lock: added new spinlock interface and decoupled it from RTOS spin_lock: cleaned-up port files and removed portmux files components/soc: decoupled compare and set operations from FreeRTOS soc/spinlock: filled initial implementation of spinlock refactor It will decouple the spinlocks into separated components with not depencences of freertos an similar interface was provided focusing the readabillity and maintenance, also naming to spinlocks were adopted. On FreeRTOS side the legacy portMUX macros gained a form of wrapper functions that calls the spinlocks component thus minimizing the impact on RTOS side. This feature aims to close IDF-967 soc/spinlock: spinlocks passed on unit test, missing test corner cases components/compare_set: added better function namings plus minor performance optimization on spinlocks soc/spinlock: code reordering to remove ISC C90 mix error freertos/portmacro: gor rid of critical sections multiline macros, placed inline functions instead soc/spinlock: improved spinlock performance from internal RAM For cases where the spinlock is executed from IRAM, there is no need to check where the spinlock object is placed on memory, removing this checks caused a great improvement on performance. --- components/freertos/Kconfig | 29 -- .../freertos/include/freertos/portable.h | 13 +- .../freertos/include/freertos/portmacro.h | 327 +++++++----------- components/freertos/port.c | 144 +++----- components/freertos/portmux_impl.h | 118 ------- components/freertos/portmux_impl.inc.h | 169 --------- components/freertos/tasks.c | 37 +- components/soc/CMakeLists.txt | 1 + components/soc/include/soc/compare_set.h | 56 +++ components/soc/include/soc/spinlock.h | 153 ++++++++ components/soc/src/compare_set.c | 40 +++ docs/en/api-guides/freertos-smp.rst | 19 +- 12 files changed, 436 insertions(+), 670 deletions(-) delete mode 100644 components/freertos/portmux_impl.h delete mode 100644 components/freertos/portmux_impl.inc.h create mode 100644 components/soc/include/soc/compare_set.h create mode 100644 components/soc/include/soc/spinlock.h create mode 100644 components/soc/src/compare_set.c diff --git a/components/freertos/Kconfig b/components/freertos/Kconfig index 86544ab91..abc8ff218 100644 --- a/components/freertos/Kconfig +++ b/components/freertos/Kconfig @@ -389,35 +389,6 @@ menu "FreeRTOS" FreeRTOS will enter light sleep mode if no tasks need to run for this number of ticks. - menuconfig FREERTOS_DEBUG_INTERNALS - bool "Debug FreeRTOS internals" - default n - help - Enable this option to show the menu with internal FreeRTOS debugging features. - This option does not change any code by itself, it just shows/hides some options. - - if FREERTOS_DEBUG_INTERNALS - - config FREERTOS_PORTMUX_DEBUG - bool "Debug portMUX portENTER_CRITICAL/portEXIT_CRITICAL" - depends on FREERTOS_DEBUG_INTERNALS - default n - help - If enabled, debug information (including integrity checks) will be printed - to UART for the port-specific MUX implementation. - - if !FREERTOS_UNICORE - config FREERTOS_PORTMUX_DEBUG_RECURSIVE - bool "Debug portMUX Recursion" - depends on FREERTOS_PORTMUX_DEBUG - default n - help - If enabled, additional debug information will be printed for recursive - portMUX usage. - endif #FREERTOS_UNICORE - - endif # FREERTOS_DEBUG_INTERNALS - config FREERTOS_TASK_FUNCTION_WRAPPER bool "Enclose all task functions in a wrapper function" depends on COMPILER_OPTIMIZATION_DEFAULT diff --git a/components/freertos/include/freertos/portable.h b/components/freertos/include/freertos/portable.h index 7fab98527..b87495841 100644 --- a/components/freertos/include/freertos/portable.h +++ b/components/freertos/include/freertos/portable.h @@ -179,12 +179,6 @@ void vPortYieldOtherCore( BaseType_t coreid) PRIVILEGED_FUNCTION; */ void vPortSetStackWatchpoint( void* pxStackStart ); -/* - * Returns true if the current core is in ISR context; low prio ISR, med prio ISR or timer tick ISR. High prio ISRs - * aren't detected here, but they normally cannot call C code, so that should not be an issue anyway. - */ -BaseType_t xPortInIsrContext(void); - /* * This function will be called in High prio ISRs. Returns true if the current core was in ISR context * before calling into high prio ISR context. @@ -239,7 +233,12 @@ static inline bool IRAM_ATTR xPortCanYield(void) } #endif -void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set); +static inline void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + compare_and_set_extram(addr, compare, set); +#endif +} #endif /* PORTABLE_H */ diff --git a/components/freertos/include/freertos/portmacro.h b/components/freertos/include/freertos/portmacro.h index f544c2d76..1ba84d68a 100644 --- a/components/freertos/include/freertos/portmacro.h +++ b/components/freertos/include/freertos/portmacro.h @@ -82,8 +82,7 @@ extern "C" { #include #include "esp_private/crosscore_int.h" #include "esp_timer.h" /* required for FreeRTOS run time stats */ - - +#include "soc/spinlock.h" #include #include "sdkconfig.h" @@ -133,57 +132,24 @@ typedef unsigned portBASE_TYPE UBaseType_t; #include "sdkconfig.h" #include "esp_attr.h" -/* "mux" data structure (spinlock) */ -typedef struct { - /* owner field values: - * 0 - Uninitialized (invalid) - * portMUX_FREE_VAL - Mux is free, can be locked by either CPU - * CORE_ID_REGVAL_PRO / CORE_ID_REGVAL_APP - Mux is locked to the particular core - * - * Note that for performance reasons we use the full Xtensa CORE ID values - * (CORE_ID_REGVAL_PRO, CORE_ID_REGVAL_APP) and not the 0,1 values which are used in most - * other FreeRTOS code. - * - * Any value other than portMUX_FREE_VAL, CORE_ID_REGVAL_PRO, CORE_ID_REGVAL_APP indicates corruption - */ - uint32_t owner; - /* count field: - * If mux is unlocked, count should be zero. - * If mux is locked, count is non-zero & represents the number of recursive locks on the mux. - */ - uint32_t count; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - const char *lastLockedFn; - int lastLockedLine; -#endif -} portMUX_TYPE; +static inline uint32_t xPortGetCoreID(void); -#define portMUX_FREE_VAL 0xB33FFFFF +// Critical section management. NW-TODO: replace XTOS_SET_INTLEVEL with more efficient version, if any? +// These cannot be nested. They should be used with a lot of care and cannot be called from interrupt level. +// +// Only applies to one CPU. See notes above & below for reasons not to use these. +#define portDISABLE_INTERRUPTS() do { XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); portbenchmarkINTERRUPT_DISABLE(); } while (0) +#define portENABLE_INTERRUPTS() do { portbenchmarkINTERRUPT_RESTORE(0); XTOS_SET_INTLEVEL(0); } while (0) -/* Special constants for vPortCPUAcquireMutexTimeout() */ -#define portMUX_NO_TIMEOUT (-1) /* When passed for 'timeout_cycles', spin forever if necessary */ -#define portMUX_TRY_LOCK 0 /* Try to acquire the spinlock a single time only */ - -// Keep this in sync with the portMUX_TYPE struct definition please. -#ifndef CONFIG_FREERTOS_PORTMUX_DEBUG -#define portMUX_INITIALIZER_UNLOCKED { \ - .owner = portMUX_FREE_VAL, \ - .count = 0, \ - } -#else -#define portMUX_INITIALIZER_UNLOCKED { \ - .owner = portMUX_FREE_VAL, \ - .count = 0, \ - .lastLockedFn = "(never locked)", \ - .lastLockedLine = -1 \ - } -#endif - - -#define portASSERT_IF_IN_ISR() vPortAssertIfInISR() -void vPortAssertIfInISR(void); - -#define portCRITICAL_NESTING_IN_TCB 1 +// Cleaner solution allows nested interrupts disabling and restoring via local registers or stack. +// They can be called from interrupts too. +// WARNING: Only applies to current CPU. See notes above. +static inline unsigned portENTER_CRITICAL_NESTED(void) { + unsigned state = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + portbenchmarkINTERRUPT_DISABLE(); + return state; +} +#define portEXIT_CRITICAL_NESTED(state) do { portbenchmarkINTERRUPT_RESTORE(state); XTOS_RESTORE_JUST_INTLEVEL(state); } while (0) /* Modifications to portENTER_CRITICAL. @@ -211,125 +177,119 @@ Remark: For the ESP32, portENTER_CRITICAL and portENTER_CRITICAL_ISR both alias that either function can be called both from ISR as well as task context. This is not standard FreeRTOS behaviour; please keep this in mind if you need any compatibility with other FreeRTOS implementations. */ -void vPortCPUInitializeMutex(portMUX_TYPE *mux); -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUAcquireMutex(portMUX_TYPE *mux, const char *function, int line); -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles, const char *function, int line); -void vPortCPUReleaseMutex(portMUX_TYPE *mux, const char *function, int line); +/* "mux" data structure (spinlock) */ +typedef struct { + spinlock_t spinlock; +} portMUX_TYPE; -void vTaskEnterCritical( portMUX_TYPE *mux, const char *function, int line ); -void vTaskExitCritical( portMUX_TYPE *mux, const char *function, int line ); +#define portMUX_FREE_VAL SPINLOCK_FREE +#define portMUX_NO_TIMEOUT SPINLOCK_WAIT_FOREVER /* When passed for 'timeout_cycles', spin forever if necessary */ +#define portMUX_TRY_LOCK SPINLOCK_NO_WAIT /* Try to acquire the spinlock a single time only */ +#define portMUX_INITIALIZER_UNLOCKED {.spinlock=SPINLOCK_INITIALIZER} -#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -/* Calling port*_CRITICAL from ISR context would cause an assert failure. - * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE - */ -#define portENTER_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskEnterCritical(mux, __FUNCTION__, __LINE__); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) +#define portASSERT_IF_IN_ISR() vPortAssertIfInISR() +void vPortAssertIfInISR(void); -#define portEXIT_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskExitCritical(mux, __FUNCTION__, __LINE__); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) -#else -#define portENTER_CRITICAL(mux) vTaskEnterCritical(mux, __FUNCTION__, __LINE__) -#define portEXIT_CRITICAL(mux) vTaskExitCritical(mux, __FUNCTION__, __LINE__) -#endif -#define portENTER_CRITICAL_ISR(mux) vTaskEnterCritical(mux, __FUNCTION__, __LINE__) -#define portEXIT_CRITICAL_ISR(mux) vTaskExitCritical(mux, __FUNCTION__, __LINE__) -#else -void vTaskExitCritical( portMUX_TYPE *mux ); -void vTaskEnterCritical( portMUX_TYPE *mux ); -void vPortCPUAcquireMutex(portMUX_TYPE *mux); +#define portCRITICAL_NESTING_IN_TCB 0 -/** @brief Acquire a portmux spinlock with a timeout - * - * @param mux Pointer to portmux to acquire. - * @param timeout_cycles Timeout to spin, in CPU cycles. Pass portMUX_NO_TIMEOUT to wait forever, - * portMUX_TRY_LOCK to try a single time to acquire the lock. - * - * @return true if mutex is successfully acquired, false on timeout. - */ -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles); -void vPortCPUReleaseMutex(portMUX_TYPE *mux); - -#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -/* Calling port*_CRITICAL from ISR context would cause an assert failure. - * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE - */ -#define portENTER_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskEnterCritical(mux); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) - -#define portEXIT_CRITICAL(mux) do { \ - if(!xPortInIsrContext()) { \ - vTaskExitCritical(mux); \ - } else { \ - ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, \ - __FUNCTION__); \ - abort(); \ - } \ - } while(0) -#else -#define portENTER_CRITICAL(mux) vTaskEnterCritical(mux) -#define portEXIT_CRITICAL(mux) vTaskExitCritical(mux) -#endif -#define portENTER_CRITICAL_ISR(mux) vTaskEnterCritical(mux) -#define portEXIT_CRITICAL_ISR(mux) vTaskExitCritical(mux) -#endif - -#define portENTER_CRITICAL_SAFE(mux) do { \ - if (xPortInIsrContext()) { \ - portENTER_CRITICAL_ISR(mux); \ - } else { \ - portENTER_CRITICAL(mux); \ - } \ - } while(0) - -#define portEXIT_CRITICAL_SAFE(mux) do { \ - if (xPortInIsrContext()) { \ - portEXIT_CRITICAL_ISR(mux); \ - } else { \ - portEXIT_CRITICAL(mux); \ - } \ - } while(0) - - -// Critical section management. NW-TODO: replace XTOS_SET_INTLEVEL with more efficient version, if any? -// These cannot be nested. They should be used with a lot of care and cannot be called from interrupt level. -// -// Only applies to one CPU. See notes above & below for reasons not to use these. -#define portDISABLE_INTERRUPTS() do { XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); portbenchmarkINTERRUPT_DISABLE(); } while (0) -#define portENABLE_INTERRUPTS() do { portbenchmarkINTERRUPT_RESTORE(0); XTOS_SET_INTLEVEL(0); } while (0) - -// Cleaner solution allows nested interrupts disabling and restoring via local registers or stack. -// They can be called from interrupts too. -// WARNING: Only applies to current CPU. See notes above. -static inline unsigned portENTER_CRITICAL_NESTED(void) { - unsigned state = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); - portbenchmarkINTERRUPT_DISABLE(); - return state; +static inline void __attribute__((always_inline)) vPortCPUInitializeMutex(portMUX_TYPE *mux) +{ + spinlock_initialize(&mux->spinlock); +} + +static inline void __attribute__((always_inline)) vPortCPUAcquireMutex(portMUX_TYPE *mux) +{ + spinlock_acquire(&mux->spinlock, portMUX_NO_TIMEOUT); +} + +static inline bool __attribute__((always_inline)) vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout) +{ + return (spinlock_acquire(&mux->spinlock, timeout)); +} + +static inline void __attribute__((always_inline)) vPortCPUReleaseMutex(portMUX_TYPE *mux) +{ + spinlock_release(&mux->spinlock); +} + +void vPortEnterCritical(portMUX_TYPE *mux); +void vPortExitCritical(portMUX_TYPE *mux); + +/* + * Returns true if the current core is in ISR context; low prio ISR, med prio ISR or timer tick ISR. High prio ISRs + * aren't detected here, but they normally cannot call C code, so that should not be an issue anyway. + */ +BaseType_t xPortInIsrContext(void); + +static inline void __attribute__((always_inline)) vPortEnterCriticalCompliance(portMUX_TYPE *mux) +{ + if(!xPortInIsrContext()) { + vPortEnterCritical(mux); + } else { + ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, + __FUNCTION__); + abort(); + } +} + +static inline void __attribute__((always_inline)) vPortExitCriticalCompliance(portMUX_TYPE *mux) +{ + if(!xPortInIsrContext()) { + vPortExitCritical(mux); + } else { + ets_printf("%s:%d (%s)- port*_CRITICAL called from ISR context!\n", __FILE__, __LINE__, + __FUNCTION__); + abort(); + } +} + +#ifdef CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE +/* Calling port*_CRITICAL from ISR context would cause an assert failure. + * If the parent function is called from both ISR and Non-ISR context then call port*_CRITICAL_SAFE + */ +#define portENTER_CRITICAL(mux) vPortEnterCriticalCompliance(mux) +#define portEXIT_CRITICAL(mux) vPortExitCriticalCompliance(mux) +#else +#define portENTER_CRITICAL(mux) vPortEnterCritical(mux) +#define portEXIT_CRITICAL(mux) vPortExitCritical(mux) +#endif + +#define portENTER_CRITICAL_ISR(mux) vPortEnterCritical(mux) +#define portEXIT_CRITICAL_ISR(mux) vPortExitCritical(mux) + +static inline void __attribute__((always_inline)) vPortEnterCriticalSafe(portMUX_TYPE *mux) +{ + if (xPortInIsrContext()) { + portENTER_CRITICAL_ISR(mux); + } else { + portENTER_CRITICAL(mux); + } +} + +static inline void __attribute__((always_inline)) vPortExitCriticalSafe(portMUX_TYPE *mux) +{ + if (xPortInIsrContext()) { + portEXIT_CRITICAL_ISR(mux); + } else { + portEXIT_CRITICAL(mux); + } +} + +#define portENTER_CRITICAL_SAFE(mux) vPortEnterCriticalSafe(mux) +#define portEXIT_CRITICAL_SAFE(mux) vPortExitCriticalSafe(mux) +/* + * Wrapper for the Xtensa compare-and-set instruction. This subroutine will atomically compare + * *addr to 'compare'. If *addr == compare, *addr is set to *set. *set is updated with the previous + * value of *addr (either 'compare' or some other value.) + * + * Warning: From the ISA docs: in some (unspecified) cases, the s32c1i instruction may return the + * *bitwise inverse* of the old mem if the mem wasn't written. This doesn't seem to happen on the + * ESP32 (portMUX assertions would fail). + */ +static inline void __attribute__((always_inline)) uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { + compare_and_set_native(addr, compare, set); } -#define portEXIT_CRITICAL_NESTED(state) do { portbenchmarkINTERRUPT_RESTORE(state); XTOS_RESTORE_JUST_INTLEVEL(state); } while (0) // These FreeRTOS versions are similar to the nested versions above #define portSET_INTERRUPT_MASK_FROM_ISR() portENTER_CRITICAL_NESTED() @@ -343,43 +303,6 @@ static inline unsigned portENTER_CRITICAL_NESTED(void) { #define pvPortMallocTcbMem(size) heap_caps_malloc(size, portTcbMemoryCaps) #define pvPortMallocStackMem(size) heap_caps_malloc(size, portStackMemoryCaps) -/* - * Wrapper for the Xtensa compare-and-set instruction. This subroutine will atomically compare - * *addr to 'compare'. If *addr == compare, *addr is set to *set. *set is updated with the previous - * value of *addr (either 'compare' or some other value.) - * - * Warning: From the ISA docs: in some (unspecified) cases, the s32c1i instruction may return the - * *bitwise inverse* of the old mem if the mem wasn't written. This doesn't seem to happen on the - * ESP32 (portMUX assertions would fail). - */ -static inline void uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { -#if XCHAL_HAVE_S32C1I - __asm__ __volatile__ ( - "WSR %2,SCOMPARE1 \n" - "S32C1I %0, %1, 0 \n" - :"=r"(*set) - :"r"(addr), "r"(compare), "0"(*set) - ); -#else - // No S32C1I, so do this by disabling and re-enabling interrupts (slower) - uint32_t intlevel, old_value; - __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" - : "=r"(intlevel)); - - old_value = *addr; - if (old_value == compare) { - *addr = *set; - } - - __asm__ __volatile__ ("memw \n" - "wsr %0, ps\n" - :: "r"(intlevel)); - - *set = old_value; -#endif -} - - /*-----------------------------------------------------------*/ /* Architecture specifics. */ @@ -400,14 +323,12 @@ static inline void uxPortCompareSet(volatile uint32_t *addr, uint32_t compare, u #endif - /* Kernel utilities. */ void vPortYield( void ); void _frxt_setup_switch( void ); #define portYIELD() vPortYield() #define portYIELD_FROM_ISR() {traceISR_EXIT_TO_SCHEDULER(); _frxt_setup_switch();} -static inline uint32_t xPortGetCoreID(void); /* Yielding within an API call (when interrupts are off), means the yield should be delayed until interrupts are re-enabled. diff --git a/components/freertos/port.c b/components/freertos/port.c index 0fb5a97a9..0ae24f981 100644 --- a/components/freertos/port.c +++ b/components/freertos/port.c @@ -127,10 +127,10 @@ extern void _xt_coproc_init(void); _Static_assert(tskNO_AFFINITY == CONFIG_FREERTOS_NO_AFFINITY, "incorrect tskNO_AFFINITY value"); /*-----------------------------------------------------------*/ - unsigned port_xSchedulerRunning[portNUM_PROCESSORS] = {0}; // Duplicate of inaccessible xSchedulerRunning; needed at startup to avoid counting nesting unsigned port_interruptNesting[portNUM_PROCESSORS] = {0}; // Interrupt nesting level. Increased/decreased in portasm.c, _frxt_int_enter/_frxt_int_exit - +BaseType_t port_uxCriticalNesting[portNUM_PROCESSORS] = {0}; +BaseType_t port_uxOldInterruptState[portNUM_PROCESSORS] = {0}; /*-----------------------------------------------------------*/ // User exception dispatcher when exiting @@ -355,74 +355,6 @@ void vPortAssertIfInISR(void) configASSERT(xPortInIsrContext()); } -/* - * For kernel use: Initialize a per-CPU mux. Mux will be initialized unlocked. - */ -void vPortCPUInitializeMutex(portMUX_TYPE *mux) { - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - ets_printf("Initializing mux %p\n", mux); - mux->lastLockedFn="(never locked)"; - mux->lastLockedLine=-1; -#endif - mux->owner=portMUX_FREE_VAL; - mux->count=0; -} - -#include "portmux_impl.h" - -/* - * For kernel use: Acquire a per-CPU mux. Spinlocks, so don't hold on to these muxes for too long. - */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUAcquireMutex(portMUX_TYPE *mux, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUAcquireMutexIntsDisabled(mux, portMUX_NO_TIMEOUT, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); -} - -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - bool result = vPortCPUAcquireMutexIntsDisabled(mux, timeout_cycles, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); - return result; -} - -#else -void vPortCPUAcquireMutex(portMUX_TYPE *mux) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUAcquireMutexIntsDisabled(mux, portMUX_NO_TIMEOUT); - portEXIT_CRITICAL_NESTED(irqStatus); -} - -bool vPortCPUAcquireMutexTimeout(portMUX_TYPE *mux, int timeout_cycles) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - bool result = vPortCPUAcquireMutexIntsDisabled(mux, timeout_cycles); - portEXIT_CRITICAL_NESTED(irqStatus); - return result; -} -#endif - - -/* - * For kernel use: Release a per-CPU mux - * - * Mux must be already locked by this core - */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -void vPortCPUReleaseMutex(portMUX_TYPE *mux, const char *fnName, int line) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUReleaseMutexIntsDisabled(mux, fnName, line); - portEXIT_CRITICAL_NESTED(irqStatus); -} -#else -void vPortCPUReleaseMutex(portMUX_TYPE *mux) { - unsigned int irqStatus = portENTER_CRITICAL_NESTED(); - vPortCPUReleaseMutexIntsDisabled(mux); - portEXIT_CRITICAL_NESTED(irqStatus); -} -#endif - void vPortSetStackWatchpoint( void* pxStackStart ) { //Set watchpoint 1 to watch the last 32 bytes of the stack. //Unfortunately, the Xtensa watchpoints can't set a watchpoint on a random [base - base+n] region because @@ -435,39 +367,45 @@ void vPortSetStackWatchpoint( void* pxStackStart ) { esp_set_watchpoint(1, (char*)addr, 32, ESP_WATCHPOINT_STORE); } -#if defined(CONFIG_SPIRAM) -/* - * Compare & set (S32C1) does not work in external RAM. Instead, this routine uses a mux (in internal memory) to fake it. - */ -static portMUX_TYPE extram_mux = portMUX_INITIALIZER_UNLOCKED; - -void uxPortCompareSetExtram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) { - uint32_t prev; - - uint32_t oldlevel = portENTER_CRITICAL_NESTED(); - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutexIntsDisabled(&extram_mux, portMUX_NO_TIMEOUT, __FUNCTION__, __LINE__); -#else - vPortCPUAcquireMutexIntsDisabled(&extram_mux, portMUX_NO_TIMEOUT); -#endif - prev=*addr; - if (prev==compare) { - *addr=*set; - } - *set=prev; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutexIntsDisabled(&extram_mux, __FUNCTION__, __LINE__); -#else - vPortCPUReleaseMutexIntsDisabled(&extram_mux); -#endif - - portEXIT_CRITICAL_NESTED(oldlevel); -} -#endif //defined(CONFIG_SPIRAM) - - - uint32_t xPortGetTickRateHz(void) { return (uint32_t)configTICK_RATE_HZ; } + +void __attribute__((optimize("-O3"))) vPortEnterCritical(portMUX_TYPE *mux) +{ + BaseType_t oldInterruptLevel = portENTER_CRITICAL_NESTED(); + /* Interrupts may already be disabled (because we're doing this recursively) + * but we can't get the interrupt level after + * vPortCPUAquireMutex, because it also may mess with interrupts. + * Get it here first, then later figure out if we're nesting + * and save for real there. + */ + vPortCPUAcquireMutex( mux ); + BaseType_t coreID = xPortGetCoreID(); + BaseType_t newNesting = port_uxCriticalNesting[coreID] + 1; + port_uxCriticalNesting[coreID] = newNesting; + + if( newNesting == 1 ) + { + //This is the first time we get called. Save original interrupt level. + port_uxOldInterruptState[coreID] = oldInterruptLevel; + } +} + +void __attribute__((optimize("-O3"))) vPortExitCritical(portMUX_TYPE *mux) +{ + vPortCPUReleaseMutex( mux ); + BaseType_t coreID = xPortGetCoreID(); + BaseType_t nesting = port_uxCriticalNesting[coreID]; + + if(nesting > 0U) + { + nesting--; + port_uxCriticalNesting[coreID] = nesting; + + if( nesting == 0U ) + { + portEXIT_CRITICAL_NESTED(port_uxOldInterruptState[coreID]); + } + } +} diff --git a/components/freertos/portmux_impl.h b/components/freertos/portmux_impl.h deleted file mode 100644 index af170bb3e..000000000 --- a/components/freertos/portmux_impl.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - Copyright (C) 2016-2017 Espressif Shanghai PTE LTD - Copyright (C) 2015 Real Time Engineers Ltd. - - All rights reserved - - FreeRTOS is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License (version 2) as published by the - Free Software Foundation >>!AND MODIFIED BY!<< the FreeRTOS exception. - - *************************************************************************** - >>! NOTE: The modification to the GPL is included to allow you to !<< - >>! distribute a combined work that includes FreeRTOS without being !<< - >>! obliged to provide the source code for proprietary components !<< - >>! outside of the FreeRTOS kernel. !<< - *************************************************************************** - - FreeRTOS is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - FOR A PARTICULAR PURPOSE. Full license text is available on the following - link: http://www.freertos.org/a00114.html -*/ - -/* This header exists for performance reasons, in order to inline the - implementation of vPortCPUAcquireMutexIntsDisabled and - vPortCPUReleaseMutexIntsDisabled into the - vTaskEnterCritical/vTaskExitCritical functions in task.c as well as the - vPortCPUAcquireMutex/vPortCPUReleaseMutex implementations. - - Normally this kind of performance hack is over the top, but - vTaskEnterCritical/vTaskExitCritical is called a great - deal by FreeRTOS internals. - - It should be #included by freertos port.c or tasks.c, in esp-idf. - - The way it works is that it essentially uses portmux_impl.inc.h as a - generator template of sorts. When no external memory is used, this - template is only used to generate the vPortCPUAcquireMutexIntsDisabledInternal - and vPortCPUReleaseMutexIntsDisabledInternal functions, which use S32C1 to - do an atomic compare & swap. When external memory is used the functions - vPortCPUAcquireMutexIntsDisabledExtram and vPortCPUReleaseMutexIntsDisabledExtram - are also generated, which use uxPortCompareSetExtram to fake the S32C1 instruction. - The wrapper functions vPortCPUAcquireMutexIntsDisabled and - vPortCPUReleaseMutexIntsDisabled will then use the appropriate function to do the - actual lock/unlock. -*/ -#include "soc/cpu.h" -#include "portable.h" -#include "soc/soc_memory_layout.h" - -/* XOR one core ID with this value to get the other core ID */ -#define CORE_ID_REGVAL_XOR_SWAP (CORE_ID_REGVAL_PRO ^ CORE_ID_REGVAL_APP) - - - - -//Define the mux routines for use with muxes in internal RAM -#define PORTMUX_AQUIRE_MUX_FN_NAME vPortCPUAcquireMutexIntsDisabledInternal -#define PORTMUX_RELEASE_MUX_FN_NAME vPortCPUReleaseMutexIntsDisabledInternal -#define PORTMUX_COMPARE_SET_FN_NAME uxPortCompareSet -#include "portmux_impl.inc.h" -#undef PORTMUX_AQUIRE_MUX_FN_NAME -#undef PORTMUX_RELEASE_MUX_FN_NAME -#undef PORTMUX_COMPARE_SET_FN_NAME - - -#if defined(CONFIG_SPIRAM) - -#define PORTMUX_AQUIRE_MUX_FN_NAME vPortCPUAcquireMutexIntsDisabledExtram -#define PORTMUX_RELEASE_MUX_FN_NAME vPortCPUReleaseMutexIntsDisabledExtram -#define PORTMUX_COMPARE_SET_FN_NAME uxPortCompareSetExtram -#include "portmux_impl.inc.h" -#undef PORTMUX_AQUIRE_MUX_FN_NAME -#undef PORTMUX_RELEASE_MUX_FN_NAME -#undef PORTMUX_COMPARE_SET_FN_NAME - -#endif - - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -#define PORTMUX_AQUIRE_MUX_FN_ARGS portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line -#define PORTMUX_RELEASE_MUX_FN_ARGS portMUX_TYPE *mux, const char *fnName, int line -#define PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(x) x, timeout_cycles, fnName, line -#define PORTMUX_RELEASE_MUX_FN_CALL_ARGS(x) x, fnName, line -#else -#define PORTMUX_AQUIRE_MUX_FN_ARGS portMUX_TYPE *mux, int timeout_cycles -#define PORTMUX_RELEASE_MUX_FN_ARGS portMUX_TYPE *mux -#define PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(x) x, timeout_cycles -#define PORTMUX_RELEASE_MUX_FN_CALL_ARGS(x) x -#endif - - -static inline bool __attribute__((always_inline)) vPortCPUAcquireMutexIntsDisabled(PORTMUX_AQUIRE_MUX_FN_ARGS) { -#if !defined(CONFIG_FREERTOS_UNICORE) -#if defined(CONFIG_SPIRAM) - if (esp_ptr_external_ram(mux)) { - return vPortCPUAcquireMutexIntsDisabledExtram(PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(mux)); - } -#endif - return vPortCPUAcquireMutexIntsDisabledInternal(PORTMUX_AQUIRE_MUX_FN_CALL_ARGS(mux)); -#else - return true; -#endif -} - - -static inline void vPortCPUReleaseMutexIntsDisabled(PORTMUX_RELEASE_MUX_FN_ARGS) { -#if !defined(CONFIG_FREERTOS_UNICORE) -#if defined(CONFIG_SPIRAM) - if (esp_ptr_external_ram(mux)) { - vPortCPUReleaseMutexIntsDisabledExtram(PORTMUX_RELEASE_MUX_FN_CALL_ARGS(mux)); - return; - } -#endif - vPortCPUReleaseMutexIntsDisabledInternal(PORTMUX_RELEASE_MUX_FN_CALL_ARGS(mux)); -#endif -} - diff --git a/components/freertos/portmux_impl.inc.h b/components/freertos/portmux_impl.inc.h deleted file mode 100644 index 908fec153..000000000 --- a/components/freertos/portmux_impl.inc.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright (C) 2016-2017 Espressif Shanghai PTE LTD - Copyright (C) 2015 Real Time Engineers Ltd. - - All rights reserved - - FreeRTOS is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License (version 2) as published by the - Free Software Foundation >>!AND MODIFIED BY!<< the FreeRTOS exception. - - *************************************************************************** - >>! NOTE: The modification to the GPL is included to allow you to !<< - >>! distribute a combined work that includes FreeRTOS without being !<< - >>! obliged to provide the source code for proprietary components !<< - >>! outside of the FreeRTOS kernel. !<< - *************************************************************************** - - FreeRTOS is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - FOR A PARTICULAR PURPOSE. Full license text is available on the following - link: http://www.freertos.org/a00114.html -*/ - - -/* - Warning: funky preprocessor hackery ahead. Including these headers will generate two - functions, which names are defined by the preprocessor macros - PORTMUX_AQUIRE_MUX_FN_NAME and PORTMUX_RELEASE_MUX_FN_NAME. In order to do the compare - and exchange function, they will use whatever PORTMUX_COMPARE_SET_FN_NAME resolves to. - - In some scenarios, this header is included *twice* in portmux_impl.h: one time - for the 'normal' mux code which uses a compare&exchange routine, another time - to generate code for a second set of these routines that use a second mux - (in internal ram) to fake a compare&exchange on a variable in external memory. -*/ - - - -static inline bool __attribute__((always_inline)) -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -PORTMUX_AQUIRE_MUX_FN_NAME(portMUX_TYPE *mux, int timeout_cycles, const char *fnName, int line) { -#else -PORTMUX_AQUIRE_MUX_FN_NAME(portMUX_TYPE *mux, int timeout_cycles) { -#endif - - -#if !CONFIG_FREERTOS_UNICORE - uint32_t res; - portBASE_TYPE coreID, otherCoreID; - uint32_t ccount_start; - bool set_timeout = timeout_cycles > portMUX_NO_TIMEOUT; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (!set_timeout) { - timeout_cycles = 10000; // Always set a timeout in debug mode - set_timeout = true; - } -#endif - if (set_timeout) { // Timeout - RSR(CCOUNT, ccount_start); - } - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - uint32_t owner = mux->owner; - if (owner != portMUX_FREE_VAL && owner != CORE_ID_REGVAL_PRO && owner != CORE_ID_REGVAL_APP) { - ets_printf("ERROR: vPortCPUAcquireMutex: mux %p is uninitialized (0x%X)! Called from %s line %d.\n", mux, owner, fnName, line); - mux->owner=portMUX_FREE_VAL; - } -#endif - - /* Spin until we own the core */ - - RSR(PRID, coreID); - /* Note: coreID is the full 32 bit core ID (CORE_ID_REGVAL_PRO/CORE_ID_REGVAL_APP), - not the 0/1 value returned by xPortGetCoreID() - */ - otherCoreID = CORE_ID_REGVAL_XOR_SWAP ^ coreID; - do { - /* mux->owner should be one of portMUX_FREE_VAL, CORE_ID_REGVAL_PRO, - CORE_ID_REGVAL_APP: - - - If portMUX_FREE_VAL, we want to atomically set to 'coreID'. - - If "our" coreID, we can drop through immediately. - - If "otherCoreID", we spin here. - */ - res = coreID; - PORTMUX_COMPARE_SET_FN_NAME(&mux->owner, portMUX_FREE_VAL, &res); - - if (res != otherCoreID) { - break; // mux->owner is "our" coreID - } - - if (set_timeout) { - uint32_t ccount_now; - RSR(CCOUNT, ccount_now); - if (ccount_now - ccount_start > (unsigned)timeout_cycles) { -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - ets_printf("Timeout on mux! last non-recursive lock %s line %d, curr %s line %d\n", mux->lastLockedFn, mux->lastLockedLine, fnName, line); - ets_printf("Owner 0x%x count %d\n", mux->owner, mux->count); -#endif - return false; - } - } - } while (1); - - assert(res == coreID || res == portMUX_FREE_VAL); /* any other value implies memory corruption or uninitialized mux */ - assert((res == portMUX_FREE_VAL) == (mux->count == 0)); /* we're first to lock iff count is zero */ - assert(mux->count < 0xFF); /* Bad count value implies memory corruption */ - - /* now we own it, we can increment the refcount */ - mux->count++; - - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (res==portMUX_FREE_VAL) { //initial lock - mux->lastLockedFn=fnName; - mux->lastLockedLine=line; - } else { - ets_printf("Recursive lock: count=%d last non-recursive lock %s line %d, curr %s line %d\n", mux->count-1, - mux->lastLockedFn, mux->lastLockedLine, fnName, line); - } -#endif /* CONFIG_FREERTOS_PORTMUX_DEBUG */ -#endif /* CONFIG_FREERTOS_UNICORE */ - return true; -} - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG -static inline void PORTMUX_RELEASE_MUX_FN_NAME(portMUX_TYPE *mux, const char *fnName, int line) { -#else -static inline void PORTMUX_RELEASE_MUX_FN_NAME(portMUX_TYPE *mux) { -#endif - - -#if !CONFIG_FREERTOS_UNICORE - portBASE_TYPE coreID; -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - const char *lastLockedFn=mux->lastLockedFn; - int lastLockedLine=mux->lastLockedLine; - mux->lastLockedFn=fnName; - mux->lastLockedLine=line; - uint32_t owner = mux->owner; - if (owner != portMUX_FREE_VAL && owner != CORE_ID_REGVAL_PRO && owner != CORE_ID_REGVAL_APP) { - ets_printf("ERROR: vPortCPUReleaseMutex: mux %p is invalid (0x%x)!\n", mux, mux->owner); - } -#endif - -#if CONFIG_FREERTOS_PORTMUX_DEBUG || !defined(NDEBUG) - RSR(PRID, coreID); -#endif - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - if (coreID != mux->owner) { - ets_printf("ERROR: vPortCPUReleaseMutex: mux %p was already unlocked!\n", mux); - ets_printf("Last non-recursive unlock %s line %d, curr unlock %s line %d\n", lastLockedFn, lastLockedLine, fnName, line); - } -#endif - - assert(coreID == mux->owner); // This is a mutex we didn't lock, or it's corrupt - - mux->count--; - if(mux->count == 0) { - mux->owner = portMUX_FREE_VAL; - } else { - assert(mux->count < 0x100); // Indicates memory corruption -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG_RECURSIVE - ets_printf("Recursive unlock: count=%d last locked %s line %d, curr %s line %d\n", mux->count, lastLockedFn, lastLockedLine, fnName, line); -#endif - } -#endif //!CONFIG_FREERTOS_UNICORE -} diff --git a/components/freertos/tasks.c b/components/freertos/tasks.c index f34227d3b..c057cb4c0 100644 --- a/components/freertos/tasks.c +++ b/components/freertos/tasks.c @@ -2776,11 +2776,7 @@ void vTaskSwitchContext( void ) swapping that out here. Instead, we're going to do the work here ourselves. Because interrupts are already disabled, we only need to acquire the mutex. */ -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutex( &xTaskQueueMutex, __FUNCTION__, __LINE__ ); -#else vPortCPUAcquireMutex( &xTaskQueueMutex ); -#endif #if !configUSE_PORT_OPTIMISED_TASK_SELECTION unsigned portBASE_TYPE foundNonExecutingWaiter = pdFALSE, ableToSchedule = pdFALSE, resetListHead; @@ -2881,11 +2877,7 @@ void vTaskSwitchContext( void ) //Exit critical region manually as well: release the mux now, interrupts will be re-enabled when we //exit the function. -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutex( &xTaskQueueMutex, __FUNCTION__, __LINE__ ); -#else vPortCPUReleaseMutex( &xTaskQueueMutex ); -#endif #if CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK @@ -4189,13 +4181,7 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po #if ( portCRITICAL_NESTING_IN_TCB == 1 ) -#include "portmux_impl.h" - -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - void vTaskEnterCritical( portMUX_TYPE *mux, const char *function, int line ) -#else void vTaskEnterCritical( portMUX_TYPE *mux ) -#endif { BaseType_t oldInterruptLevel=0; BaseType_t schedulerRunning = xSchedulerRunning; @@ -4206,13 +4192,9 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po //and save for real there. oldInterruptLevel=portENTER_CRITICAL_NESTED(); } -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUAcquireMutexIntsDisabled( mux, portMUX_NO_TIMEOUT, function, line ); -#else - vPortCPUAcquireMutexIntsDisabled( mux, portMUX_NO_TIMEOUT ); -#endif - if(schedulerRunning != pdFALSE) + vPortCPUAcquireMutex( mux ); + if( schedulerRunning != pdFALSE ) { TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()]; BaseType_t newNesting = tcb->uxCriticalNesting + 1; @@ -4253,25 +4235,16 @@ For ESP32 FreeRTOS, vTaskEnterCritical implements both portENTER_CRITICAL and po #endif /* portCRITICAL_NESTING_IN_TCB */ /*-----------------------------------------------------------*/ - - /* For ESP32 FreeRTOS, vTaskExitCritical implements both portEXIT_CRITICAL and portEXIT_CRITICAL_ISR. */ #if ( portCRITICAL_NESTING_IN_TCB == 1 ) -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - void vTaskExitCritical( portMUX_TYPE *mux, const char *function, int line ) -#else void vTaskExitCritical( portMUX_TYPE *mux ) -#endif { -#ifdef CONFIG_FREERTOS_PORTMUX_DEBUG - vPortCPUReleaseMutexIntsDisabled( mux, function, line ); -#else - vPortCPUReleaseMutexIntsDisabled( mux ); -#endif - if(xSchedulerRunning != pdFALSE) + + vPortCPUReleaseMutex( mux ); + if( xSchedulerRunning != pdFALSE ) { TCB_t *tcb = pxCurrentTCB[xPortGetCoreID()]; BaseType_t nesting = tcb->uxCriticalNesting; diff --git a/components/soc/CMakeLists.txt b/components/soc/CMakeLists.txt index 4386c7277..07e304232 100644 --- a/components/soc/CMakeLists.txt +++ b/components/soc/CMakeLists.txt @@ -35,6 +35,7 @@ list(APPEND srcs "src/hal/uart_hal_iram.c" "src/hal/spi_flash_hal.c" "src/hal/spi_flash_hal_iram.c" + "src/compare_set.c" ) if(IDF_TARGET STREQUAL "esp32") diff --git a/components/soc/include/soc/compare_set.h b/components/soc/include/soc/compare_set.h new file mode 100644 index 000000000..626752ad5 --- /dev/null +++ b/components/soc/include/soc/compare_set.h @@ -0,0 +1,56 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __COMPARE_SET_H +#define __COMPARE_SET_H + +#include +#include +#include "soc/cpu.h" +#include "soc/soc_memory_layout.h" +#include "xtensa/xtruntime.h" + + +static inline void __attribute__((always_inline)) compare_and_set_native(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ +#if (XCHAL_HAVE_S32C1I > 0) + __asm__ __volatile__ ( + "WSR %2,SCOMPARE1 \n" + "S32C1I %0, %1, 0 \n" + :"=r"(*set) + :"r"(addr), "r"(compare), "0"(*set) + ); +#else + // No S32C1I, so do this by disabling and re-enabling interrupts (slower) + uint32_t intlevel, old_value; + __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" + : "=r"(intlevel)); + + old_value = *addr; + if (old_value == compare) { + *addr = *set; + } + + __asm__ __volatile__ ("memw \n" + "wsr %0, ps\n" + :: "r"(intlevel)); + + *set = old_value; +#endif +} + + +void compare_and_set_extram(volatile uint32_t *addr, uint32_t compare, uint32_t *set); + +#endif \ No newline at end of file diff --git a/components/soc/include/soc/spinlock.h b/components/soc/include/soc/spinlock.h new file mode 100644 index 000000000..a19b41e25 --- /dev/null +++ b/components/soc/include/soc/spinlock.h @@ -0,0 +1,153 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef __SOC_SPINLOCK_H +#define __SOC_SPINLOCK_H + +#include +#include +#include "soc/cpu.h" +#include "soc/soc_memory_layout.h" +#include "soc/compare_set.h" +#include "xtensa/xtruntime.h" + + +#define SPINLOCK_FREE 0xB33FFFFF +#define SPINLOCK_WAIT_FOREVER (-1) +#define SPINLOCK_NO_WAIT 0 +#define SPINLOCK_INITIALIZER {.owner = SPINLOCK_FREE,.count = 0} +#define CORE_ID_REGVAL_XOR_SWAP (0xCDCD ^ 0xABAB) + +typedef struct { + uint32_t owner; + uint32_t count; +}spinlock_t; + +/** + * @brief Initialize a lock to its default state - unlocked + * @param lock - spinlock object to initialize + */ +static inline void __attribute__((always_inline)) spinlock_initialize(spinlock_t *lock) +{ + assert(lock); + +#if !CONFIG_FREERTOS_UNICORE + lock->owner = SPINLOCK_FREE; + lock->count = 0; +#endif +} + +/** + * @brief Top level spinlock acquire function, spins until get the lock + * @param lock - target spinlock object + * @param timeout - cycles to wait, passing SPINLOCK_WAIT_FOREVER blocs indefinitely + */ +static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *lock, int32_t timeout) +{ +#if !CONFIG_FREERTOS_UNICORE + uint32_t result; + uint32_t irq_status; + uint32_t ccount_start; + uint32_t core_id, other_core_id; + + assert(lock); + irq_status = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + + if(timeout != SPINLOCK_WAIT_FOREVER){ + RSR(CCOUNT, ccount_start); + } + + /*spin until we own a core */ + RSR(PRID, core_id); + + /* Note: coreID is the full 32 bit core ID (CORE_ID_REGVAL_PRO/CORE_ID_REGVAL_APP) */ + + other_core_id = CORE_ID_REGVAL_XOR_SWAP ^ core_id; + do { + + /* lock->owner should be one of SPINLOCK_FREE, CORE_ID_REGVAL_PRO, + * CORE_ID_REGVAL_APP: + * - If SPINLOCK_FREE, we want to atomically set to 'core_id'. + * - If "our" core_id, we can drop through immediately. + * - If "other_core_id", we spin here. + */ + result = core_id; + +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + if (esp_ptr_external_ram(lock)) { + compare_and_set_extram(&lock->owner, SPINLOCK_FREE, &result); + } else { +#endif + compare_and_set_native(&lock->owner, SPINLOCK_FREE, &result); +#if defined(CONFIG_ESP32_SPIRAM_SUPPORT) + } +#endif + if(result != other_core_id) { + break; + } + + if (timeout != SPINLOCK_WAIT_FOREVER) { + uint32_t ccount_now; + RSR(CCOUNT, ccount_now); + if (ccount_now - ccount_start > (unsigned)timeout) { + XTOS_RESTORE_INTLEVEL(irq_status); + return false; + } + } + }while(1); + + /* any other value implies memory corruption or uninitialized mux */ + assert(result == core_id || result == SPINLOCK_FREE); + assert((result == SPINLOCK_FREE) == (lock->count == 0)); /* we're first to lock iff count is zero */ + assert(lock->count < 0xFF); /* Bad count value implies memory corruption */ + + lock->count++; + XTOS_RESTORE_INTLEVEL(irq_status); + return true; + +#else + return true; +#endif + +} + +/** + * @brief Top level spinlock unlock function, unlocks a previously locked spinlock + * @param lock - target, locked before, spinlock object + */ +static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *lock) +{ + +#if !CONFIG_FREERTOS_UNICORE + uint32_t irq_status; + uint32_t core_id; + + assert(lock); + irq_status = XTOS_SET_INTLEVEL(XCHAL_EXCM_LEVEL); + + RSR(PRID, core_id); + assert(core_id == lock->owner); // This is a mutex we didn't lock, or it's corrupt + lock->count--; + + if(!lock->count) { + lock->owner = SPINLOCK_FREE; + } else { + assert(lock->count < 0x100); // Indicates memory corruption + } + + XTOS_RESTORE_INTLEVEL(irq_status); +#endif +} + +#endif + diff --git a/components/soc/src/compare_set.c b/components/soc/src/compare_set.c new file mode 100644 index 000000000..1a396c0ad --- /dev/null +++ b/components/soc/src/compare_set.c @@ -0,0 +1,40 @@ +// Copyright 2015-2019 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "soc/compare_set.h" +#include "soc/spinlock.h" + +static spinlock_t global_extram_lock = SPINLOCK_INITIALIZER; + +void compare_and_set_extram(volatile uint32_t *addr, uint32_t compare, uint32_t *set) +{ + uint32_t intlevel, old_value; + __asm__ __volatile__ ("rsil %0, " XTSTR(XCHAL_EXCM_LEVEL) "\n" + : "=r"(intlevel)); + + spinlock_acquire(&global_extram_lock, SPINLOCK_WAIT_FOREVER); + + old_value = *addr; + if (old_value == compare) { + *addr = *set; + } + + spinlock_release(&global_extram_lock); + + __asm__ __volatile__ ("memw \n" + "wsr %0, ps\n" + :: "r"(intlevel)); + + *set = old_value; +} + diff --git a/docs/en/api-guides/freertos-smp.rst b/docs/en/api-guides/freertos-smp.rst index 558f43ecd..f4ffb8167 100644 --- a/docs/en/api-guides/freertos-smp.rst +++ b/docs/en/api-guides/freertos-smp.rst @@ -356,14 +356,15 @@ a valid protection method against simultaneous access to shared data as it leaves the other core free to access the data even if the current core has disabled its own interrupts. -For this reason, ESP-IDF FreeRTOS implements critical sections using mutexes, -and calls to enter or exit a critical must provide a mutex that is associated -with a shared resource requiring access protection. When entering a critical -section in ESP-IDF FreeRTOS, the calling core will disable its scheduler and -interrupts similar to the vanilla FreeRTOS implementation. However, the calling -core will also take the mutex whilst the other core is left unaffected during -the critical section. If the other core attempts to take the same mutex, it -will spin until the mutex is released. Therefore, the ESP-IDF FreeRTOS +For this reason, ESP-IDF FreeRTOS implements critical sections using special mutexes, +referred by portMUX_Type objects on top of specific ESP32 spinlock component +and calls to enter or exit a critical must provide a spinlock object that +is associated with a shared resource requiring access protection. +When entering a critical section in ESP-IDF FreeRTOS, the calling core will disable +its scheduler and interrupts similar to the vanilla FreeRTOS implementation. However, +the calling core will also take the locks whilst the other core is left unaffected during +the critical section. If the other core attempts to take the spinlock, it +will spin until the lock is released. Therefore, the ESP-IDF FreeRTOS implementation of critical sections allows a core to have protected access to a shared resource without disabling the other core. The other core will only be affected if it tries to concurrently access the same resource. @@ -389,7 +390,7 @@ and :component_file:`freertos/task.c` It should be noted that when modifying vanilla FreeRTOS code to be ESP-IDF FreeRTOS compatible, it is trivial to modify the type of critical section called as they are all defined to call the same function. As long as the same -mutex is provided upon entering and exiting, the type of call should not +spinlock is provided upon entering and exiting, the type of call should not matter.