diff --git a/components/bootloader_support/src/bootloader_clock.c b/components/bootloader_support/src/bootloader_clock.c
index 09186402b..688878053 100644
--- a/components/bootloader_support/src/bootloader_clock.c
+++ b/components/bootloader_support/src/bootloader_clock.c
@@ -21,9 +21,11 @@
 #ifdef CONFIG_IDF_TARGET_ESP32
 #include "esp32/rom/uart.h"
 #include "esp32/rom/rtc.h"
+#define CPU_RESET_REASON SW_CPU_RESET
 #elif CONFIG_IDF_TARGET_ESP32S2
 #include "esp32s2/rom/uart.h"
 #include "esp32s2/rom/rtc.h"
+#define CPU_RESET_REASON RTC_SW_CPU_RESET
 #endif
 
 void bootloader_clock_configure(void)
@@ -50,21 +52,22 @@ void bootloader_clock_configure(void)
         cpu_freq_mhz = 240;
     }
 #endif
-
-    rtc_clk_config_t clk_cfg = RTC_CLK_CONFIG_DEFAULT();
-#if CONFIG_IDF_TARGET_ESP32
-    clk_cfg.xtal_freq = CONFIG_ESP32_XTAL_FREQ;
-#endif
-    /* ESP32-S2 doesn't have XTAL_FREQ choice, always 40MHz */
-    clk_cfg.cpu_freq_mhz = cpu_freq_mhz;
-    clk_cfg.slow_freq = rtc_clk_slow_freq_get();
-    clk_cfg.fast_freq = rtc_clk_fast_freq_get();
-    rtc_clk_init(clk_cfg);
-    /* As a slight optimization, if 32k XTAL was enabled in sdkconfig, we enable
-     * it here. Usually it needs some time to start up, so we amortize at least
-     * part of the start up time by enabling 32k XTAL early.
-     * App startup code will wait until the oscillator has started up.
-     */
+    if (rtc_clk_apb_freq_get() < APB_CLK_FREQ || rtc_get_reset_reason(0) != CPU_RESET_REASON) {
+        rtc_clk_config_t clk_cfg = RTC_CLK_CONFIG_DEFAULT();
+    #if CONFIG_IDF_TARGET_ESP32
+        clk_cfg.xtal_freq = CONFIG_ESP32_XTAL_FREQ;
+    #endif
+        /* ESP32-S2 doesn't have XTAL_FREQ choice, always 40MHz */
+        clk_cfg.cpu_freq_mhz = cpu_freq_mhz;
+        clk_cfg.slow_freq = rtc_clk_slow_freq_get();
+        clk_cfg.fast_freq = rtc_clk_fast_freq_get();
+        rtc_clk_init(clk_cfg);
+        /* As a slight optimization, if 32k XTAL was enabled in sdkconfig, we enable
+        * it here. Usually it needs some time to start up, so we amortize at least
+        * part of the start up time by enabling 32k XTAL early.
+        * App startup code will wait until the oscillator has started up.
+        */
+    }
 
     /* TODO: move the clock option into esp_system, so that this doesn't have
      * to continue:
diff --git a/components/soc/src/esp32s2/rtc_clk.c b/components/soc/src/esp32s2/rtc_clk.c
index 75969a804..096f2a8af 100644
--- a/components/soc/src/esp32s2/rtc_clk.c
+++ b/components/soc/src/esp32s2/rtc_clk.c
@@ -41,7 +41,8 @@ static const char *TAG = "rtc_clk";
 #define RTC_PLL_FREQ_480M   480
 
 // Current PLL frequency, in MHZ (320 or 480). Zero if PLL is not enabled.
-static int s_cur_pll_freq;
+// On the ESP32-S2, 480MHz PLL is enabled at reset.
+static int s_cur_pll_freq = RTC_PLL_FREQ_480M;
 
 static void rtc_clk_cpu_freq_to_8m(void);
 
@@ -374,7 +375,7 @@ void rtc_clk_cpu_freq_set_config(const rtc_cpu_freq_config_t* config)
     if (soc_clk_sel != DPORT_SOC_CLK_SEL_XTAL) {
         rtc_clk_cpu_freq_to_xtal(xtal_freq, 1);
     }
-    if (soc_clk_sel == DPORT_SOC_CLK_SEL_PLL) {
+    if (soc_clk_sel == DPORT_SOC_CLK_SEL_PLL && config->source_freq_mhz != s_cur_pll_freq) {
         rtc_clk_bbpll_disable();
     }
     if (config->source == RTC_CPU_FREQ_SRC_XTAL) {
@@ -463,7 +464,7 @@ void rtc_clk_cpu_freq_set_xtal(void)
     int freq_mhz = (int) rtc_clk_xtal_freq_get();
 
     rtc_clk_cpu_freq_to_xtal(freq_mhz, 1);
-    rtc_clk_bbpll_disable();
+    /* BBPLL is kept enabled */
 }
 
 /**