Merge branch 'temp/release_v3_2_fastforward' into 'release/v3.2'

release v3.2 fastforward

See merge request idf/esp-idf!3678

.gitlab-ci.yml

@@ -218,6 +218,8 @@ build_esp_idf_tests:
 
 .build_examples_make_template: &build_examples_make_template
   <<: *build_template
+  # This is a workaround for a rarely encountered issue with building examples in CI.
+  # Probably related to building of Kconfig in 'make clean' stage
   retry: 1
   artifacts:
     when: always
@@ -251,7 +253,6 @@ build_esp_idf_tests:
 # same as above, but for CMake
 .build_examples_cmake_template: &build_examples_cmake_template
   <<: *build_template
-  retry: 1
   artifacts:
     when: always
     paths:
@@ -362,7 +363,7 @@ build_docs:
     - ./check_lang_folder_sync.sh
     - cd en
     - make gh-linkcheck
-    - make html || cat /tmp/sphinx-err*.log
+    - make html
     - ../check_doc_warnings.sh
     - cd ../zh_CN
     - make gh-linkcheck
@@ -939,7 +940,7 @@ example_test_002_01:
   image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - ESP32
-    - Example_ShieldBox
+    - Example_ShieldBox_Basic
 
 example_test_003_01:
   <<: *example_test_template
@@ -959,6 +960,16 @@ example_test_005_01:
     - ESP32
     - Example_WIFI_BT
 
+example_test_006_01:
+  <<: *example_test_template
+  image: $CI_DOCKER_REGISTRY/ubuntu-test-env$BOT_DOCKER_IMAGE_TAG
+  only:
+    variables:
+      - $BOT_LABEL_IPERF_STRESS_TEST
+  tags:
+    - ESP32
+    - Example_ShieldBox
+
 UT_001_01:
   <<: *unit_test_template
   tags:
@@ -1187,6 +1198,18 @@ UT_001_38:
     - ESP32_IDF
     - UT_T1_1
 
+UT_001_39:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
+UT_001_40:
+  <<: *unit_test_template
+  tags:
+    - ESP32_IDF
+    - UT_T1_1
+
 UT_002_01:
   <<: *unit_test_template
   tags:
@@ -1719,4 +1742,4 @@ IT_015_01:
   <<: *test_template
   tags:
     - ESP32_IDF
-    - SSC_T2_4
+    - SSC_T2_4

components/bootloader/subproject/main/esp32.bootloader.ld

@@ -35,7 +35,6 @@ SECTIONS
   .iram_loader.text :
   {
     . = ALIGN (16);
-    _stext = .;
     _loader_text_start = ABSOLUTE(.);
     *(.stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
      *(.iram1 .iram1.*) /* catch stray IRAM_ATTR */
@@ -59,7 +58,6 @@ SECTIONS
     *(.fini)
     *(.gnu.version)
     _loader_text_end = ABSOLUTE(.);
-    _etext = .;
   } > iram_loader_seg
 
   .iram.text :

components/bootloader_support/include/bootloader_util.h

@@ -0,0 +1,34 @@
+// Copyright 2018 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stddef.h>
+
+/**
+ * @brief Check if half-open intervals overlap
+ *
+ * @param start1  interval 1 start
+ * @param end1    interval 1 end
+ * @param start2  interval 2 start
+ * @param end2    interval 2 end
+ * @return true iff [start1; end1) overlaps [start2; end2)
+ */
+static inline bool bootloader_util_regions_overlap(
+        const intptr_t start1, const intptr_t end1,
+        const intptr_t start2, const intptr_t end2)
+{
+    return (end1 > start2 && end2 > start1) ||
+           !(end1 <= start2 || end2 <= start1);
+}

components/bootloader_support/src/esp_image_format.c

@@ -23,6 +23,7 @@
 #include <bootloader_flash.h>
 #include <bootloader_random.h>
 #include <bootloader_sha.h>
+#include "bootloader_util.h"
 
 /* Checking signatures as part of verifying images is necessary:
    - Always if secure boot is enabled
@@ -56,6 +57,10 @@ static const char *TAG = "esp_image";
    (Means loaded code isn't executable until after the secure boot check.)
 */
 static uint32_t ram_obfs_value[2];
+
+/* Range of IRAM used by the loader, defined in ld script */
+extern int _loader_text_start;
+extern int _loader_text_end;
 #endif
 
 /* Return true if load_addr is an address the bootloader should load into */
@@ -328,18 +333,41 @@ static esp_err_t process_segment(int index, uint32_t flash_addr, esp_image_segme
                  (do_load)?"load":(is_mapping)?"map":"");
     }
 
+
+#ifdef BOOTLOADER_BUILD
+    /* Before loading segment, check it doesn't clobber bootloader RAM. */
     if (do_load) {
-        /* Before loading segment, check it doesn't clobber bootloader RAM... */
-        uint32_t end_addr = load_addr + data_len;
-        if (end_addr < 0x40000000) {
+        const intptr_t load_end = load_addr + data_len;
+        if (load_end <= (intptr_t) SOC_DIRAM_DRAM_HIGH) {
+            /* Writing to DRAM */
             intptr_t sp = (intptr_t)get_sp();
-            if (end_addr > sp - STACK_LOAD_HEADROOM) {
-                ESP_LOGE(TAG, "Segment %d end address 0x%08x too high (bootloader stack 0x%08x liimit 0x%08x)",
-                         index, end_addr, sp, sp - STACK_LOAD_HEADROOM);
+            if (load_end > sp - STACK_LOAD_HEADROOM) {
+                /* Bootloader .data/.rodata/.bss is above the stack, so this
+                 * also checks that we aren't overwriting these segments.
+                 *
+                 * TODO: This assumes specific arrangement of sections we have
+                 * in the ESP32. Rewrite this in a generic way to support other
+                 * layouts.
+                 */
+                ESP_LOGE(TAG, "Segment %d end address 0x%08x too high (bootloader stack 0x%08x limit 0x%08x)",
+                         index, load_end, sp, sp - STACK_LOAD_HEADROOM);
+                return ESP_ERR_IMAGE_INVALID;
+            }
+        } else {
+            /* Writing to IRAM */
+            const intptr_t loader_iram_start = (intptr_t) &_loader_text_start;
+            const intptr_t loader_iram_end = (intptr_t) &_loader_text_end;
+
+            if (bootloader_util_regions_overlap(loader_iram_start, loader_iram_end,
+                    load_addr, load_end)) {
+                ESP_LOGE(TAG, "Segment %d (0x%08x-0x%08x) overlaps bootloader IRAM (0x%08x-0x%08x)",
+                         index, load_addr, load_end, loader_iram_start, loader_iram_end);
                 return ESP_ERR_IMAGE_INVALID;
             }
         }
     }
+#endif // BOOTLOADER_BUILD
+
 #ifndef BOOTLOADER_BUILD
     uint32_t free_page_count = spi_flash_mmap_get_free_pages(SPI_FLASH_MMAP_DATA);
     ESP_LOGD(TAG, "free data page_count 0x%08x",free_page_count);

components/bootloader_support/test/test_verify_image.c

@@ -14,6 +14,7 @@
 #include "freertos/xtensa_api.h"
 #include "unity.h"
 #include "bootloader_common.h"
+#include "bootloader_util.h"
 #include "esp_partition.h"
 #include "esp_ota_ops.h"
 #include "esp_image_format.h"
@@ -92,3 +93,23 @@ TEST_CASE("Test label_search", "[bootloader_support]")
     check_label_search(25,  "phy, 1234567890123456, nvs1",  "12345678901234567",    true);
 
 }
+
+
+TEST_CASE("Test regions_overlap", "[bootloader_support]")
+{
+    TEST_ASSERT( bootloader_util_regions_overlap(1, 2, 1, 2) );
+
+    TEST_ASSERT( bootloader_util_regions_overlap(1, 2, 0, 2) );
+    TEST_ASSERT( bootloader_util_regions_overlap(1, 2, 1, 3) );
+    TEST_ASSERT( bootloader_util_regions_overlap(1, 2, 0, 3) );
+
+    TEST_ASSERT( bootloader_util_regions_overlap(0, 2, 1, 2) );
+    TEST_ASSERT( bootloader_util_regions_overlap(1, 3, 1, 2) );
+    TEST_ASSERT( bootloader_util_regions_overlap(0, 3, 1, 2) );
+
+    TEST_ASSERT( !bootloader_util_regions_overlap(2, 3, 1, 2) );
+    TEST_ASSERT( !bootloader_util_regions_overlap(1, 2, 2, 3) );
+
+    TEST_ASSERT( !bootloader_util_regions_overlap(3, 4, 1, 2) );
+    TEST_ASSERT( !bootloader_util_regions_overlap(1, 2, 3, 4) );
+}

components/bt/bluedroid/api/esp_gatts_api.c

@@ -321,6 +321,7 @@ esp_gatt_status_t esp_ble_gatts_get_attr_value(uint16_t attr_handle, uint16_t *l
     ESP_BLUEDROID_STATUS_CHECK(ESP_BLUEDROID_STATUS_ENABLED);
 
     if (attr_handle == ESP_GATT_ILLEGAL_HANDLE) {
+        *length = 0;
         return ESP_GATT_INVALID_HANDLE;
     }
 

components/bt/bluedroid/btc/profile/std/spp/btc_spp.c

@@ -52,7 +52,7 @@ typedef struct {
     list_t *list;
     list_t *incoming_list;
     uint8_t service_uuid[16];
-    char service_name[ESP_SPP_SERVER_NAME_MAX];
+    char service_name[ESP_SPP_SERVER_NAME_MAX + 1];
 } spp_slot_t;
 
 static struct spp_local_param_t {

components/bt/bluedroid/hci/hci_hal_h4.c

@@ -25,10 +25,10 @@
 #include "hci/hci_layer.h"
 #include "osi/thread.h"
 #include "esp_bt.h"
+#include "stack/hcimsgs.h"
 
 #if (C2H_FLOW_CONTROL_INCLUDED == TRUE)
 #include "l2c_int.h"
-#include "stack/hcimsgs.h"
 #endif ///C2H_FLOW_CONTROL_INCLUDED == TRUE
 
 #define HCI_HAL_SERIAL_BUFFER_SIZE 1026
@@ -332,7 +332,7 @@ static int host_recv_pkt_cb(uint8_t *data, uint16_t len)
     pkt->layer_specific = 0;
     memcpy(pkt->data, data, len);
     fixed_queue_enqueue(hci_hal_env.rx_q, pkt);
-    hci_hal_h4_task_post(100 / portTICK_PERIOD_MS);
+    hci_hal_h4_task_post(0);
 
     BTTRC_DUMP_BUFFER("Recv Pkt", pkt->data, len);
 

components/bt/bluedroid/stack/gatt/gatt_api.c

@@ -766,7 +766,8 @@ tGATT_STATUS GATTS_GetAttributeValue(UINT16 attr_handle, UINT16 *length, UINT8 *
                     attr_handle);
 
      if ((p_decl = gatt_find_hdl_buffer_by_attr_handle(attr_handle)) == NULL) {
-         GATT_TRACE_ERROR("Service not created\n"); 
+         GATT_TRACE_ERROR("Service not created\n");
+         *length = 0;
          return GATT_INVALID_HANDLE;
      }
 

components/bt/bluedroid/stack/gatt/gatt_db.c

@@ -783,10 +783,12 @@ tGATT_STATUS gatts_get_attribute_value(tGATT_SVC_DB *p_db, UINT16 attr_handle,
 
     if (p_db == NULL) {
         GATT_TRACE_ERROR("gatts_get_attribute_value Fail:p_db is NULL.\n");
+        *length = 0;
         return GATT_INVALID_PDU;
     }
     if (p_db->p_attr_list == NULL) {
         GATT_TRACE_ERROR("gatts_get_attribute_value Fail:p_db->p_attr_list is NULL.\n");
+        *length = 0;
         return GATT_INVALID_PDU;
     }
     if (length == NULL){
@@ -795,6 +797,7 @@ tGATT_STATUS gatts_get_attribute_value(tGATT_SVC_DB *p_db, UINT16 attr_handle,
     }
     if (value == NULL){
         GATT_TRACE_ERROR("gatts_get_attribute_value Fail:value is NULL.\n");
+        *length = 0;
         return GATT_INVALID_PDU;
     }
 
@@ -814,19 +817,19 @@ tGATT_STATUS gatts_get_attribute_value(tGATT_SVC_DB *p_db, UINT16 attr_handle,
                         *value = p_cur->p_value->attr_val.attr_val;
                         return GATT_SUCCESS;
                     } else {
-                        GATT_TRACE_ERROR("gatts_get_attribute_value failed:the value length is 0");
-                        return GATT_INVALID_ATTR_LEN;
+                        *length = 0;
+                        return GATT_SUCCESS;
                     }
                     break;
                 }
             } else {
-                if (p_cur->p_value->attr_val.attr_len != 0) {
+                if (p_cur->p_value && p_cur->p_value->attr_val.attr_len != 0) {
                     *length = p_cur->p_value->attr_val.attr_len;
                     *value = p_cur->p_value->attr_val.attr_val;
                     return GATT_SUCCESS;
                 } else {
-                    GATT_TRACE_ERROR("gatts_get_attribute_value failed:the value length is 0");
-                    return GATT_INVALID_ATTR_LEN;
+                    *length = 0;
+                    return GATT_SUCCESS;
                 }
 
             }

components/bt/bluedroid/stack/include/stack/hcimsgs.h

@@ -275,7 +275,10 @@ BOOLEAN btsnd_hcic_hold_mode(UINT16 handle, UINT16 max_hold_period,
 #define HCI_HOLD_MODE_MIN_PER_OFF       4
 /* Hold Mode */
 
-/* Sniff Mode */
+/**
+ * Sniff Mode
+ * sniff_attempt should no more than 0xFF
+ */
 BOOLEAN btsnd_hcic_sniff_mode(UINT16 handle,
                               UINT16 max_sniff_period,
                               UINT16 min_sniff_period,

components/bt/lib

@@ -1 +1 @@
-Subproject commit d65d35f4e0102f0f3640267f94c0ac6ce2a5fa91
+Subproject commit 16f95952efc63017167c863a7eadd2e8dfd27a61

components/driver/periph_ctrl.c

@@ -23,7 +23,7 @@ static portMUX_TYPE periph_spinlock = portMUX_INITIALIZER_UNLOCKED;
 
 /* Static functions to return register address & mask for clk_en / rst of each peripheral */
 static uint32_t get_clk_en_mask(periph_module_t periph);
-static uint32_t get_rst_en_mask(periph_module_t periph);
+static uint32_t get_rst_en_mask(periph_module_t periph, bool enable);
 static uint32_t get_clk_en_reg(periph_module_t periph);
 static uint32_t get_rst_en_reg(periph_module_t periph);
 
@@ -31,7 +31,7 @@ void periph_module_enable(periph_module_t periph)
 {
     portENTER_CRITICAL(&periph_spinlock);
     DPORT_SET_PERI_REG_MASK(get_clk_en_reg(periph), get_clk_en_mask(periph));
-    DPORT_CLEAR_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph));
+    DPORT_CLEAR_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph, true));
     portEXIT_CRITICAL(&periph_spinlock);
 }
 
@@ -39,15 +39,15 @@ void periph_module_disable(periph_module_t periph)
 {
     portENTER_CRITICAL(&periph_spinlock);
     DPORT_CLEAR_PERI_REG_MASK(get_clk_en_reg(periph), get_clk_en_mask(periph));
-    DPORT_SET_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph));
+    DPORT_SET_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph, false));
     portEXIT_CRITICAL(&periph_spinlock);
 }
 
 void periph_module_reset(periph_module_t periph)
 {
     portENTER_CRITICAL(&periph_spinlock);
-    DPORT_SET_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph));
-    DPORT_CLEAR_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph));
+    DPORT_SET_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph, false));
+    DPORT_CLEAR_PERI_REG_MASK(get_rst_en_reg(periph), get_rst_en_mask(periph, false));
     portEXIT_CRITICAL(&periph_spinlock);
 }
 
@@ -118,12 +118,18 @@ static uint32_t get_clk_en_mask(periph_module_t periph)
             return DPORT_BT_BASEBAND_EN;
         case PERIPH_BT_LC_MODULE:
             return DPORT_BT_LC_EN;
+        case PERIPH_AES_MODULE:
+            return DPORT_PERI_EN_AES;
+        case PERIPH_SHA_MODULE:
+            return DPORT_PERI_EN_SHA;
+        case PERIPH_RSA_MODULE:
+            return DPORT_PERI_EN_RSA;
         default:
             return 0;
     }
 }
 
-static uint32_t get_rst_en_mask(periph_module_t periph)
+static uint32_t get_rst_en_mask(periph_module_t periph, bool enable)
 {
     switch(periph) {
         case PERIPH_RMT_MODULE:
@@ -178,6 +184,30 @@ static uint32_t get_rst_en_mask(periph_module_t periph)
             return DPORT_CAN_RST;
         case PERIPH_EMAC_MODULE:
             return DPORT_EMAC_RST;
+        case PERIPH_AES_MODULE:
+            if (enable == true) {
+                // Clear reset on digital signature & secure boot units, otherwise AES unit is held in reset also.
+                return (DPORT_PERI_EN_AES | DPORT_PERI_EN_DIGITAL_SIGNATURE | DPORT_PERI_EN_SECUREBOOT);
+            } else {
+                //Don't return other units to reset, as this pulls reset on RSA & SHA units, respectively.
+                return DPORT_PERI_EN_AES;
+            }
+        case PERIPH_SHA_MODULE:
+            if (enable == true) {
+                // Clear reset on secure boot, otherwise SHA is held in reset
+                return (DPORT_PERI_EN_SHA | DPORT_PERI_EN_SECUREBOOT);
+            } else {
+                // Don't assert reset on secure boot, otherwise AES is held in reset
+                return DPORT_PERI_EN_SHA;
+            }
+        case PERIPH_RSA_MODULE:
+            if (enable == true) {
+                // Also clear reset on digital signature, otherwise RSA is held in reset
+                return (DPORT_PERI_EN_RSA | DPORT_PERI_EN_DIGITAL_SIGNATURE);
+            } else {
+                // Don't reset digital signature unit, as this resets AES also
+                return DPORT_PERI_EN_RSA;
+            }
         case PERIPH_WIFI_MODULE:
         case PERIPH_BT_MODULE:
         case PERIPH_WIFI_BT_COMMON_MODULE:
@@ -211,12 +241,20 @@ static bool is_wifi_clk_peripheral(periph_module_t periph)
 
 static uint32_t get_clk_en_reg(periph_module_t periph)
 {
-    return is_wifi_clk_peripheral(periph) ? DPORT_WIFI_CLK_EN_REG : DPORT_PERIP_CLK_EN_REG;
+    if (periph == PERIPH_AES_MODULE || periph == PERIPH_SHA_MODULE || periph == PERIPH_RSA_MODULE) {
+        return DPORT_PERI_CLK_EN_REG;
+    } else {
+        return is_wifi_clk_peripheral(periph) ? DPORT_WIFI_CLK_EN_REG : DPORT_PERIP_CLK_EN_REG;
+    }
 }
 
 static uint32_t get_rst_en_reg(periph_module_t periph)
 {
-    return is_wifi_clk_peripheral(periph) ? DPORT_CORE_RST_EN_REG : DPORT_PERIP_RST_EN_REG;
+    if (periph == PERIPH_AES_MODULE || periph == PERIPH_SHA_MODULE || periph == PERIPH_RSA_MODULE) {
+        return DPORT_PERI_RST_EN_REG;
+    } else {
+        return is_wifi_clk_peripheral(periph) ? DPORT_CORE_RST_EN_REG : DPORT_PERIP_RST_EN_REG;
+    }
 }
 
 

components/esp-tls/esp_tls.h

@@ -158,7 +158,7 @@ esp_tls_t *esp_tls_conn_new(const char *hostname, int hostlen, int port, const e
  */
 esp_tls_t *esp_tls_conn_http_new(const char *url, const esp_tls_cfg_t *cfg);
    
-/*
+/**
  * @brief      Create a new non-blocking TLS/SSL connection
  *
  * This function initiates a non-blocking TLS/SSL connection with the specified host, but due to
@@ -171,9 +171,10 @@ esp_tls_t *esp_tls_conn_http_new(const char *url, const esp_tls_cfg_t *cfg);
  *                       this structure should be set to be true.
  * @param[in]  tls       pointer to esp-tls as esp-tls handle.
  *
- * @return     - 1       If connection establishment fails.
- *             - 0       If connection establishment is in progress.
- *             - 1       If connection establishment is successful.
+ * @return
+ *             - -1      If connection establishment fails.
+ *             -  0      If connection establishment is in progress.
+ *             -  1      If connection establishment is successful.
  */
 int esp_tls_conn_new_async(const char *hostname, int hostlen, int port, const esp_tls_cfg_t *cfg, esp_tls_t *tls);
 
@@ -183,12 +184,13 @@ int esp_tls_conn_new_async(const char *hostname, int hostlen, int port, const es
  * The behaviour is same as esp_tls_conn_new() API. However this API accepts host's url.
  *
  * @param[in]  url     url of host.
- * @param[in]  tls     pointer to esp-tls as esp-tls handle.
  * @param[in]  cfg     TLS configuration as esp_tls_cfg_t.
+ * @param[in]  tls     pointer to esp-tls as esp-tls handle.
  *
- * @return     - 1     If connection establishment fails.
- *             - 0     If connection establishment is in progress.
- *             - 1     If connection establishment is successful.
+ * @return
+ *             - -1     If connection establishment fails.
+ *             -  0     If connection establishment is in progress.
+ *             -  1     If connection establishment is successful.
  */
 int esp_tls_conn_http_new_async(const char *url, const esp_tls_cfg_t *cfg, esp_tls_t *tls);
 

components/esp32/hwcrypto/aes.c

@@ -36,6 +36,7 @@
 
 #include "soc/cpu.h"
 #include <stdio.h>
+#include "driver/periph_ctrl.h"
 
 
 /* AES uses a spinlock mux not a lock as the underlying block operation
@@ -50,26 +51,16 @@ static portMUX_TYPE aes_spinlock = portMUX_INITIALIZER_UNLOCKED;
 
 void esp_aes_acquire_hardware( void )
 {
-    /* newlib locks lazy initialize on ESP-IDF */
     portENTER_CRITICAL(&aes_spinlock);
 
     /* Enable AES hardware */
-    DPORT_REG_SET_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_AES);
-    /* Clear reset on digital signature & secure boot units,
-       otherwise AES unit is held in reset also. */
-    DPORT_REG_CLR_BIT(DPORT_PERI_RST_EN_REG,
-                       DPORT_PERI_EN_AES
-                       | DPORT_PERI_EN_DIGITAL_SIGNATURE
-                       | DPORT_PERI_EN_SECUREBOOT);
+    periph_module_enable(PERIPH_AES_MODULE);
 }
 
 void esp_aes_release_hardware( void )
 {
     /* Disable AES hardware */
-    DPORT_REG_SET_BIT(DPORT_PERI_RST_EN_REG, DPORT_PERI_EN_AES);
-    /* Don't return other units to reset, as this pulls
-       reset on RSA & SHA units, respectively. */
-    DPORT_REG_CLR_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_AES);
+    periph_module_disable(PERIPH_AES_MODULE);
 
     portEXIT_CRITICAL(&aes_spinlock);
 }
@@ -116,7 +107,10 @@ static inline void esp_aes_setkey_hardware( esp_aes_context *ctx, int mode)
     const uint32_t MODE_DECRYPT_BIT = 4;
     unsigned mode_reg_base = (mode == ESP_AES_ENCRYPT) ? 0 : MODE_DECRYPT_BIT;
 
-    memcpy((uint32_t *)AES_KEY_BASE, ctx->key, ctx->key_bytes);
+    for (int i = 0; i < ctx->key_bytes/4; ++i) {
+        DPORT_REG_WRITE(AES_KEY_BASE + i * 4, *(((uint32_t *)ctx->key) + i));
+    }
+
     DPORT_REG_WRITE(AES_MODE_REG, mode_reg_base + ((ctx->key_bytes / 8) - 2));
 }
 
@@ -431,7 +425,7 @@ static int esp_aes_xts_decode_keys( const unsigned char *key,
     return 0;
 }
 
-int esp_aes_xts_setkey_enc( mbedtls_aes_xts_context *ctx,
+int esp_aes_xts_setkey_enc( esp_aes_xts_context *ctx,
                                 const unsigned char *key,
                                 unsigned int keybits)
 {
@@ -453,7 +447,7 @@ int esp_aes_xts_setkey_enc( mbedtls_aes_xts_context *ctx,
     return esp_aes_setkey( &ctx->crypt, key1, key1bits );
 }
 
-int esp_aes_xts_setkey_dec( mbedtls_aes_xts_context *ctx,
+int esp_aes_xts_setkey_dec( esp_aes_xts_context *ctx,
                                 const unsigned char *key,
                                 unsigned int keybits)
 {
@@ -532,7 +526,7 @@ static void esp_gf128mul_x_ble( unsigned char r[16],
 /*
  * AES-XTS buffer encryption/decryption
  */
-int esp_aes_crypt_xts( mbedtls_aes_xts_context *ctx,
+int esp_aes_crypt_xts( esp_aes_xts_context *ctx,
                            int mode,
                            size_t length,
                            const unsigned char data_unit[16],

components/esp32/hwcrypto/sha.c

@@ -35,6 +35,7 @@
 #include "rom/ets_sys.h"
 #include "soc/dport_reg.h"
 #include "soc/hwcrypto_reg.h"
+#include "driver/periph_ctrl.h"
 
 inline static uint32_t SHA_LOAD_REG(esp_sha_type sha_type) {
     return SHA_1_LOAD_REG + sha_type * 0x10;
@@ -160,11 +161,7 @@ static void esp_sha_lock_engine_inner(sha_engine_state *engine)
 
     if (sha_engines_all_idle()) {
         /* Enable SHA hardware */
-        DPORT_REG_SET_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_SHA);
-        /* also clear reset on secure boot, otherwise SHA is held in reset */
-        DPORT_REG_CLR_BIT(DPORT_PERI_RST_EN_REG,
-                           DPORT_PERI_EN_SHA
-                           | DPORT_PERI_EN_SECUREBOOT);
+        periph_module_enable(PERIPH_SHA_MODULE);
         DPORT_STALL_OTHER_CPU_START();
         ets_sha_enable();
         DPORT_STALL_OTHER_CPU_END();
@@ -188,9 +185,7 @@ void esp_sha_unlock_engine(esp_sha_type sha_type)
 
     if (sha_engines_all_idle()) {
         /* Disable SHA hardware */
-        /* Don't assert reset on secure boot, otherwise AES is held in reset */
-        DPORT_REG_SET_BIT(DPORT_PERI_RST_EN_REG, DPORT_PERI_EN_SHA);
-        DPORT_REG_CLR_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_SHA);
+        periph_module_disable(PERIPH_SHA_MODULE);
     }
 
     _lock_release(&state_change_lock);
@@ -276,9 +271,9 @@ void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, uns
 
     SHA_CTX ctx;
     ets_sha_init(&ctx);
+    esp_sha_lock_memory_block();
     while(ilen > 0) {
         size_t chunk_len = (ilen > block_len) ? block_len : ilen;
-        esp_sha_lock_memory_block();
         esp_sha_wait_idle();
         DPORT_STALL_OTHER_CPU_START();
         {
@@ -286,11 +281,9 @@ void esp_sha(esp_sha_type sha_type, const unsigned char *input, size_t ilen, uns
             ets_sha_update(&ctx, sha_type, input, chunk_len * 8);
         }
         DPORT_STALL_OTHER_CPU_END();
-        esp_sha_unlock_memory_block();
         input += chunk_len;
         ilen -= chunk_len;
     }
-    esp_sha_lock_memory_block();
     esp_sha_wait_idle();
     DPORT_STALL_OTHER_CPU_START();
     {

components/esp32/ld/esp32.rom.ld

@@ -670,9 +670,11 @@ PROVIDE ( ld_acl_tx = 0x4002ffdc );
 PROVIDE ( ld_acl_rx_sync = 0x4002fbec );
 PROVIDE ( ld_acl_rx_sync2 = 0x4002fd8c );
 PROVIDE ( ld_acl_rx_no_sync = 0x4002fe78 );
+PROVIDE ( ld_acl_clk_isr = 0x40030cf8 );
 PROVIDE ( ld_sco_modify = 0x40031778 );
 PROVIDE ( lm_cmd_cmp_send = 0x40051838 );
 PROVIDE ( ld_sco_frm_cbk = 0x400349dc );
+PROVIDE ( ld_acl_sniff_frm_cbk = 0x4003482c );
 PROVIDE ( r_ld_acl_active_hop_types_get = 0x40036e10 );
 PROVIDE ( r_ld_acl_afh_confirm = 0x40036d40 );
 PROVIDE ( r_ld_acl_afh_prepare = 0x40036c84 );
@@ -1374,8 +1376,8 @@ PROVIDE ( esp_rom_spiflash_attach = 0x40062a6c );
 PROVIDE ( esp_rom_spiflash_config_clk = 0x40062bc8 );
 PROVIDE ( g_rom_spiflash_chip = 0x3ffae270 );
 
-/* 
-These functions are xtos-related (or call xtos-related functions) and do not play well 
+/*
+These functions are xtos-related (or call xtos-related functions) and do not play well
 with multicore FreeRTOS. Where needed, we provide alternatives that are multicore
 compatible. These functions also use a chunk of static RAM, by not using them we can
 allocate that RAM for general use.

components/esp32/test/test_aes_sha_rsa.c

@@ -0,0 +1,287 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "esp_types.h"
+#include "esp_clk.h"
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "freertos/semphr.h"
+#include "freertos/xtensa_timer.h"
+#include "soc/cpu.h"
+#include "unity.h"
+#include "rom/uart.h"
+#include "rom/sha.h"
+#include "soc/uart_reg.h"
+#include "soc/dport_reg.h"
+#include "soc/rtc.h"
+#include "esp_log.h"
+#include "mbedtls/sha256.h"
+#include "hwcrypto/sha.h"
+#include "hwcrypto/aes.h"
+#include "mbedtls/rsa.h"
+
+static const char *TAG = "test";
+static volatile bool exit_flag = false;
+#define TASK_STACK_SIZE (8*1024)
+
+static void aes_task(void *pvParameters)
+{
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
+    ESP_LOGI(TAG, "aes_task is started");
+    esp_aes_context ctx = {
+            .key_bytes = 16,
+            .key = {101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116}
+    };
+    const unsigned char input[16] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+    unsigned char output[16];
+    unsigned char output2[16];
+    while (exit_flag == false) {
+        memset(output, 0, sizeof(output));
+        memset(output, 0, sizeof(output2));
+        esp_internal_aes_encrypt(&ctx, input, output);
+        esp_internal_aes_decrypt(&ctx, output, output2);
+        TEST_ASSERT_EQUAL_MEMORY_MESSAGE(input, output2, sizeof(input), "AES must match");
+    }
+    xSemaphoreGive(*sema);
+    vTaskDelete(NULL);
+}
+
+static void sha_task(void *pvParameters)
+{
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
+    ESP_LOGI(TAG, "sha_task is started");
+    const char *input = "Space!#$%&()*+,-.0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_abcdefghijklmnopqrstuvwxyz~DEL0123456789";
+    unsigned char output[64];
+    unsigned char output_origin[64];
+    esp_sha(SHA2_512, (const unsigned char *)input, sizeof(input), output);
+    memcpy(output_origin, output, sizeof(output));
+    while (exit_flag == false) {
+        memset(output, 0, sizeof(output));
+        esp_sha(SHA2_512, (const unsigned char *)input, sizeof(input), output);
+        TEST_ASSERT_EQUAL_MEMORY_MESSAGE(output, output_origin, sizeof(output), "SHA256 must match");
+    }
+    xSemaphoreGive(*sema);
+    vTaskDelete(NULL);
+}
+
+static void mbedtls_sha256_task(void *pvParameters)
+{
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
+    ESP_LOGI(TAG, "mbedtls_sha256_task is started");
+    const char *input = "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_abcdefghijklmnopqrstuvwxyz~DEL0123456789Space!#$%&()*+,-.0123456789:;<=>?";
+    mbedtls_sha256_context sha256_ctx;
+    unsigned char output[32];
+    unsigned char output_origin[32];
+
+    mbedtls_sha256_init(&sha256_ctx);
+    memset(output, 0, sizeof(output));
+    mbedtls_sha256_starts_ret(&sha256_ctx, false);
+    for (int i = 0; i < 3; ++i) {
+        mbedtls_sha256_update_ret(&sha256_ctx, (unsigned char *)input, 100);
+    }
+    mbedtls_sha256_finish_ret(&sha256_ctx, output);
+    memcpy(output_origin, output, sizeof(output));
+
+    while (exit_flag == false) {
+        mbedtls_sha256_init(&sha256_ctx);
+        memset(output, 0, sizeof(output));
+        mbedtls_sha256_starts_ret(&sha256_ctx, false);
+        for (int i = 0; i < 3; ++i) {
+            mbedtls_sha256_update_ret(&sha256_ctx, (unsigned char *)input, 100);
+        }
+        mbedtls_sha256_finish_ret(&sha256_ctx, output);
+
+        TEST_ASSERT_EQUAL_MEMORY_MESSAGE(output, output_origin, sizeof(output), "MBEDTLS SHA256 must match");
+    }
+    xSemaphoreGive(*sema);
+    vTaskDelete(NULL);
+}
+
+TEST_CASE("Test shared using AES SHA512 SHA256", "[hw_crypto]")
+{
+#ifndef CONFIG_FREERTOS_UNICORE
+    const int max_tasks = 6;
+#else
+    const int max_tasks = 3;
+#endif
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+    exit_flag = false;
+#ifndef CONFIG_FREERTOS_UNICORE
+    xTaskCreatePinnedToCore(&aes_task,            "aes_task",            TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+    xTaskCreatePinnedToCore(&aes_task,            "aes_task",            TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&sha_task,            "sha_task",            TASK_STACK_SIZE, &exit_sema[2], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+    xTaskCreatePinnedToCore(&sha_task,            "sha_task",            TASK_STACK_SIZE, &exit_sema[3], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&mbedtls_sha256_task, "mbedtls_sha256_task", TASK_STACK_SIZE, &exit_sema[4], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+    xTaskCreatePinnedToCore(&mbedtls_sha256_task, "mbedtls_sha256_task", TASK_STACK_SIZE, &exit_sema[5], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+#else
+    xTaskCreate(&aes_task,            "aes_task",            TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&sha_task,            "sha_task",            TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&mbedtls_sha256_task, "mbedtls_sha256_task", TASK_STACK_SIZE, &exit_sema[2], UNITY_FREERTOS_PRIORITY - 1, NULL);
+#endif
+
+    ESP_LOGI(TAG, "Waiting for 10s ...");
+    vTaskDelay(10000 / portTICK_PERIOD_MS);
+
+    // set exit flag to let thread exit
+    exit_flag = true;
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
+}
+
+static void rsa_task(void *pvParameters)
+{
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
+    ESP_LOGI(TAG, "rsa_task is started");
+    while (exit_flag == false) {
+        mbedtls_rsa_self_test(0);
+    }
+    xSemaphoreGive(*sema);
+    vTaskDelete(NULL);
+}
+
+TEST_CASE("Test shared using AES RSA", "[hw_crypto]")
+{
+#ifndef CONFIG_FREERTOS_UNICORE
+    const int max_tasks = 2;
+#else
+    const int max_tasks = 2;
+#endif
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+    exit_flag = false;
+#ifndef CONFIG_FREERTOS_UNICORE
+    xTaskCreatePinnedToCore(&aes_task, "aes_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+    xTaskCreatePinnedToCore(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+#else
+    xTaskCreate(&aes_task, "aes_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL);
+#endif
+
+    ESP_LOGI(TAG, "Waiting for 10s ...");
+    vTaskDelay(10000 / portTICK_PERIOD_MS);
+
+    // set exit flag to let thread exit
+    exit_flag = true;
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
+}
+
+TEST_CASE("Test shared using SHA512 RSA", "[hw_crypto]")
+{
+#ifndef CONFIG_FREERTOS_UNICORE
+    const int max_tasks = 2;
+#else
+    const int max_tasks = 2;
+#endif
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+    exit_flag = false;
+#ifndef CONFIG_FREERTOS_UNICORE
+    xTaskCreatePinnedToCore(&sha_task, "sha_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 2, NULL, 1);
+    xTaskCreatePinnedToCore(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+#else
+    xTaskCreate(&sha_task, "sha_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL);
+#endif
+
+    ESP_LOGI(TAG, "Waiting for 10s ...");
+    vTaskDelay(10000 / portTICK_PERIOD_MS);
+
+    // set exit flag to let thread exit
+    exit_flag = true;
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
+}
+
+TEST_CASE("Test shared using SHA256 RSA", "[hw_crypto]")
+{
+#ifndef CONFIG_FREERTOS_UNICORE
+    const int max_tasks = 2;
+#else
+    const int max_tasks = 2;
+#endif
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+    exit_flag = false;
+#ifndef CONFIG_FREERTOS_UNICORE
+    xTaskCreatePinnedToCore(&mbedtls_sha256_task, "mbedtls_sha256_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+    xTaskCreatePinnedToCore(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+#else
+    xTaskCreate(&mbedtls_sha256_task, "mbedtls_sha256_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&rsa_task,            "rsa_task",            TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL);
+#endif
+
+    ESP_LOGI(TAG, "Waiting for 10s ...");
+    vTaskDelay(10000 / portTICK_PERIOD_MS);
+
+    // set exit flag to let thread exit
+    exit_flag = true;
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
+}
+
+TEST_CASE("Test shared using AES SHA RSA", "[hw_crypto]")
+{
+#ifndef CONFIG_FREERTOS_UNICORE
+    const int max_tasks = 3;
+#else
+    const int max_tasks = 3;
+#endif
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+    exit_flag = false;
+#ifndef CONFIG_FREERTOS_UNICORE
+    xTaskCreatePinnedToCore(&aes_task, "aes_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&sha_task, "sha_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL, 0);
+    xTaskCreatePinnedToCore(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[2], UNITY_FREERTOS_PRIORITY - 1, NULL, 1);
+#else
+    xTaskCreate(&aes_task, "aes_task", TASK_STACK_SIZE, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&sha_task, "sha_task", TASK_STACK_SIZE, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, NULL);
+    xTaskCreate(&rsa_task, "rsa_task", TASK_STACK_SIZE, &exit_sema[2], UNITY_FREERTOS_PRIORITY - 1, NULL);
+#endif
+
+    ESP_LOGI(TAG, "Waiting for 10s ...");
+    vTaskDelay(10000 / portTICK_PERIOD_MS);
+
+    // set exit flag to let thread exit
+    exit_flag = true;
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
+}

components/esp_event/test/test_event.c

@@ -780,8 +780,12 @@ static void performance_test(bool dedicated_task)
     // Enabling profiling will slow down event dispatch, so the set threshold
     // is not valid when it is enabled.
 #else
+#ifndef CONFIG_SPIRAM_SUPPORT
     TEST_PERFORMANCE_GREATER_THAN(EVENT_DISPATCH, "%d", average);
-#endif
+#else
+    TEST_PERFORMANCE_GREATER_THAN(EVENT_DISPATCH_PSRAM, "%d", average);
+#endif // CONFIG_SPIRAM_SUPPORT
+#endif // CONFIG_EVENT_LOOP_PROFILING
 
     if (!dedicated_task) {
         ((esp_event_loop_instance_t*) loop)->task = mtask;
@@ -1074,4 +1078,4 @@ TEST_CASE("can dump event loop profile", "[event]")
 
     TEST_TEARDOWN();
 }
-#endif
+#endif

components/esp_http_client/esp_http_client.c

@@ -943,6 +943,10 @@ static esp_err_t esp_http_client_connect(esp_http_client_handle_t client)
             int ret = esp_transport_connect_async(client->transport, client->connection_info.host, client->connection_info.port, client->timeout_ms);
             if (ret == ASYNC_TRANS_CONNECT_FAIL) {
                 ESP_LOGE(TAG, "Connection failed");
+                if (strcasecmp(client->connection_info.scheme, "http") == 0) {
+                    ESP_LOGE(TAG, "Asynchronous mode doesn't work for HTTP based connection");
+                    return ESP_ERR_INVALID_ARG;
+                }
                 return ESP_ERR_HTTP_CONNECT;
             } else if (ret == ASYNC_TRANS_CONNECTING) {
                 ESP_LOGD(TAG, "Connection not yet established");

components/esp_http_client/include/esp_http_client.h

@@ -114,7 +114,7 @@ typedef struct {
     esp_http_client_transport_t transport_type;           /*!< HTTP transport type, see `esp_http_client_transport_t` */
     int                         buffer_size;              /*!< HTTP buffer size (both send and receive) */
     void                        *user_data;               /*!< HTTP user_data context */
-    bool                        is_async;                 /*!< Set asynchronous mode */
+    bool                        is_async;                 /*!< Set asynchronous mode, only supported with HTTPS for now */
 } esp_http_client_config_t;
 
 

components/idf_test/include/idf_performance.h

@@ -28,4 +28,5 @@
 #define IDF_PERFORMANCE_MIN_UDP_RX_THROUGHPUT                                   80
 #define IDF_PERFORMANCE_MIN_UDP_TX_THROUGHPUT                                   50
 // events dispatched per second by event loop library
-#define IDF_PERFORMANCE_MIN_EVENT_DISPATCH                                      25000
+#define IDF_PERFORMANCE_MIN_EVENT_DISPATCH                                      25000
+#define IDF_PERFORMANCE_MIN_EVENT_DISPATCH_PSRAM                                21000

components/idf_test/integration_test/TC_IT_BTSTK_GAP.yml

@@ -1017,13 +1017,13 @@ test cases:
   - - "SSC SSC2 bleadv -D -z start"
     - ["R SSC2 C +BLEADV:OK"]
   - *dut1_stop_adv
-  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x0004"
+  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x0004 -w 0x0004"
     - ["R SSC1 C +BLESCAN:SetScanParam,OK"]
-  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x4000"
+  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x4000 -w 0x4000"
     - ["R SSC1 C +BLESCAN:SetScanParam,OK"]
-  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x0003"
+  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x0003 -w 0x0003"
     - ["R SSC1 C +BLESCAN:SetScanParam,ERROR"]
-  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x4001"
+  - - "SSC SSC1 blescan -L -c 0 -s 1 -i 0x4001 -i 0x4001"
     - ["R SSC1 C +BLESCAN:SetScanParam,ERROR"]
 - ID: BTSTK_GAP_04004
   <<: *GAP_CASE
@@ -1043,13 +1043,13 @@ test cases:
   test environment: SSC_T1_4
   cmd set:
   - ""
-  - - "SSC SSC1 blescan -L -c 0 -w 0x0004"
+  - - "SSC SSC1 blescan -L -c 0 -w 0x0004 -i 0x0004"
     - ["R SSC1 C +BLESCAN:SetScanParam,OK"]
-  - - "SSC SSC1 blescan -L -c 0 -w 0x4000"
+  - - "SSC SSC1 blescan -L -c 0 -w 0x4000 -i 0x4000"
     - ["R SSC1 C +BLESCAN:SetScanParam,OK"]
-  - - "SSC SSC1 blescan -L -c 0 -w 0x0003"
+  - - "SSC SSC1 blescan -L -c 0 -w 0x0003 -i 0x0003"
     - ["R SSC1 C +BLESCAN:SetScanParam,ERROR"]
-  - - "SSC SSC1 blescan -L -c 0 -w 0x4001"
+  - - "SSC SSC1 blescan -L -c 0 -w 0x4001 -i 0x4001"
     - ["R SSC1 C +BLESCAN:SetScanParam,ERROR"]
 - ID: BTSTK_GAP_05001
   <<: *GAP_CASE

components/mbedtls/port/esp_bignum.c

@@ -41,6 +41,7 @@
 #include "freertos/FreeRTOS.h"
 #include "freertos/task.h"
 #include "freertos/semphr.h"
+#include "driver/periph_ctrl.h"
 
 /* Some implementation notes:
  *
@@ -91,12 +92,8 @@ void esp_mpi_acquire_hardware( void )
     /* newlib locks lazy initialize on ESP-IDF */
     _lock_acquire(&mpi_lock);
 
-    DPORT_REG_SET_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_RSA);
-    /* also clear reset on digital signature, otherwise RSA is held in reset */
-    DPORT_REG_CLR_BIT(DPORT_PERI_RST_EN_REG,
-                       DPORT_PERI_EN_RSA
-                       | DPORT_PERI_EN_DIGITAL_SIGNATURE);
-
+    /* Enable RSA hardware */
+    periph_module_enable(PERIPH_RSA_MODULE);
     DPORT_REG_CLR_BIT(DPORT_RSA_PD_CTRL_REG, DPORT_RSA_PD);
 
     while(DPORT_REG_READ(RSA_CLEAN_REG) != 1);
@@ -111,9 +108,8 @@ void esp_mpi_release_hardware( void )
 {
     DPORT_REG_SET_BIT(DPORT_RSA_PD_CTRL_REG, DPORT_RSA_PD);
 
-    /* don't reset digital signature unit, as this resets AES also */
-    DPORT_REG_SET_BIT(DPORT_PERI_RST_EN_REG, DPORT_PERI_EN_RSA);
-    DPORT_REG_CLR_BIT(DPORT_PERI_CLK_EN_REG, DPORT_PERI_EN_RSA);
+    /* Disable RSA hardware */
+    periph_module_disable(PERIPH_RSA_MODULE);
 
     _lock_release(&mpi_lock);
 }

components/newlib/test/test_time.c

@@ -135,12 +135,10 @@ TEST_CASE("test adjtime function", "[newlib]")
 static volatile bool exit_flag;
 static bool adjtime_test_result;
 static bool gettimeofday_test_result;
-static uint64_t count_adjtime;
-static uint64_t count_settimeofday;
-static uint64_t count_gettimeofday;
 
 static void adjtimeTask2(void *pvParameters)
 {
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
     struct timeval delta = {.tv_sec = 0, .tv_usec = 0};
     struct timeval outdelta;
 
@@ -150,34 +148,23 @@ static void adjtimeTask2(void *pvParameters)
         delta.tv_usec = 900000;
         if (delta.tv_sec >= 2146) delta.tv_sec = 1;
         adjtime(&delta, &outdelta);
-        count_adjtime++;
     }
+    xSemaphoreGive(*sema);
     vTaskDelete(NULL);
 }
 
-static void settimeofdayTask2(void *pvParameters)
+static void timeTask(void *pvParameters)
 {
+    xSemaphoreHandle *sema = (xSemaphoreHandle *) pvParameters;
     struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 };
 
     // although exit flag is set in another task, checking (exit_flag == false) is safe
     while (exit_flag == false) {
         tv_time.tv_sec += 1;
         settimeofday(&tv_time, NULL);
-        count_settimeofday++;
-        vTaskDelay(1);
-    }
-    vTaskDelete(NULL);
-}
-
-static void gettimeofdayTask2(void *pvParameters)
-{
-    struct timeval tv_time;
-    // although exit flag is set in another task, checking (exit_flag == false) is safe
-    while (exit_flag == false) {
         gettimeofday(&tv_time, NULL);
-        count_gettimeofday++;
-        vTaskDelay(1);
     }
+    xSemaphoreGive(*sema);
     vTaskDelete(NULL);
 }
 
@@ -185,32 +172,37 @@ TEST_CASE("test for no interlocking adjtime, gettimeofday and settimeofday funct
 {
     TaskHandle_t th[4];
     exit_flag = false;
-    count_adjtime = 0;
-    count_settimeofday = 0;
-    count_gettimeofday = 0;
     struct timeval tv_time = { .tv_sec = 1520000000, .tv_usec = 900000 };
     TEST_ASSERT_EQUAL(settimeofday(&tv_time, NULL), 0);
 
+    const int max_tasks = 2;
+    xSemaphoreHandle exit_sema[max_tasks];
+
+    for (int i = 0; i < max_tasks; ++i) {
+        exit_sema[i] = xSemaphoreCreateBinary();
+    }
+
 #ifndef CONFIG_FREERTOS_UNICORE
     printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask \n");
-    xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
-    xTaskCreatePinnedToCore(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
-    xTaskCreatePinnedToCore(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
+    xTaskCreatePinnedToCore(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
+    xTaskCreatePinnedToCore(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
 #else
     printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask, 3 - settimeofdayTask\n");
-    xTaskCreate(adjtimeTask2, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]);
-    xTaskCreate(gettimeofdayTask2, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]);
-    xTaskCreate(settimeofdayTask2, "settimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]);
+    xTaskCreate(adjtimeTask2, "adjtimeTask2", 2048, &exit_sema[0], UNITY_FREERTOS_PRIORITY - 1, &th[0]);
+    xTaskCreate(timeTask, "timeTask", 2048, &exit_sema[1], UNITY_FREERTOS_PRIORITY - 1, &th[1]);
 #endif
 
-    printf("start wait for 10 seconds\n");
-    vTaskDelay(10000 / portTICK_PERIOD_MS);
+    printf("start wait for 5 seconds\n");
+    vTaskDelay(5000 / portTICK_PERIOD_MS);
 
     // set exit flag to let thread exit
     exit_flag = true;
-    vTaskDelay(20 / portTICK_PERIOD_MS);
-    printf("count_adjtime %lld, count_settimeofday %lld, count_gettimeofday %lld\n", count_adjtime, count_settimeofday, count_gettimeofday);
-    TEST_ASSERT(count_adjtime > 1000LL && count_settimeofday > 1000LL && count_gettimeofday > 1000LL);
+    for (int i = 0; i < max_tasks; ++i) {
+        if (!xSemaphoreTake(exit_sema[i], 2000/portTICK_PERIOD_MS)) {
+            TEST_FAIL_MESSAGE("exit_sema not released by test task");
+        }
+        vSemaphoreDelete(exit_sema[i]);
+    }
 }
 
 static void adjtimeTask(void *pvParameters)
@@ -270,20 +262,15 @@ TEST_CASE("test for thread safety adjtime and gettimeofday functions", "[newlib]
     printf("CPU0 and CPU1. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n");
     xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0], 0);
     xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1], 1);
-
-    xTaskCreatePinnedToCore(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
-    xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3], 1);
+    xTaskCreatePinnedToCore(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2], 0);
 #else
     printf("Only one CPU. Tasks run: 1 - adjtimeTask, 2 - gettimeofdayTask\n");
     xTaskCreate(adjtimeTask, "adjtimeTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[0]);
     xTaskCreate(gettimeofdayTask, "gettimeofdayTask1", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[1]);
-
-    xTaskCreate(adjtimeTask, "adjtimeTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[2]);
-    xTaskCreate(gettimeofdayTask, "gettimeofdayTask2", 2048, NULL, UNITY_FREERTOS_PRIORITY - 1, &th[3]);
 #endif
 
-    printf("start wait for 10 seconds\n");
-    vTaskDelay(10000 / portTICK_PERIOD_MS);
+    printf("start wait for 5 seconds\n");
+    vTaskDelay(5000 / portTICK_PERIOD_MS);
 
     // set exit flag to let thread exit
     exit_flag = true;

components/soc/esp32/include/soc/periph_defs.h

@@ -52,6 +52,9 @@ typedef enum {
     PERIPH_WIFI_BT_COMMON_MODULE,
     PERIPH_BT_BASEBAND_MODULE,
     PERIPH_BT_LC_MODULE,
+    PERIPH_AES_MODULE,
+    PERIPH_SHA_MODULE,
+    PERIPH_RSA_MODULE,
 } periph_module_t;
 
 #ifdef __cplusplus

components/tcp_transport/transport.c

@@ -156,7 +156,7 @@ int esp_transport_connect(esp_transport_handle_t t, const char *host, int port,
 int esp_transport_connect_async(esp_transport_handle_t t, const char *host, int port, int timeout_ms)
 {
     int ret = -1;
-    if (t && t->_connect) {
+    if (t && t->_connect_async) {
         return t->_connect_async(t, host, port, timeout_ms);
     }
     return ret;

components/wear_levelling/WL_Flash.cpp

@@ -132,7 +132,7 @@ esp_err_t WL_Flash::init()
     result = this->flash_drv->read(this->addr_state2, state_copy, sizeof(wl_state_t));
     WL_RESULT_CHECK(result);
 
-    int check_size = offsetof(wl_state_t, crc);
+    int check_size = WL_STATE_CRC_LEN_V2;
     // Chech CRC and recover state
     uint32_t crc1 = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, check_size);
     uint32_t crc2 = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)state_copy, check_size);
@@ -288,7 +288,7 @@ esp_err_t WL_Flash::initSections()
 
     this->state.max_pos = 1 + this->flash_size / this->cfg.page_size;
 
-    this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, offsetof(wl_state_t, crc));
+    this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, WL_STATE_CRC_LEN_V2);
 
     result = this->flash_drv->erase_range(this->addr_state1, this->state_size);
     WL_RESULT_CHECK(result);
@@ -327,7 +327,7 @@ esp_err_t WL_Flash::updateV1_V2()
     esp_err_t result = ESP_OK;
     // Check crc for old version and old version
     ESP_LOGV(TAG, "%s start", __func__);
-    int check_size = offsetof(wl_state_t, device_id);
+    int check_size = WL_STATE_CRC_LEN_V1;
     // Chech CRC and recover state
     uint32_t crc1 = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, check_size);
     wl_state_t sa_copy;
@@ -365,9 +365,9 @@ esp_err_t WL_Flash::updateV1_V2()
 
         this->state.version = 2;
         this->state.pos = 0;
-        this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, offsetof(wl_state_t, crc));
         this->state.device_id = esp_random();
         memset(this->state.reserved, 0, sizeof(this->state.reserved));
+        this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, WL_STATE_CRC_LEN_V2);
 
         result = this->flash_drv->erase_range(this->addr_state1, this->state_size);
         WL_RESULT_CHECK(result);
@@ -493,7 +493,7 @@ esp_err_t WL_Flash::updateWL()
             this->state.move_count = 0;
         }
         // write main state
-        this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, offsetof(wl_state_t, crc));
+        this->state.crc = crc32::crc32_le(WL_CFG_CRC_CONST, (uint8_t *)&this->state, WL_STATE_CRC_LEN_V2);
 
         result = this->flash_drv->erase_range(this->addr_state1, this->state_size);
         WL_RESULT_CHECK(result);

components/wear_levelling/private_include/WL_State.h

@@ -45,5 +45,7 @@ public:
 static_assert(sizeof(wl_state_t) % 16 == 0, "Size of wl_state_t structure should be compatible with flash encryption");
 #endif // _MSC_VER
 
+#define WL_STATE_CRC_LEN_V1 offsetof(wl_state_t, device_id)
+#define WL_STATE_CRC_LEN_V2 offsetof(wl_state_t, crc)
 
 #endif // _WL_State_H_

components/wear_levelling/test/test_wl.c

@@ -279,27 +279,29 @@ TEST_CASE("Version update test", "[wear_levelling]")
     esp_partition_erase_range(&fake_partition, 0, fake_partition.size);
 
     esp_partition_write(&fake_partition, 0, test_partition_v1_bin_start,  fake_partition.size);
+    for (int i=0 ; i< 3 ; i++)
+    {
+        printf("Pass %i\n", i);
+        wl_handle_t handle;
+        TEST_ESP_OK(wl_mount(&fake_partition, &handle));
+        size_t sector_size = wl_sector_size(handle);
+        uint32_t* buff = (uint32_t*)malloc(sector_size);
 
-    wl_handle_t handle;
-    TEST_ESP_OK(wl_mount(&fake_partition, &handle));
-    size_t sector_size = wl_sector_size(handle);
-    uint32_t* buff = (uint32_t*)malloc(sector_size);
+        uint32_t init_val = COMPARE_START_CONST;
+        int test_count = fake_partition.size/sector_size - 4;
 
-    uint32_t init_val = COMPARE_START_CONST;
-    int test_count = fake_partition.size/sector_size - 4;
-
-    for (int m=0 ; m <  test_count; m++) {
-        TEST_ESP_OK(wl_read(handle, sector_size * m, buff, sector_size));
-        for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
-            uint32_t compare_val = init_val + i +  m*sector_size;
-            if (buff[i] != compare_val)
-            {
-                printf("error compare: 0x%08x != 0x%08x \n", buff[i], compare_val);
+        for (int m=0 ; m <  test_count; m++) {
+            TEST_ESP_OK(wl_read(handle, sector_size * m, buff, sector_size));
+            for (int i=0 ; i< sector_size/sizeof(uint32_t) ; i++) {
+                uint32_t compare_val = init_val + i +  m*sector_size;
+                if (buff[i] != compare_val)
+                {
+                    printf("error compare: 0x%08x != 0x%08x \n", buff[i], compare_val);
+                }
+                TEST_ASSERT_EQUAL( buff[i], compare_val);
             }
-            TEST_ASSERT_EQUAL( buff[i], compare_val);
         }
+        free(buff);
+        wl_unmount(handle);
     }
-
-    free(buff);
-    wl_unmount(handle);
 }

docs/en/api-guides/partition-tables.rst

@@ -6,7 +6,7 @@ Overview
 
 A single ESP32's flash can contain multiple apps, as well as many different kinds of data (calibration data, filesystems, parameter storage, etc). For this reason a partition table is flashed to (:ref:`default offset <CONFIG_PARTITION_TABLE_OFFSET>`) 0x8000 in the flash.
 
-Partition table length is 0xC00 bytes (maximum 95 partition table entries). An MD5 checksum is appended after the table data. If the partition table is signed due to `secure boot`, the signature is appended after the partition table.
+Partition table length is 0xC00 bytes (maximum 95 partition table entries). An MD5 checksum, which is used for checking the integrity of the partition table, is appended after the table data. If the partition table is signed due to `secure boot`, the signature is appended after the partition table.
 
 Each entry in the partition table has a name (label), type (app, data, or something else), subtype and the offset in flash where the partition is loaded.
 
@@ -42,9 +42,8 @@ Here is the summary printed for the "Factory app, two OTA definitions" configura
   ota_0,    0,    ota_0,  0x110000, 1M,
   ota_1,    0,    ota_1,  0x210000, 1M,
 
-* There are now three app partition definitions.
-* The type of all three are set as "app", but the subtype varies between the factory app at 0x10000 and the next two "OTA" apps.
-* There is also a new "ota data" slot, which holds the data for OTA updates. The bootloader consults this data in order to know which app to execute. If "ota data" is empty, it will execute the factory app.
+* There are now three app partition definitions. The type of the factory app (at 0x10000) and the next two "OTA" apps are all set to "app", but their subtypes are different.
+* There is also a new "otadata" slot, which holds the data for OTA updates. The bootloader consults this data in order to know which app to execute. If "ota data" is empty, it will execute the factory app.
 
 Creating Custom Tables
 ----------------------
@@ -80,45 +79,38 @@ If your application needs to store data, please add a custom partition type in t
 
 The bootloader ignores any partition types other than app (0) & data (1).
 
-Subtype
+SubType
 ~~~~~~~
 
-The 8-bit subtype field is specific to a given partition type.
+The 8-bit subtype field is specific to a given partition type. esp-idf currently only specifies the meaning of the subtype field for "app" and "data" partition types.
 
-esp-idf currently only specifies the meaning of the subtype field for "app" and "data" partition types.
+* When type is "app", the subtype field can be specified as factory (0), ota_0 (0x10) ... ota_15 (0x1F) or test (0x20).
 
-App Subtypes
-~~~~~~~~~~~~
+  - factory (0) is the default app partition. The bootloader will execute the factory app unless there it sees a partition of type data/ota, in which case it reads this partition to determine which OTA image to boot.
 
-When type is "app", the subtype field can be specified as factory (0), ota_0 (0x10) ... ota_15 (0x1F) or test (0x20).
+    - OTA never updates the factory partition.
+    - If you want to conserve flash usage in an OTA project, you can remove the factory partition and use ota_0 instead.
+  - ota_0 (0x10) ... ota_15 (0x1F) are the OTA app slots. Refer to the :doc:`OTA documentation <../api-reference/system/ota>` for more details, which then use the OTA data partition to configure which app slot the bootloader should boot. If using OTA, an application should have at least two OTA application slots (ota_0 & ota_1). Refer to the :doc:`OTA documentation <../api-reference/system/ota>` for more details.
+  - test (0x2) is a reserved subtype for factory test procedures. It is not currently supported by the esp-idf bootloader.
 
-- factory (0) is the default app partition. The bootloader will execute the factory app unless there it sees a partition of type data/ota, in which case it reads this partition to determine which OTA image to boot.
+* When type is "data", the subtype field can be specified as ota (0), phy (1), nvs (2), or nvs_keys (4).
 
-  - OTA never updates the factory partition.
-  - If you want to conserve flash usage in an OTA project, you can remove the factory partition and use ota_0 instead.
-- ota_0 (0x10) ... ota_15 (0x1F) are the OTA app slots. Refer to the :doc:`OTA documentation <../api-reference/system/ota>` for more details, which then use the OTA data partition to configure which app slot the bootloader should boot. If using OTA, an application should have at least two OTA application slots (ota_0 & ota_1). Refer to the :doc:`OTA documentation <../api-reference/system/ota>` for more details.
-- test (0x2) is a reserved subtype for factory test procedures. It is not currently supported by the esp-idf bootloader.
+  - ota (0) is the :ref:`OTA data partition <ota_data_partition>` which stores information about the currently selected OTA application. This partition should be 0x2000 bytes in size. Refer to the :ref:`OTA documentation <ota_data_partition>` for more details.
+  - phy (1) is for storing PHY initialisation data. This allows PHY to be configured per-device, instead of in firmware.
 
-Data Subtypes
-~~~~~~~~~~~~~
+    - In the default configuration, the phy partition is not used and PHY initialisation data is compiled into the app itself. As such, this partition can be removed from the partition table to save space.
+    - To load PHY data from this partition, run ``make menuconfig`` and enable :ref:`CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION` option. You will also need to flash your devices with phy init data as the esp-idf build system does not do this automatically.
+  - nvs (2) is for the :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>`.
 
-When type is "data", the subtype field can be specified as ota (0), phy (1), nvs (2).
+    - NVS is used to store per-device PHY calibration data (different to initialisation data).
+    - NVS is used to store WiFi data if the :doc:`esp_wifi_set_storage(WIFI_STORAGE_FLASH) <../api-reference/wifi/esp_wifi>` initialisation function is used.
+    - The NVS API can also be used for other application data.
+    - It is strongly recommended that you include an NVS partition of at least 0x3000 bytes in your project.
+    - If using NVS API to store a lot of data, increase the NVS partition size from the default 0x6000 bytes.
+  - nvs_keys (4) is for the NVS key partition. See :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>` for more details.
 
-- ota (0) is the :ref:`OTA data partition <ota_data_partition>` which stores information about the currently selected OTA application. This partition should be 0x2000 bytes in size. Refer to the :ref:`OTA documentation <ota_data_partition>` for more details.
-- phy (1) is for storing PHY initialisation data. This allows PHY to be configured per-device, instead of in firmware.
-
-  - In the default configuration, the phy partition is not used and PHY initialisation data is compiled into the app itself. As such, this partition can be removed from the partition table to save space.
-  - To load PHY data from this partition, run ``make menuconfig`` and enable :ref:`CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION` option. You will also need to flash your devices with phy init data as the esp-idf build system does not do this automatically.
-- nvs (2) is for the :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>`.
-
-  - NVS is used to store per-device PHY calibration data (different to initialisation data).
-  - NVS is used to store WiFi data if the :doc:`esp_wifi_set_storage(WIFI_STORAGE_FLASH) <../api-reference/wifi/esp_wifi>` initialisation function is used.
-  - The NVS API can also be used for other application data.
-  - It is strongly recommended that you include an NVS partition of at least 0x3000 bytes in your project.
-  - If using NVS API to store a lot of data, increase the NVS partition size from the default 0x6000 bytes.
-- keys (4) is for the NVS key partition. See :doc:`Non-Volatile Storage (NVS) API <../api-reference/storage/nvs_flash>` for more details.
-  - It is used to store NVS encryption keys when `NVS Encryption` feature is enabled.
-  - The size of this partition should be 4096 bytes (minimum partition size).
+    - It is used to store NVS encryption keys when `NVS Encryption` feature is enabled.
+    - The size of this partition should be 4096 bytes (minimum partition size).
 
 Other data subtypes are reserved for future esp-idf uses.
 
@@ -127,7 +119,7 @@ Offset & Size
 
 Partitions with blank offsets will start after the previous partition, or after the partition table in the case of the first partition.
 
-App partitions have to be at offsets aligned to 0x10000 (64K). If you leave the offset field blank, the tool will automatically align the partition. If you specify an unaligned offset for an app partition, the tool will return an error.
+App partitions have to be at offsets aligned to 0x10000 (64K). If you leave the offset field blank,  ``gen_esp32part.py`` will automatically align the partition. If you specify an unaligned offset for an app partition, the tool will return an error.
 
 Sizes and offsets can be specified as decimal numbers, hex numbers with the prefix 0x, or size multipliers K or M (1024 and 1024*1024 bytes).
 
@@ -151,7 +143,7 @@ To convert CSV to Binary manually::
 
   python gen_esp32part.py input_partitions.csv binary_partitions.bin
 
-To convert binary format back to CSV::
+To convert binary format back to CSV manually::
 
   python gen_esp32part.py binary_partitions.bin input_partitions.csv
 

docs/issue_template.md

@@ -26,11 +26,13 @@ If the issue cannot be solved after the steps before, please follow these instru
 
 - Development Kit:      [ESP32-Wrover-Kit|ESP32-DevKitC|ESP32-PICO-Kit|ESP32-LyraT|ESP32-LyraTD-MSC|none]
 - Kit version (for WroverKit/PicoKit/DevKitC): [v1|v2|v3|v4]
-- Core (if using chip or module): [ESP32-Wrover-I|ESP32-Wrover|ESP32-PICO-D4|ESP32-SOLO-I|ESP32-WROOM-32U|ESP-WROOM-32D|ESP-WROOM32|ESP32]
-- IDF version (``git rev-parse --short HEAD`` to get the commit id.): 
-    //bd6ea4393c7d2f059fc4decc70f1ec3eb3597268
-- Development Env:		[Arduino IDE|Make|Eclipse|other]
-- Operating System:		[Windows|Ubuntu|MacOS]
+- Module or chip used:  [ESP32-WROOM-32|ESP32-WROOM-32D|ESP32-WROOM-32U|ESP32-WROVER|ESP32-WROVER-I|ESP32-WROVER-B|ESP32-WROVER-IB|ESP32-SOLO-1|ESP32-PICO-D4|ESP32]
+- IDF version (run ``git describe --tags`` to find it): 
+    // v3.2-dev-1148-g96cd3b75c
+- Build System:         [Make|CMake]
+- Compiler version (run ``xtensa-esp32-elf-gcc --version`` to find it):
+    // 1.22.0-80-g6c4433a
+- Operating System:     [Windows|Linux|macOS]
 - Power Supply:         [USB|external 5V|external 3.3V|Battery]
 
 ## Problem Description

docs/zh_CN/api-guides/index.rst

@@ -17,7 +17,7 @@ API 指南
    High Level Interrupts <hlinterrupts>
    JTAG Debugging <jtag-debugging/index>
    Bootloader <bootloader>
-   Partition Tables <partition-tables>
+   分区表 <partition-tables>
    Secure Boot <../security/secure-boot>
    ULP Coprocessor <ulp>
    ULP Coprocessor (CMake) <ulp-cmake>

docs/zh_CN/api-guides/partition-tables.rst

@@ -1 +1,177 @@
-.. include:: ../../en/api-guides/partition-tables.rst
+分区表
+======
+
+概述
+----
+
+每片 ESP32 的 flash 可以包含多个应用程序，以及多种不同类型的数据（例如校准数据、文件系统数据、参数存储器数据等）。因此，我们需要引入分区表的概念。
+
+具体来说，ESP32 在 flash 的 :ref:`默认偏移地址 <CONFIG_PARTITION_TABLE_OFFSET>` 0x8000 处烧写一张分区表。该分区表的长度为 0xC00 字节（最多可以保存 95 条分区表条目）。分区表数据后还保存着该表的 MD5 校验和，用于验证分区表的完整性。此外，如果芯片使能了 :doc:`安全启动 </security/secure-boot>` 功能，则该分区表后还会保存签名信息。
+
+分区表中的每个条目都包括以下几个部分：Name（标签）、Type（app、data 等）、SubType 以及在 flash 中的偏移量（分区的加载地址）。
+
+在使用分区表时，最简单的方法就是用 `make menuconfig` 选择一张预定义的分区表：
+
+-  "Single factory app, no OTA"
+-  "Factory app, two OTA definitions"
+
+在以上两种选项中，出厂应用程序均将被烧录至 flash 的 0x10000 偏移地址处。这时，运行 `make partition_table` ，即可以打印当前使用分区表的信息摘要。
+
+内置分区表
+------------
+
+以下是 "Single factory app, no OTA" 选项的分区表信息摘要：
+
+
+   # Espressif ESP32 Partition Table
+   # Name,   Type, SubType, Offset,  Size,   Flags
+   nvs,      data, nvs,     0x9000,  0x6000,
+   phy_init, data, phy,     0xf000,  0x1000,
+   factory,  app,  factory, 0x10000, 1M,
+
+-  flash 的 0x10000 (64KB) 偏移地址处存放一个标记为 "factory" 的二进制应用程序，且 Bootloader 将默认加载这个应用程序。
+-  分区表中还定义了两个数据区域，分别用于存储 NVS 库专用分区和 PHY 初始化数据。
+
+以下是 "Factory app, two OTA definitions" 选项的分区表信息摘要：
+
+   # Espressif ESP32 Partition Table
+   # Name,   Type, SubType, Offset,   Size,   Flags
+   nvs,      data, nvs,     0x9000,   0x4000,
+   otadata,  data, ota,     0xd000,   0x2000,
+   phy_init, data, phy,     0xf000,   0x1000,
+   factory,  app,  factory, 0x10000,  1M,
+   ota_0,    app,  ota_0,   0x110000, 1M,
+   ota_1,    app,  ota_1,   0x210000, 1M,
+
+-  分区表中定义了三个应用程序分区，这三个分区的类型都被设置为 “app”，但具体 app 类型不同。其中，位于 0x10000 偏移地址处的为出厂应用程序（factory），其余两个为 OTA 应用程序（ota_0，ota_1）。
+-  新增了一个名为 “otadata” 的数据分区，用于保存 OTA 升级时候需要的数据。Bootloader 会查询该分区的数据，以判断该从哪个 OTA 应用程序分区加载程序。如果 “otadata” 分区为空，则会执行出厂程序。
+
+创建自定义分区表
+----------------
+
+如果在 ``menuconfig`` 中选择了 “Custom partition table CSV”，则还需要输入该分区表的 CSV 文件在项目中的路径。CSV 文件可以根据需要，描述任意数量的分区信息。
+
+CSV 文件的格式与上面摘要中打印的格式相同，但是在 CSV 文件中并非所有字段都是必需的。例如下面是一个自定义的 OTA 分区表的 CSV 文件：
+
+   # Name,   Type, SubType,  Offset,   Size,  Flags
+   nvs,      data, nvs,      0x9000,  0x4000
+   otadata,  data, ota,      0xd000,  0x2000
+   phy_init, data, phy,      0xf000,  0x1000
+   factory,  app,  factory,  0x10000,  1M
+   ota_0,    app,  ota_0,    ,         1M
+   ota_1,    app,  ota_1,    ,         1M
+   nvs_key,  data, nvs_keys, ,        0x1000
+
+-  字段之间的空格会被忽略，任何以 ``#`` 开头的行（注释）也会被忽略。
+-  CSV 文件中的每个非注释行均为一个分区定义。
+-  每个分区的 ``Offset`` 字段可以为空，``gen_esp32part.py`` 工具会从分区表位置的后面开始自动计算并填充该分区的偏移地址，同时确保每个分区的偏移地址正确对齐。
+
+Name 字段
+~~~~~~~~~
+
+Name 字段可以是任何有意义的名称，但不能超过 16 个字符（之后的内容将被截断）。该字段对 ESP32 并不是特别重要。
+
+Type 字段
+~~~~~~~~~
+
+Type 字段可以指定为 app (0) 或者 data (1)，也可以直接使用数字 0-254（或者十六进制 0x00-0xFE）。注意，0x00-0x3F 不得使用（预留给 esp-idf 的核心功能）。
+
+如果您的应用程序需要保存数据，请在 0x40-0xFE 内添加一个自定义分区类型。
+
+注意，bootloader 将忽略 app (0) 和 data (1) 以外的其他分区类型。
+
+
+SubType 字段
+~~~~~~~~~~~~
+
+SubType 字段长度为 8 bit，内容与具体 Type 有关。目前，esp-idf 仅仅规定了 “app” 和 “data” 两种子类型。
+   
+* 当 Type 定义为 ``app`` 时，SubType 字段可以指定为 factory (0)，ota_0 (0x10) ... ota_15 (0x1F) 或者 test (0x20)。
+
+   -  factory (0) 是默认的 app 分区。Bootloader 将默认加在该应用程序。但如果存在类型为 data/ota 分区，则 Bootloader 将加载 data/ota 分区中的数据，进而判断启动哪个 OTA 镜像文件。
+      -  OTA 升级永远都不会更新 factory 分区中的内容。
+      -  如果您希望在 OTA 项目中预留更多 flash，可以删除 factory 分区，转而使用 ota_0 分区。
+
+   -  ota_0 (0x10) ... ota_15 (0x1F) 为 OTA 应用程序分区，Bootloader 将根据 OTA 数据分区中的数据来决定加载哪个 OTA 应用程序分区中的程序。在使用 OTA 功能时，应用程序应至少拥有 2 个 OTA 应用程序分区（ota_0 和 ota_1）。更多详细信息，请参考 :doc:`OTA 文档 </api-reference/system/ota>` 。
+   -  test (0x2) 为预留 app 子类型，用于工厂测试过程。注意，目前，esp-idf 并不支持这种子类型。
+   
+* 当 Type 定义为 ``data`` 时，SubType 字段可以指定为 ota (0)，phy (1)，nvs (2) 或者 nvs_keys (4)。
+
+   -  ota (0) 即 :ref:`OTA 数据分区 <ota_data_partition>` ，用于存储当前所选的 OTA 应用程序的信息。这个分区的大小需要设定为 0x2000。更多详细信息，请参考 :doc:`OTA 文档 <../api-reference/system/ota>` 。
+   -  phy (1) 分区用于存放 PHY 初始化数据，从而保证可以为每个设备单独配置 PHY，而非必须采用固件中的统一 PHY 初始化数据。
+ 
+      -  默认配置下，phy 分区并不启用，而是直接将 phy 初始化数据编译至应用程序中，从而节省分区表空间（直接将此分区删掉）。
+      -  如果需要从此分区加载 phy 初始化数据，请运行 ``make menuconfig``，并且使能 :ref:`CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION` 选项。此时，您还需要手动将 phy 初始化数据烧至设备 flash（esp-idf 编译系统并不会自动完成该操作）。
+   -  nvs (2) 是专门给 :doc:`非易失性存储 (NVS) API <../api-reference/storage/nvs_flash>` 使用的分区。
+
+      -  用于存储每台设备的 PHY 校准数据（注意，并不是 PHY 初始化数据）。
+      -  用于存储 Wi-Fi 数据（如果使用了 :cpp:func:`esp_wifi_set_storage(WIFI_STORAGE_FLASH) <esp_wifi_set_storage>` 初始化函数）。
+      -  NVS API 还可以用于其他应用程序数据。
+      -  强烈建议您应为 NVS 分区分配至少 0x3000 字节空间。
+      -  如果使用 NVS API 存储大量数据，请增加 NVS 分区的大小（默认是 0x6000 字节）。
+   -  nvs_keys (4) 是 NVS 秘钥分区。详细信息，请参考 :doc:`非易失性存储 (NVS) API <../api-reference/storage/nvs_flash>` 文档。
+
+      -  用于存储加密密钥（如果启用了 `NVS 加密` 功能）。
+      -  此分区应至少设定为 4096 字节。
+
+其它数据子类型已预留给 esp-idf 的未来使用。
+
+Offset 和 Size 字段
+~~~~~~~~~~~~~~~~~~~
+
+分区若为指定偏移地址，则会紧跟着前一个分区之后开始。若此分区为首个分区，则将紧跟着分区表开始。
+
+app 分区的偏移地址必须要与 0x10000 (64K) 对齐，如果将偏移字段留空，``gen_esp32part.py`` 工具会自动计算得到一个满足对齐要求的偏移地址。如果 app 分区的偏移地址没有与 0x10000 (64K) 对齐，则该工具会报错。
+
+app 分区的大小和偏移地址可以采用十进制数、以 0x 为前缀的十六进制数，且支持 K 或 M 的倍数单位（分别代表 1024 和 1024*1024 字节）。
+
+如果您希望允许分区表中的分区采用任意起始偏移量 (:ref:`CONFIG_PARTITION_TABLE_OFFSET`)，请将分区表（CSV 文件）中所有分区的偏移字段都留空。注意，此时，如果您更改了分区表中任意分区的偏移地址，则其他分区的偏移地址也会跟着改变。这种情况下，如果您之前还曾设定某个分区采用固定偏移地址，则可能造成分区表冲突，从而导致报错。
+
+Flags 字段
+~~~~~~~~~~
+
+当前仅支持 ``encrypted`` 标记。如果 Flags 字段设置为 ``encrypted``，且已启用 :doc:`Flash Encryption </security/flash-encryption>` 功能，则该分区将会被加密。
+
+.. note:: 
+
+   ``app`` 分区始终会被加密，不管 Flags 字段是否设置。
+
+生成二进制分区表
+----------------
+
+烧写到 ESP32 中的分区表采用二进制格式，而不是 CSV 文件本身。此时，:component_file:`partition_table/gen_esp32part.py` 工具可以实现 CSV 和二进制文件之间的转换。
+
+如果您在 ``make menuconfig`` 指定了分区表 CSV 文件的名称，然后执行 ``make partition_table``。这时，转换将在编译过程中自动完成。
+
+手动将 CSV 文件转换为二进制文件:
+
+   python gen_esp32part.py input_partitions.csv binary_partitions.bin
+
+手动将二进制文件转换为 CSV 文件:
+
+   python gen_esp32part.py binary_partitions.bin input_partitions.csv
+
+在标准输出（stdout）上，打印二进制分区表的内容（在运行  ``make partition_table`` 时，我们正是这样打印上文展示的信息摘要的）:
+
+   python gen_esp32part.py binary_partitions.bin
+
+MD5 校验和
+~~~~~~~~~~
+
+二进制格式的分区表中含有一个 MD5 校验和。这个 MD5 校验和是根据分区表内容计算的，可在设备启动阶段，用于验证分区表的完整性。
+
+注意，一些版本较老的 bootloader 无法支持 MD5 校验，如果发现 MD5 校验和则将报错 ``invalid magic number 0xebeb``。此时，用户可通过 ``gen_esp32part.py`` 的 ``--disable-md5sum`` 选项或者 ``menuconfig`` 的 :ref:`CONFIG_PARTITION_TABLE_MD5` 选项关闭 MD5 校验。
+
+烧写分区表
+----------
+
+-  ``make partition_table-flash`` ：使用 esptool.py 工具烧写分区表。
+-  ``make flash`` ：会烧写所有内容，包括分区表。
+
+在执行 ``make partition_table`` 命令时，手动烧写分区表的命令也将打印在终端上。
+
+.. note:: 
+
+   分区表的更新并不会擦除根据之前分区表存储的数据。此时，您可以使用 ``make erase_flash`` 命令或者 ``esptool.py erase_flash`` 命令来擦除 flash 中的所有内容。
+
+.. _secure boot: security/secure-boot.rst

examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py

@@ -5,66 +5,29 @@ import time
 import socket
 import imp
 import ssl
-
-use_mqtt_client_sketch = False
-
-try:
-    imp.find_module('paho')
-    import paho.mqtt.client as mqtt
-    # Make things with supposed existing module
-except ImportError:
-    use_mqtt_client_sketch = True
-    pass
-
-global g_recv_topic
-global g_recv_data
+import paho.mqtt.client as mqtt
 
 g_recv_data=""
-
-# This is only a workaround for running mqtt client with 'hardcoded' data using plain socket interface
-def mqtt_client_sketch():
-    global g_recv_topic
-    global g_recv_data
-    connect_msg = bytearray([0x10, 0x0c, 00, 0x04, 0x4d, 0x51, 0x54, 0x54, 0x04, 0x02, 00, 0x3c, 00, 00])
-    send_qos0_msg = bytearray([ 0x30, 0x1a, 0x00, 0x0b, 0x2f, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x2f, 0x71, 0x6f, 0x73, 0x30, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x6f, 0x5f, 0x65, 0x73, 0x70, 0x33, 0x32])
-    subscribe_qos0 = bytearray([ 0x82, 0x10, 0x00, 0x01, 0x00, 0x0b, 0x2f, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x2f, 0x71, 0x6f, 0x73, 0x30, 0x00] )
-    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    client.settimeout(30)
-    cli = ssl.wrap_socket(client)
-    cli.connect(("iot.eclipse.org", 8883))
-    cli.send(connect_msg)
-    data = cli.recv(1024)
-    print("Connect ack received {}".format(data))
-    cli.send(subscribe_qos0)
-    data = cli.recv(1024)
-    print("Subscibe ack received {}".format(data))
-    start = time.time()
-    while (time.time() - start) <= 20:
-        data = cli.recv(1024)
-        print("Data received {}".format(data[-17:]))
-        if data[-15:] == "/topic/qos0data":
-            g_recv_topic = data[-15:][:11]
-            g_recv_data = data[-4:]
-            cli.send(send_qos0_msg)
-            data = cli.recv(1024)
-            print("data ack received {}".format(data))
-            break
-    cli.close()
+g_recv_topic=""
+g_broker_connected=0
 
 # The callback for when the client receives a CONNACK response from the server.
 def on_connect(client, userdata, flags, rc):
+    global g_broker_connected
     print("Connected with result code "+str(rc))
+    g_broker_connected = 1
     client.subscribe("/topic/qos0")
 
 # The callback for when a PUBLISH message is received from the server.
 def on_message(client, userdata, msg):
     global g_recv_topic
     global g_recv_data
-    if g_recv_data == "" and  msg.payload == "data":
+    payload = msg.payload.decode()
+    if g_recv_data == "" and  payload == "data":
         client.publish("/topic/qos0", "data_to_esp32")
         g_recv_topic = msg.topic
-        g_recv_data = msg.payload
-    print(msg.topic+" "+str(msg.payload))
+        g_recv_data = payload
+    print(msg.topic+" "+str(payload))
 
 # this is a test case write with tiny-test-fw.
 # to run test cases outside tiny-test-fw,
@@ -76,7 +39,7 @@ if test_fw_path and test_fw_path not in sys.path:
 
 import TinyFW
 import IDF
-
+import DUT
 
 
 
@@ -84,6 +47,8 @@ import IDF
 def test_examples_protocol_mqtt_ssl(env, extra_data):
     global g_recv_topic
     global g_recv_data
+    global g_broker_connected
+    broker_url="iot.eclipse.org"
     """
     steps: |
       1. join AP and connects to ssl broker
@@ -97,12 +62,13 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
     bin_size = os.path.getsize(binary_file)
     IDF.log_performance("mqtt_ssl_bin_size", "{}KB".format(bin_size//1024))
     IDF.check_performance("mqtt_ssl_size", bin_size//1024)
-    # 1. start test
+    # 1. start test (and check the environment is healthy)
     dut1.start_app()
+    client = None
     # 2. Test connects to a broker
-    if use_mqtt_client_sketch:
-        mqtt_client_sketch()
-    else:
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
         client = mqtt.Client()
         client.on_connect = on_connect
         client.on_message = on_message
@@ -110,15 +76,21 @@ def test_examples_protocol_mqtt_ssl(env, extra_data):
                     None,
                     None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1, ciphers=None)
         client.tls_insecure_set(True)
-
-        print "Connecting..."
-        client.connect("iot.eclipse.org", 8883, 60)
-        print "...done"
-        print "Start Looping..."
-        start = time.time()
-        while (time.time() - start) <= 20:
-            client.loop()
-        print "...done"
+        print("Connecting...")
+        client.connect(broker_url, 8883, 60)
+        print("...done")
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+    except:
+        print("ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:".format(broker_url, sys.exc_info()[0]))
+        raise
+    print("Start Looping...")
+    start = time.time()
+    while (time.time() - start) <= 20:
+        client.loop()
+    print("...done")
+    if g_broker_connected == 0:
+        raise ValueError('ENV_TEST_FAILURE: Test script cannot connect to broker: {}'.format(broker_url))
     # 3. check the message received back from the server
     if g_recv_topic == "/topic/qos0" and g_recv_data == "data" :
         print("PASS: Received correct message")

examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py

@@ -3,9 +3,10 @@ import os
 import sys
 from socket import *
 from threading import Thread
+import struct
 import time
 
-global msgid
+msgid=-1
 
 def get_my_ip():
     s1 = socket(AF_INET, SOCK_DGRAM)
@@ -17,14 +18,20 @@ def get_my_ip():
 def mqqt_server_sketch(my_ip, port):
     global msgid
     print("Starting the server on {}".format(my_ip))
-    s=socket(AF_INET, SOCK_STREAM)
-    s.settimeout(60)
-    s.bind((my_ip, port))
-    s.listen(1)
-    q,addr=s.accept()
-    q.settimeout(30)
-    print("connection accepted")
-    # q.send(g_msg_to_client)
+    s = None
+    try:
+        s=socket(AF_INET, SOCK_STREAM)
+        s.settimeout(60)
+        s.bind((my_ip, port))
+        s.listen(1)
+        q,addr=s.accept()
+        q.settimeout(30)
+        print("connection accepted")
+    except:
+        print("Local server on {}:{} listening/accepting failure: {}"
+                "Possibly check permissions or firewall settings"
+                "to accept connections on this address".format(my_ip, port, sys.exc_info()[0]))
+        raise
     data = q.recv(1024)
     # check if received initial empty message
     print("received from client {}".format(data))
@@ -32,7 +39,7 @@ def mqqt_server_sketch(my_ip, port):
     q.send(data)
     # try to receive qos1
     data = q.recv(1024)
-    msgid = ord(data[15])*256+ord(data[16])
+    msgid = struct.unpack(">H", data[15:17])[0]
     print("received from client {}, msgid: {}".format(data, msgid))
     data = bytearray([0x40, 0x02, data[15], data[16]])
     q.send(data)
@@ -50,6 +57,7 @@ if test_fw_path and test_fw_path not in sys.path:
 
 import TinyFW
 import IDF
+import DUT
 
 
 
@@ -75,21 +83,21 @@ def test_examples_protocol_mqtt_qos1(env, extra_data):
     thread1 = Thread(target = mqqt_server_sketch, args = (host_ip,1883))
     thread1.start()
     # 2. start the dut test and wait till client gets IP address
-    dut1.start_app()
+    dut1.start_app()    
     # waiting for getting the IP address
-    data = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
-    # time.sleep(15)
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+
     print ("writing to device: {}".format("mqtt://" + host_ip + "\n"))
     dut1.write("mqtt://" + host_ip + "\n")
     thread1.join()
     print ("Message id received from server: {}".format(msgid))
     # 3. check the message id was enqueued and then deleted
     msgid_enqueued = dut1.expect(re.compile(r"OUTBOX: ENQUEUE msgid=([0-9]+)"), timeout=30)
-    # expect_txt="OUTBOX: ENQUEUE msgid=" + str(msgid)
-    # dut1.expect(re.compile(expect_txt), timeout=30)
     msgid_deleted = dut1.expect(re.compile(r"OUTBOX: DELETED msgid=([0-9]+)"), timeout=30)
-    # expect_txt="OUTBOX: DELETED msgid=" + str(msgid)
-    # dut1.expect(re.compile(expect_txt), timeout=30)
     # 4. check the msgid of received data are the same as that of enqueued and deleted from outbox
     if (msgid_enqueued[0] == str(msgid) and msgid_deleted[0] == str(msgid)):
         print("PASS: Received correct msg id")

examples/protocols/mqtt/ws/mqtt_ws_example_test.py

@@ -1,70 +1,35 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+from builtins import str
 import re
 import os
 import sys
 import time
 import socket
 import imp
-
-use_mqtt_client_sketch = False
-
-try:
-    imp.find_module('paho')
-    import paho.mqtt.client as mqtt
-    # Make things with supposed existing module
-except ImportError:
-    use_mqtt_client_sketch = True
-    pass
-
-global g_recv_topic
-global g_recv_data
+import paho.mqtt.client as mqtt
 
 g_recv_data=""
-
-# This is only a workaround for running mqtt client with 'hardcoded' data using plain socket interface
-def mqtt_client_sketch():
-    global g_recv_topic
-    global g_recv_data
-    http_connect = bytearray([ 0x47, 0x45, 0x54, 0x20, 0x2f, 0x77, 0x73, 0x20, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31, 0x0d, 0x0a, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x69, 0x6f, 0x74, 0x2e, 0x65, 0x63, 0x6c, 0x69, 0x70, 0x73, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x3a, 0x38, 0x30, 0x0d, 0x0a, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x3a, 0x20, 0x77, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x31, 0x33, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x20, 0x6d, 0x71, 0x74, 0x74, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x4b, 0x65, 0x79, 0x3a, 0x20, 0x6c, 0x35, 0x61, 0x50, 0x41, 0x64, 0x6d, 0x4a, 0x52, 0x65, 0x32, 0x79, 0x55, 0x42, 0x79, 0x68, 0x37, 0x35, 0x72, 0x58, 0x68, 0x51, 0x3d, 0x3d, 0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20, 0x69, 0x6f, 0x74, 0x2e, 0x65, 0x63, 0x6c, 0x69, 0x70, 0x73, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x3a, 0x38, 0x30, 0x0d, 0x0a, 0x0d, 0x0a])
-    connect_msg = bytearray([0x82, 0x8e, 0x82, 0x1a, 0xe6, 0x22, 0x92, 0x16, 0xe6, 0x26, 0xcf, 0x4b, 0xb2, 0x76, 0x86, 0x18, 0xe6, 0x1e, 0x82, 0x1a])
-    send_qos0_msg = bytearray([ 0x82, 0x9c, 0x44, 0x78, 0xdf, 0x8e, 0x74, 0x62, 0xdf, 0x85, 0x6b, 0x0c, 0xb0, 0xfe, 0x2d, 0x1b, 0xf0, 0xff, 0x2b, 0x0b, 0xef, 0xea, 0x25, 0x0c, 0xbe, 0xd1, 0x30, 0x17, 0x80, 0xeb, 0x37, 0x08, 0xec, 0xbc ])
-    subscribe_qos0 = bytearray([ 0x82, 0x92, 0x8e, 0x31, 0x8c, 0x4a, 0x0c, 0x21, 0x8c, 0x4b, 0x8e, 0x3a, 0xa3, 0x3e, 0xe1, 0x41, 0xe5, 0x29, 0xa1, 0x40, 0xe3, 0x39, 0xbe, 0x31] )
-    cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    cli.settimeout(30)
-    cli.connect(("iot.eclipse.org", 80))
-    cli.send(http_connect)
-    cli.send(connect_msg)
-    data = cli.recv(1024)
-    print("Connect ack received {}".format(data))
-    cli.send(subscribe_qos0)
-    data = cli.recv(1024)
-    print("Subscibe ack received {}".format(data))
-    start = time.time()
-    while (time.time() - start) <= 20:
-        data = cli.recv(1024)
-        print("Data received {}".format(data[-17:]))
-        if data[-15:] == "/topic/qos0data":
-            g_recv_topic = data[-15:][:11]
-            g_recv_data = data[-4:]
-            cli.send(send_qos0_msg)
-            data = cli.recv(1024)
-            print("data ack received {}".format(data))
-            break
-    cli.close()
+g_recv_topic=""
+g_broker_connected=0
 
 # The callback for when the client receives a CONNACK response from the server.
 def on_connect(client, userdata, flags, rc):
+    global g_broker_connected
     print("Connected with result code "+str(rc))
+    g_broker_connected = 1
     client.subscribe("/topic/qos0")
 
 # The callback for when a PUBLISH message is received from the server.
 def on_message(client, userdata, msg):
     global g_recv_topic
     global g_recv_data
-    if g_recv_data == "" and  msg.payload == "data":
+    payload = msg.payload.decode()
+    if g_recv_data == "" and  payload == "data":
         client.publish("/topic/qos0", "data_to_esp32")
         g_recv_topic = msg.topic
-        g_recv_data = msg.payload
-    print(msg.topic+" "+str(msg.payload))
+        g_recv_data = payload
+    print(msg.topic+" "+payload)
 
 # this is a test case write with tiny-test-fw.
 # to run test cases outside tiny-test-fw,
@@ -76,14 +41,15 @@ if test_fw_path and test_fw_path not in sys.path:
 
 import TinyFW
 import IDF
-
-
+import DUT
 
 
 @IDF.idf_example_test(env_tag="Example_WIFI")
 def test_examples_protocol_mqtt_ws(env, extra_data):
     global g_recv_topic
     global g_recv_data
+    global g_broker_connected
+    broker_url="iot.eclipse.org"
     """
     steps: |
       1. join AP and connects to ws broker
@@ -97,24 +63,32 @@ def test_examples_protocol_mqtt_ws(env, extra_data):
     bin_size = os.path.getsize(binary_file)
     IDF.log_performance("mqtt_websocket_bin_size", "{}KB".format(bin_size//1024))
     IDF.check_performance("mqtt_websocket_size", bin_size//1024)
-    # 1. start test
+    # 1. start test (and check the environment is healthy)
     dut1.start_app()
+    client = None
     # 2. Test connects to a broker
-    if use_mqtt_client_sketch:
-        mqtt_client_sketch()
-    else:
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
         client = mqtt.Client(transport="websockets")
         client.on_connect = on_connect
         client.on_message = on_message
         client.ws_set_options(path="/ws", headers=None)
-        print "Connecting..."
-        client.connect("iot.eclipse.org", 80, 60)
-        print "...done"
-        print "Start Looping..."
-        start = time.time()
-        while (time.time() - start) <= 20:
-            client.loop()
-        print "...done"
+        print("Connecting...")
+        client.connect(broker_url, 80, 60)
+        print("...done")
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+    except:
+        print("ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:".format(broker_url, sys.exc_info()[0]))
+        raise
+    print("Start Looping...")
+    start = time.time()
+    while (time.time() - start) <= 20:
+        client.loop()
+    print("...done")
+    if g_broker_connected == 0:
+        raise ValueError('ENV_TEST_FAILURE: Test script cannot connect to broker: {}'.format(broker_url))
     # 3. check the message received back from the server
     if g_recv_topic == "/topic/qos0" and g_recv_data == "data" :
         print("PASS: Received correct message")

examples/protocols/mqtt/wss/mqtt_wss_example_test.py

@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import re
 import os
 import sys
@@ -5,68 +6,29 @@ import time
 import socket
 import imp
 import ssl
-
-use_mqtt_client_sketch = False
-
-try:
-    imp.find_module('paho')
-    import paho.mqtt.client as mqtt
-    # Make things with supposed existing module
-except ImportError:
-    use_mqtt_client_sketch = True
-    pass
-
-global g_recv_topic
-global g_recv_data
+import paho.mqtt.client as mqtt
 
 g_recv_data=""
-
-# This is only a workaround for running mqtt client with 'hardcoded' data using plain socket interface
-def mqtt_client_sketch():
-    global g_recv_topic
-    global g_recv_data
-    http_connect = bytearray([ 0x47, 0x45, 0x54, 0x20, 0x2f, 0x77, 0x73, 0x20, 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31, 0x0d, 0x0a, 0x4f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x3a, 0x20, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x69, 0x6f, 0x74, 0x2e, 0x65, 0x63, 0x6c, 0x69, 0x70, 0x73, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x3a, 0x38, 0x30, 0x0d, 0x0a, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x3a, 0x20, 0x77, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x31, 0x33, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x3a, 0x20, 0x6d, 0x71, 0x74, 0x74, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x0d, 0x0a, 0x53, 0x65, 0x63, 0x2d, 0x57, 0x65, 0x62, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2d, 0x4b, 0x65, 0x79, 0x3a, 0x20, 0x6c, 0x35, 0x61, 0x50, 0x41, 0x64, 0x6d, 0x4a, 0x52, 0x65, 0x32, 0x79, 0x55, 0x42, 0x79, 0x68, 0x37, 0x35, 0x72, 0x58, 0x68, 0x51, 0x3d, 0x3d, 0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20, 0x69, 0x6f, 0x74, 0x2e, 0x65, 0x63, 0x6c, 0x69, 0x70, 0x73, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x3a, 0x38, 0x30, 0x0d, 0x0a, 0x0d, 0x0a])
-    connect_msg = bytearray([0x82, 0x8e, 0x82, 0x1a, 0xe6, 0x22, 0x92, 0x16, 0xe6, 0x26, 0xcf, 0x4b, 0xb2, 0x76, 0x86, 0x18, 0xe6, 0x1e, 0x82, 0x1a])
-    send_qos0_msg = bytearray([ 0x82, 0x9c, 0x44, 0x78, 0xdf, 0x8e, 0x74, 0x62, 0xdf, 0x85, 0x6b, 0x0c, 0xb0, 0xfe, 0x2d, 0x1b, 0xf0, 0xff, 0x2b, 0x0b, 0xef, 0xea, 0x25, 0x0c, 0xbe, 0xd1, 0x30, 0x17, 0x80, 0xeb, 0x37, 0x08, 0xec, 0xbc ])
-    subscribe_qos0 = bytearray([ 0x82, 0x92, 0x8e, 0x31, 0x8c, 0x4a, 0x0c, 0x21, 0x8c, 0x4b, 0x8e, 0x3a, 0xa3, 0x3e, 0xe1, 0x41, 0xe5, 0x29, 0xa1, 0x40, 0xe3, 0x39, 0xbe, 0x31] )
-    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    client.settimeout(30)
-    cli = ssl.wrap_socket(client)
-    cli.connect(("iot.eclipse.org", 443))
-    cli.send(http_connect)
-    cli.send(connect_msg)
-    data = cli.recv(1024)
-    print("Connect ack received {}".format(data))
-    cli.send(subscribe_qos0)
-    data = cli.recv(1024)
-    print("Subscibe ack received {}".format(data))
-    start = time.time()
-    while (time.time() - start) <= 20:
-        data = cli.recv(1024)
-        print("Data received {}".format(data[-17:]))
-        if data[-15:] == "/topic/qos0data":
-            g_recv_topic = data[-15:][:11]
-            g_recv_data = data[-4:]
-            cli.send(send_qos0_msg)
-            data = cli.recv(1024)
-            print("data ack received {}".format(data))
-            break
-    cli.close()
+g_recv_topic=""
+g_broker_connected=0
 
 # The callback for when the client receives a CONNACK response from the server.
 def on_connect(client, userdata, flags, rc):
+    global g_broker_connected
     print("Connected with result code "+str(rc))
+    g_broker_connected = 1
     client.subscribe("/topic/qos0")
 
 # The callback for when a PUBLISH message is received from the server.
 def on_message(client, userdata, msg):
     global g_recv_topic
     global g_recv_data
-    if g_recv_data == "" and  msg.payload == "data":
+    payload = msg.payload.decode()
+    if g_recv_data == "" and  payload == "data":
         client.publish("/topic/qos0", "data_to_esp32")
         g_recv_topic = msg.topic
-        g_recv_data = msg.payload
-    print(msg.topic+" "+str(msg.payload))
+        g_recv_data = payload
+    print(msg.topic+" "+str(payload))
 
 # this is a test case write with tiny-test-fw.
 # to run test cases outside tiny-test-fw,
@@ -78,7 +40,7 @@ if test_fw_path and test_fw_path not in sys.path:
 
 import TinyFW
 import IDF
-
+import DUT
 
 
 
@@ -86,6 +48,8 @@ import IDF
 def test_examples_protocol_mqtt_wss(env, extra_data):
     global g_recv_topic
     global g_recv_data
+    global g_broker_connected
+    broker_url="iot.eclipse.org"
     """
     steps: |
       1. join AP and connects to wss broker
@@ -99,26 +63,34 @@ def test_examples_protocol_mqtt_wss(env, extra_data):
     bin_size = os.path.getsize(binary_file)
     IDF.log_performance("mqtt_websocket_secure_bin_size", "{}KB".format(bin_size//1024))
     IDF.check_performance("mqtt_websocket_secure_size", bin_size//1024)
-    # 1. start test
+    # 1. start test (and check the environment is healthy)
     dut1.start_app()
+    client = None
     # 2. Test connects to a broker
-    if use_mqtt_client_sketch:
-        mqtt_client_sketch()
-    else:
+    try:
+        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        print("Connected to AP with IP: {}".format(ip_address))
         client = mqtt.Client(transport="websockets")
         client.on_connect = on_connect
         client.on_message = on_message
         client.tls_set(None,
                     None,
                     None, cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLSv1, ciphers=None)
-        print "Connecting..."
-        client.connect("iot.eclipse.org", 443, 60)
-        print "...done"
-        print "Start Looping..."
-        start = time.time()
-        while (time.time() - start) <= 20:
-            client.loop()
-        print "...done"
+        print("Connecting...")
+        client.connect(broker_url, 443, 60)
+        print("...done")
+    except DUT.ExpectTimeout:
+        raise ValueError('ENV_TEST_FAILURE: Cannot connect to AP')
+    except:
+        print("ENV_TEST_FAILURE: Unexpected error while connecting to broker {}: {}:".format(broker_url, sys.exc_info()[0]))
+        raise
+    print("Start Looping...")
+    start = time.time()
+    while (time.time() - start) <= 20:
+        client.loop()
+    print("...done")
+    if g_broker_connected == 0:
+        raise ValueError('ENV_TEST_FAILURE: Test script cannot connect to broker: {}'.format(broker_url))
     # 3. check the message received back from the server
     if g_recv_topic == "/topic/qos0" and g_recv_data == "data" :
         print("PASS: Received correct message")

examples/storage/nvs_rw_blob/README.md

@@ -1,14 +1,79 @@
 # Non-Volatile Storage (NVS) Read and Write Example
 
-Demonstrates how to read and write a single integer value and a blob (binary large object) using NVS to preserve them between ESP32 module restarts.
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
+This example demonstrates how to read and write a single integer value and a blob (binary large object) using NVS to preserve them between ESP32 module restarts.
 
   * value - tracks number of ESP32 module soft and hard restarts.
   * blob - contains a table with module run times. The table is read from NVS to dynamically allocated RAM. New run time is added to the table on each manually triggered soft restart and written back to NVS. Triggering is done by pulling down GPIO0.
 
 Example also shows how to implement diagnostics if read / write operation was successful. 
 
-If not done already, consider checking simpler example *storage/nvs_rw_value*, that has been used as a starting point for preparing this one. 
-
 Detailed functional description of NVS and API is provided in [documentation](https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/storage/nvs_flash.html).
 
-See the README.md file in the upper level 'examples' directory for more information about examples.
+If not done already, consider checking simpler example *storage/nvs_rw_value*, that has been used as a starting point for preparing this one. 
+
+## How to use example
+
+### Hardware required
+
+This example can be run on most common development boards which have an active button connected to GPIO0. On most boards, this button is labeled as "Boot". When pressed, the button connects GPIO0 to ground.
+
+### Configure the project
+
+If using Make based build system, run `make menuconfig` and set serial port under Serial Flasher Options.
+
+If using CMake based build system, no configuration is required.
+
+### Build and flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+Or, for CMake based build system (replace PORT with serial port name):
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example Output
+
+First run:
+```
+Restart counter = 0
+Run time:
+Nothing saved yet!
+```
+
+At this point, press "Boot" button and hold it for a second. The board will perform software restart, printing:
+
+```
+Restarting...
+```
+
+After booting again, restart counter and run time array will be printed:
+
+```
+Restart counter = 1
+Run time:
+1: 5110
+```
+
+After pressing "Boot" once more:
+
+```
+Restart counter = 2
+Run time:
+1: 5110
+2: 5860
+```
+
+To reset the counter and run time array, erase the contents of flash memory using `make erase_flash` (or `idf.py erase_flash`, if using CMake build system), then upload the program again as described above.
+

examples/storage/nvs_rw_value/README.md

@@ -1,13 +1,94 @@
 # Non-Volatile Storage (NVS) Read and Write Example
 
-Demonstrates how to read and write a single integer value using NVS.
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
 
-The value holds the number of ESP32 module restarts. Since it is written to NVS, the value is preserved between restarts.
+This example demonstrates how to read and write a single integer value using NVS.
+
+In this example, value which is saved holds the number of ESP32 module restarts. Since it is written to NVS, the value is preserved between restarts.
 
 Example also shows how to check if read / write operation was successful, or certain value is not initialized in NVS. Diagnostic is provided in plain text to help track program flow and capture any issues on the way.
 
-Check another example *storage/nvs_rw_blob*, that shows how to read and write variable length binary data (blob).
-
 Detailed functional description of NVS and API is provided in [documentation](https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/storage/nvs_flash.html).
 
-See the README.md file in the upper level 'examples' directory for more information about examples.
+Check another example *storage/nvs_rw_blob*, which shows how to read and write variable length binary data (blob).
+
+## How to use example
+
+### Hardware required
+
+This example does not require any special hardware, and can be run on any common development board.
+
+### Configure the project
+
+If using Make based build system, run `make menuconfig` and set serial port under Serial Flasher Options.
+
+If using CMake based build system, no configuration is required.
+
+### Build and flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+Or, for CMake based build system (replace PORT with serial port name):
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
+## Example Output
+
+First run:
+```
+Opening Non-Volatile Storage (NVS) handle... Done
+Reading restart counter from NVS ... The value is not initialized yet!
+Updating restart counter in NVS ... Done
+Committing updates in NVS ... Done
+
+Restarting in 10 seconds...
+Restarting in 9 seconds...
+Restarting in 8 seconds...
+Restarting in 7 seconds...
+Restarting in 6 seconds...
+Restarting in 5 seconds...
+Restarting in 4 seconds...
+Restarting in 3 seconds...
+Restarting in 2 seconds...
+Restarting in 1 seconds...
+Restarting in 0 seconds...
+Restarting now.
+```
+
+Subsequent runs:
+
+```
+Opening Non-Volatile Storage (NVS) handle... Done
+Reading restart counter from NVS ... Done
+Restart counter = 1
+Updating restart counter in NVS ... Done
+Committing updates in NVS ... Done
+
+Restarting in 10 seconds...
+Restarting in 9 seconds...
+Restarting in 8 seconds...
+Restarting in 7 seconds...
+Restarting in 6 seconds...
+Restarting in 5 seconds...
+Restarting in 4 seconds...
+Restarting in 3 seconds...
+Restarting in 2 seconds...
+Restarting in 1 seconds...
+Restarting in 0 seconds...
+Restarting now.
+```
+
+Restart counter will increment on each run.
+
+To reset the counter, erase the contents of flash memory using `make erase_flash` (or `idf.py erase_flash`, if using CMake build system), then upload the program again as described above.
+

examples/storage/sd_card/README.md

@@ -1,5 +1,7 @@
 # SD Card example
 
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
 This example demonstrates how to use an SD card with ESP32. Example does the following steps:
 
 1. Use an "all-in-one" `esp_vfs_fat_sdmmc_mount` function to:
@@ -16,15 +18,17 @@ This example support SD (SDSC, SDHC, SDXC) cards and eMMC chips.
 
 ## Hardware
 
-To run this example, ESP32 development board needs to be connected to SD card as follows:
+This example runs on ESP-WROVER-KIT boards without any extra modifications required, only the SD card needs to be inserted into the slot.
+
+Other ESP32 development boards need to be connected to SD card as follows:
 
 ESP32 pin     | SD card pin | SPI pin | Notes
 --------------|-------------|---------|------------
 GPIO14 (MTMS) | CLK         | SCK     | 10k pullup in SD mode
 GPIO15 (MTDO) | CMD         | MOSI    | 10k pullup, both in SD and SPI modes
-GPIO2         | D0          | MISO    | 10k pullup in SD mode, pull low to go into download mode (see note below!)
+GPIO2         | D0          | MISO    | 10k pullup in SD mode, pull low to go into download mode (see Note about GPIO2 below!)
 GPIO4         | D1          | N/C     | not used in 1-line SD mode; 10k pullup in 4-line SD mode
-GPIO12 (MTDI) | D2          | N/C     | not used in 1-line SD mode; 10k pullup in 4-line SD mode (see note below!)
+GPIO12 (MTDI) | D2          | N/C     | not used in 1-line SD mode; 10k pullup in 4-line SD mode (see Note about GPIO12 below!)
 GPIO13 (MTCK) | D3          | CS      | not used in 1-line SD mode, but card's D3 pin must have a 10k pullup
 N/C           | CD          |         | optional, not used in the example
 N/C           | WP          |         | optional, not used in the example
@@ -32,7 +36,7 @@ N/C           | WP          |         | optional, not used in the example
 This example doesn't utilize card detect (CD) and write protect (WP) signals from SD card slot.
 
 With the given pinout for SPI mode, same connections between the SD card and ESP32 can be used to test both SD and SPI modes, provided that the appropriate pullups are in place. 
-See document `sd_pullup_requirements.rst` in `docs/en/api-reference/peripherals/` for more details about pullup support and compatiblities about modules and devkits. 
+See [the document about pullup requirements](https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/peripherals/sd_pullup_requirements.html) for more details about pullup support and compatibility of modules and development boards.
 
 In SPI mode, pins can be customized. See the initialization of ``sdspi_slot_config_t`` structure in the example code.
 
@@ -62,7 +66,17 @@ This command will burn the `XPD_SDIO_TIEH`, `XPD_SDIO_FORCE`, and `XPD_SDIO_REG`
 
 `espefuse.py` has a `--do-not-confirm` option if running from an automated flashing script.
 
-## 4-line and 1-line modes
+## How to use example
+
+### Configure the project
+
+If using Make based build system, run `make menuconfig` and set serial port under Serial Flasher Options.
+
+If using CMake based build system, no configuration is required.
+
+SD card can be used in various modes. See below on choosing the mode to be used.
+
+### 4-line and 1-line SD modes
 
 By default, example code uses the following initializer for SDMMC slot configuration:
 
@@ -74,30 +88,76 @@ Among other things, this sets `slot_config.width = 0`, which means that SD/MMC d
 
 Note that even if card's D3 line is not connected to the ESP32, it still has to be pulled up, otherwise the card will go into SPI protocol mode.
 
-## SPI mode
+### SPI mode
+
+By default, the example uses SDMMC Host peripheral to access the card using SD protocol. To use SPI peripheral instead (and SPI protocol), uncomment ``#define USE_SPI_MODE`` line in the example code.
+
+### Build and flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+Or, for CMake based build system (replace PORT with serial port name):
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
 
-By default, the example uses SDMMC Host peripheral to access SD card. To use SPI peripheral instead, uncomment ``#define USE_SPI_MODE`` in the example code.
 
 ## Example output
 
 Here is an example console output. In this case a 128MB SDSC card was connected, and `format_if_mount_failed` parameter was set to `true` in the source code. Card was unformatted, so the initial mount has failed. Card was then partitioned, formatted, and mounted again.
 
 ```
-I (1776) example: Initializing SD card
-W (1856) vfs_fat_sdmmc: failed to mount card (13)
-W (1856) vfs_fat_sdmmc: partitioning card
-W (1856) vfs_fat_sdmmc: formatting card
-W (2726) vfs_fat_sdmmc: mounting again
-I (2736) example: Card info:
-I (2736) example: Name: SU128
-I (2736) example: Type: SDSC
-I (2736) example: Capacity: 120 MB
-I (2736) example: Max clock speed: 25 MHz
-I (2736) example: Opening file
-I (2756) example: File written
-I (2756) example: Renaming file
-I (2756) example: Reading file
-I (2756) example: Read from file: 'Hello SU128!'
-I (2756) example: Card unmounted
+I (336) example: Initializing SD card
+I (336) example: Using SDMMC peripheral
+I (336) gpio: GPIO[13]| InputEn: 0| OutputEn: 1| OpenDrain: 0| Pullup: 0| Pulldown: 0| Intr:0 
+W (596) vfs_fat_sdmmc: failed to mount card (13)
+W (596) vfs_fat_sdmmc: partitioning card
+W (596) vfs_fat_sdmmc: formatting card, allocation unit size=16384
+W (7386) vfs_fat_sdmmc: mounting again
+Name: XA0E5
+Type: SDHC/SDXC
+Speed: 20 MHz
+Size: 61068MB
+I (7386) example: Opening file
+I (7396) example: File written
+I (7396) example: Renaming file
+I (7396) example: Reading file
+I (7396) example: Read from file: 'Hello XA0E5!'
+I (7396) example: Card unmounted
 ```
 
+## Troubleshooting
+
+### Failure to upload the example
+
+```
+Connecting........_____....._____....._____....._____....._____....._____....._____
+
+A fatal error occurred: Failed to connect to Espressif device: Invalid head of packet (0x34)
+```
+
+Disconnect the SD card D0/MISO line from GPIO2 and try uploading again. Read "Note about GPIO2" above.
+
+### Card fails to initialize with `sdmmc_init_sd_scr: send_scr (1) returned 0x107` error
+
+Check connections between the card and the ESP32. For example, if you have disconnected GPIO2 to work around the flashing issue, connect it back and reset the ESP32 (using a button on the development board, or by pressing Ctrl-T Ctrl-R in IDF Monitor).
+
+### Card fails to initialize with `sdmmc_check_scr: send_scr returned 0xffffffff` error
+
+Connections between the card and the ESP32 are too long for the frequency used. Try using shorter connections, or try reducing the clock speed of SD interface.
+
+### Failure to mount filesystem
+
+```
+example: Failed to mount filesystem. If you want the card to be formatted, set format_if_mount_failed = true.
+```
+The example will be able to mount only cards formatted using FAT32 filesystem. If the card is formatted as exFAT or some other filesystem, you have an option to format it in the example code. Modify `format_if_mount_failed = false` to `format_if_mount_failed = true` in the example code, then build and flash the example.

examples/storage/spiffs/README.md

@@ -1,5 +1,7 @@
 # SPIFFS example
 
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
 This example demonstrates how to use SPIFFS with ESP32. Example does the following steps:
 
 1. Use an "all-in-one" `esp_vfs_spiffs_register` function to:
@@ -10,18 +12,52 @@ This example demonstrates how to use SPIFFS with ESP32. Example does the followi
 3. Rename the file. Before renaming, check if destination file already exists using `stat` function, and remove it using `unlink` function.
 4. Open renamed file for reading, read back the line, and print it to the terminal.
 
+SPIFFS partition size is set in partitions_example.csv file. See [Partition Tables](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html) documentation for more information.
+
+## How to use example
+
+### Hardware required
+
+This example does not require any special hardware, and can be run on any common development board.
+
+### Configure the project
+
+If using Make based build system, run `make menuconfig` and set serial port under Serial Flasher Options.
+
+If using CMake based build system, no configuration is required.
+
+### Build and flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+Or, for CMake based build system (replace PORT with serial port name):
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
 ## Example output
 
 Here is an example console output. In this case `format_if_mount_failed` parameter was set to `true` in the source code. SPIFFS was unformatted, so the initial mount has failed. SPIFFS was then formatted, and mounted again.
 
 ```
-I (195) example: Initializing SPIFFS
-E (195) SPIFFS: mount failed, -10025. formatting...
-I (4525) example: Opening file
-I (4635) example: File written
-I (4685) example: Renaming file
-I (4735) example: Reading file
-I (4735) example: Read from file: 'Hello World!'
-I (4735) example: SPIFFS unmounted
+I (324) example: Initializing SPIFFS
+W (324) SPIFFS: mount failed, -10025. formatting...
+I (19414) example: Partition size: total: 896321, used: 0
+I (19414) example: Opening file
+I (19504) example: File written
+I (19544) example: Renaming file
+I (19584) example: Reading file
+I (19584) example: Read from file: 'Hello World!'
+I (19584) example: SPIFFS unmounted
 ```
 
+To erase the contents of SPIFFS partition, run `make erase_flash` command (or `idf.py erase_flash`, if using CMake build system). Then upload the example again as described above.

examples/storage/wear_levelling/README.md

@@ -1,5 +1,7 @@
 # Wear levelling example
 
+(See the README.md file in the upper level 'examples' directory for more information about examples.)
+
 This example demonstrates how to use wear levelling library and FATFS library to store files in a partition inside SPI flash. Example does the following steps:
 
 1. Use an "all-in-one" `esp_vfs_fat_spiflash_mount` function to:
@@ -10,6 +12,38 @@ This example demonstrates how to use wear levelling library and FATFS library to
 2. Create a file using `fopen` and write to it using `fprintf`.
 3. Open file for reading, read back the line, and print it to the terminal.
 
+Wear levelling partition size is set in partitions_example.csv file. See [Partition Tables](https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html) documentation for more information.
+
+## How to use example
+
+### Hardware required
+
+This example does not require any special hardware, and can be run on any common development board.
+
+### Configure the project
+
+If using Make based build system, run `make menuconfig` and set serial port under Serial Flasher Options.
+
+If using CMake based build system, no configuration is required.
+
+### Build and flash
+
+Build the project and flash it to the board, then run monitor tool to view serial output:
+
+```
+make -j4 flash monitor
+```
+
+Or, for CMake based build system (replace PORT with serial port name):
+
+```
+idf.py -p PORT flash monitor
+```
+
+(To exit the serial monitor, type ``Ctrl-]``.)
+
+See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects.
+
 ## Example output
 
 Here is an typical example console output. 
@@ -26,3 +60,5 @@ I (920) example: Read from file: 'written using ESP-IDF v3.1-dev-171-gf9ad17eee-
 I (920) example: Unmounting FAT filesystem
 I (1000) example: Done
 ```
+
+To erase the contents of wear levelling partition, run `make erase_flash` command (or `idf.py erase_flash`, if using CMake build system). Then upload the example again as described above.

examples/wifi/iperf/iperf_test.py

@@ -483,24 +483,19 @@ def build_iperf_with_config(config_name):
         os.chdir(cwd)
 
 
-def get_configs(env):
-    att_port = env.get_variable("attenuator_port")
-    ap_list = env.get_variable("ap_list")
-    pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"]
-    apc_ip = env.get_variable("apc_ip")
-    pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
-    return att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file
-
-
-@IDF.idf_example_test(env_tag="Example_ShieldBox", category="stress")
+@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic", category="stress")
 def test_wifi_throughput_with_different_configs(env, extra_data):
     """
     steps: |
       1. build iperf with specified configs
       2. test throughput for all routers
     """
-    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
-    ap_info = ap_list[0]
+    pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"]
+    pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
+    ap_info = {
+        "ssid": env.get_variable("ap_ssid"),
+        "password": env.get_variable("ap_password"),
+    }
 
     config_names_raw = subprocess.check_output(["ls", os.path.dirname(os.path.abspath(__file__))])
 
@@ -529,14 +524,6 @@ def test_wifi_throughput_with_different_configs(env, extra_data):
         test_utility = IperfTestUtility(dut, config_name, ap_info["ssid"],
                                         ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result[config_name])
 
-        PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF")
-        PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"})
-        assert Attenuator.set_att(att_port, 0) is True
-
-        if not test_utility.wait_ap_power_on():
-            Utility.console_log("[{}] failed to power on, skip testing this AP"
-                                .format(ap_info["ssid"]), color="red")
-
         for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
             test_utility.run_all_cases(0)
 
@@ -563,8 +550,10 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
       3. set attenuator value from 0-60 for each router
       4. test TCP tx rx and UDP tx rx throughput
     """
-    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
-
+    att_port = env.get_variable("attenuator_port")
+    ap_list = env.get_variable("ap_list")
+    pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"]
+    apc_ip = env.get_variable("apc_ip")
     pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
 
     test_result = {
@@ -609,15 +598,19 @@ def test_wifi_throughput_vs_rssi(env, extra_data):
     report.generate_report()
 
 
-@IDF.idf_example_test(env_tag="Example_ShieldBox")
+@IDF.idf_example_test(env_tag="Example_ShieldBox_Basic")
 def test_wifi_throughput_basic(env, extra_data):
     """
     steps: |
       1. test TCP tx rx and UDP tx rx throughput
       2. compare with the pre-defined pass standard
     """
-    att_port, ap_list, pc_nic_ip, apc_ip, pc_iperf_log_file = get_configs(env)
-    ap_info = ap_list[0]
+    pc_nic_ip = env.get_pc_nic_info("pc_nic", "ipv4")["addr"]
+    pc_iperf_log_file = os.path.join(env.log_path, "pc_iperf_log.md")
+    ap_info = {
+        "ssid": env.get_variable("ap_ssid"),
+        "password": env.get_variable("ap_password"),
+    }
 
     # 1. build iperf with best config
     build_iperf_with_config(BEST_PERFORMANCE_CONFIG)
@@ -638,14 +631,6 @@ def test_wifi_throughput_basic(env, extra_data):
     test_utility = IperfTestUtility(dut, BEST_PERFORMANCE_CONFIG, ap_info["ssid"],
                                     ap_info["password"], pc_nic_ip, pc_iperf_log_file, test_result)
 
-    PowerControl.Control.control_rest(apc_ip, ap_info["outlet"], "OFF")
-    PowerControl.Control.control(apc_ip, {ap_info["outlet"]: "ON"})
-    assert Attenuator.set_att(att_port, 0) is True
-
-    if not test_utility.wait_ap_power_on():
-        Utility.console_log("[{}] failed to power on, skip testing this AP"
-                            .format(ap_info["ssid"]), color="red")
-
     # 4. run test for TCP Tx, Rx and UDP Tx, Rx
     for _ in range(RETRY_COUNT_FOR_BEST_PERFORMANCE):
         test_utility.run_all_cases(0)

tools/ci/build_examples.sh

@@ -75,7 +75,8 @@ else
     [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
 
     # parse number 'NUM' at the end of string 'some_your_text_NUM'
-    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_([0-9]+)$/\1/p' )
+    # NOTE: Getting rid of the leading zero to get the decimal
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
     [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
 
     # count number of the jobs
@@ -120,14 +121,19 @@ build_example () {
         local BUILDLOG=${LOG_PATH}/ex_${ID}_log.txt
         touch ${BUILDLOG}
 
+        local FLASH_ARGS=build/download.config
+
         make clean >>${BUILDLOG} 2>&1 &&
         make defconfig >>${BUILDLOG} 2>&1 &&
         make all >>${BUILDLOG} 2>&1 &&
-        ( make print_flash_cmd | tail -n 1 >build/download.config ) >>${BUILDLOG} 2>&1 ||
+        make print_flash_cmd >${FLASH_ARGS}.full 2>>${BUILDLOG} ||
         {
             RESULT=$?; FAILED_EXAMPLES+=" ${EXAMPLE_NAME}" ;
         }
 
+        tail -n 1 ${FLASH_ARGS}.full > ${FLASH_ARGS} || :
+        test -s ${FLASH_ARGS} || die "Error: ${FLASH_ARGS} file is empty"
+
         cat ${BUILDLOG}
     popd
 

tools/ci/build_examples_cmake.sh

@@ -80,7 +80,8 @@ else
     [ -z ${JOB_PATTERN} ] && die "JOB_PATTERN is bad"
 
     # parse number 'NUM' at the end of string 'some_your_text_NUM'
-    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_([0-9]+)$/\1/p' )
+    # NOTE: Getting rid of the leading zero to get the decimal
+    JOB_NUM=$( echo ${JOB_NAME} | sed -n -r 's/^.*_0*([0-9]+)$/\1/p' )
     [ -z ${JOB_NUM} ] && die "JOB_NUM is bad"
 
     # count number of the jobs

tools/ci/setup_python.sh

@@ -1,7 +1,9 @@
 #! /bin/bash
 
 # Regexp for matching job names which are incompatible with Python 3
-py3_incomp='assign_test|UT|IT'
+# - assign_test, nvs_compatible_test, IT - auto_test_script causes the incompatibility
+# - UT_009_ - RS485 multi-device test is not started properly
+py3_incomp='assign_test|nvs_compatible_test|IT|UT_009_'
 
 if [ -z ${PYTHON_VER+x} ] || [[ $CI_JOB_NAME =~ $py3_incomp ]]; then
     # Use this version of the Python interpreter if it was not defined before or

tools/tiny-test-fw/IDF/IDFDUT.py

@@ -174,7 +174,11 @@ class IDFDUT(DUT.SerialDUT):
             # Filter out invalid port by our own will be much simpler.
             return [x for x in ports if not cls.INVALID_PORT_PATTERN.search(x)]
 
-        port_hint = espport.decode('utf8')
+        # On MacOs with python3.6: type of espport is already utf8
+        if type(espport) is type(u''):
+            port_hint = espport
+        else:
+            port_hint = espport.decode('utf8')
 
         # If $ESPPORT is a valid port, make it appear first in the list
         if port_hint in ports:

tools/unit-test-app/configs/aes_no_hw

@@ -0,0 +1,3 @@
+TEST_EXCLUDE_COMPONENTS=libsodium bt app_update
+TEST_COMPONENTS=mbedtls
+CONFIG_MBEDTLS_HARDWARE_AES=n

tools/unit-test-app/idf_ext.py

@@ -141,12 +141,12 @@ def add_action_extensions(base_functions, base_actions):
                 # config folder to build config
                 sdkconfig_default = os.path.join(PROJECT_PATH, "sdkconfig.defaults")
 
-                with open(sdkconfig_default, "r") as sdkconfig_default_file:
+                with open(sdkconfig_default, "rb") as sdkconfig_default_file:
                     sdkconfig_temp.write(sdkconfig_default_file.read())
 
                 sdkconfig_config = os.path.join(PROJECT_PATH, "configs", config_name)
-                with open(sdkconfig_config, "r") as sdkconfig_config_file:
-                    sdkconfig_temp.write("\n")
+                with open(sdkconfig_config, "rb") as sdkconfig_config_file:
+                    sdkconfig_temp.write(b"\n")
                     sdkconfig_temp.write(sdkconfig_config_file.read())
 
                 sdkconfig_temp.flush()
@@ -276,4 +276,4 @@ idf.py ut-apply-config-NAME - Generates configuration based on configs/NAME in s
 
     extensions["ut-help"] = (ut_help, [], [])
 
-    base_actions.update(extensions)
+    base_actions.update(extensions)

tools/unit-test-app/unit_test.py

@@ -53,9 +53,12 @@ SIMPLE_TEST_ID = 0
 MULTI_STAGE_ID = 1
 MULTI_DEVICE_ID = 2
 
-STARTUP_TIMEOUT=10
 DEFAULT_TIMEOUT=20
 
+DUT_STARTUP_CHECK_RETRY_COUNT = 5
+TEST_HISTROY_CHECK_TIMEOUT = 1
+
+
 def format_test_case_config(test_case_data):
     """
     convert the test case data to unified format.
@@ -139,19 +142,22 @@ def replace_app_bin(dut, name, new_app_bin):
             Utility.console_log("The replaced application binary is {}".format(new_app_bin), "O")
             break
 
-
 def reset_dut(dut):
-    # We do flush before test, in case we already have bootup pattern in data cache
-    dut.write("", flush=True)
     dut.reset()
     # esptool ``run`` cmd takes quite long time.
     # before reset finish, serial port is closed. therefore DUT could already bootup before serial port opened.
     # this could cause checking bootup print failed.
-    # now we input cmd `-`, and check either bootup print or test history,
-    # to determine if DUT is ready to test.
-    dut.write("-", flush=False)
-    dut.expect_any(UT_APP_BOOT_UP_DONE,
-                   "0 Tests 0 Failures 0 Ignored", timeout=STARTUP_TIMEOUT)
+    # now use input cmd `-` and check test history to check if DUT is bootup.
+    # we'll retry this step for a few times in case `dut.reset` returns during DUT bootup (when DUT can't process any command).
+    for _ in range(DUT_STARTUP_CHECK_RETRY_COUNT):
+        dut.write("-")
+        try:
+            dut.expect("0 Tests 0 Failures 0 Ignored", timeout=TEST_HISTROY_CHECK_TIMEOUT)
+            break
+        except ExpectTimeout:
+            pass
+    else:
+        raise AssertationError("Reset {} ({}) failed!".format(dut.name, dut.port))
 
 
 @IDF.idf_unit_test(env_tag="UT_T1_1")