From cb2e1f93d1321376890feab6fddeefd7c8f65d3b Mon Sep 17 00:00:00 2001 From: baohongde Date: Fri, 6 Sep 2019 12:00:54 +0800 Subject: [PATCH 01/30] components/bt: Fix sending LMP_features_req_ext in a loop Remove xtensa/xtruntime.h at the same time --- components/bt/controller/lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/bt/controller/lib b/components/bt/controller/lib index d122b0802..484307604 160000 --- a/components/bt/controller/lib +++ b/components/bt/controller/lib @@ -1 +1 @@ -Subproject commit d122b080242fdf045bd5a8ba8b5879f2f9c7885e +Subproject commit 484307604fad19b82afca4ff2e57fda63158c2b4 From d885df946aaabe105792a93ef6185bcc08a41523 Mon Sep 17 00:00:00 2001 From: zhangyanjiao Date: Wed, 11 Sep 2019 10:42:16 +0800 Subject: [PATCH 02/30] fix the bug that ESP32 sends broadcast to phone after smartconfig done --- components/esp_wifi/src/smartconfig_ack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/esp_wifi/src/smartconfig_ack.c b/components/esp_wifi/src/smartconfig_ack.c index fae5e58bb..3cda1a21c 100644 --- a/components/esp_wifi/src/smartconfig_ack.c +++ b/components/esp_wifi/src/smartconfig_ack.c @@ -85,7 +85,7 @@ static void sc_ack_send_task(void *pvParameters) bzero(&server_addr, sizeof(struct sockaddr_in)); server_addr.sin_family = AF_INET; - server_addr.sin_addr.s_addr = inet_addr((const char*)remote_ip); + memcpy(&server_addr.sin_addr.s_addr, remote_ip, sizeof(remote_ip)); server_addr.sin_port = htons(remote_port); esp_wifi_get_mac(WIFI_IF_STA, ack->ctx.mac); From d371605315efe3e707c877ce49b86bca6ff19340 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 17 Sep 2019 13:47:05 +1000 Subject: [PATCH 03/30] build: In GNU Make, use "cygpath -m" to avoid backslashes Fixes issues where backslashes in KCONFIG paths are later expanded as shell escape sequences. --- make/ldgen.mk | 8 ++++---- make/project_config.mk | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/make/ldgen.mk b/make/ldgen.mk index 8519a0133..01fd2ccb7 100644 --- a/make/ldgen.mk +++ b/make/ldgen.mk @@ -7,7 +7,7 @@ LDGEN_LIBRARIES=$(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(lib ifeq ($(OS),Windows_NT) define ldgen_process_template $(BUILD_DIR_BASE)/ldgen_libraries: $(LDGEN_LIBRARIES) $(IDF_PATH)/make/ldgen.mk - printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -w $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries + printf "$(foreach info,$(LDGEN_LIBRARIES),$(subst \,/,$(shell cygpath -m $(info)))\n)" > $(BUILD_DIR_BASE)/ldgen_libraries $(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_libraries @echo 'Generating $(notdir $(2))' @@ -18,8 +18,8 @@ $(2): $(1) $(LDGEN_FRAGMENT_FILES) $(SDKCONFIG) $(BUILD_DIR_BASE)/ldgen_librarie --libraries-file $(BUILD_DIR_BASE)/ldgen_libraries \ --output $(2) \ --kconfig $(IDF_PATH)/Kconfig \ - --env "COMPONENT_KCONFIGS=$(foreach k, $(COMPONENT_KCONFIGS), $(shell cygpath -w $(k)))" \ - --env "COMPONENT_KCONFIGS_PROJBUILD=$(foreach k, $(COMPONENT_KCONFIGS_PROJBUILD), $(shell cygpath -w $(k)))" \ + --env "COMPONENT_KCONFIGS=$(foreach k, $(COMPONENT_KCONFIGS), $(shell cygpath -m $(k)))" \ + --env "COMPONENT_KCONFIGS_PROJBUILD=$(foreach k, $(COMPONENT_KCONFIGS_PROJBUILD), $(shell cygpath -m $(k)))" \ --env "IDF_CMAKE=n" \ --objdump $(OBJDUMP) endef @@ -47,4 +47,4 @@ endif # Windows_NT define ldgen_create_commands ldgen-clean: rm -f $(BUILD_DIR_BASE)/ldgen_libraries -endef \ No newline at end of file +endef diff --git a/make/project_config.mk b/make/project_config.mk index 1a4e74126..d5e8020dc 100644 --- a/make/project_config.mk +++ b/make/project_config.mk @@ -7,9 +7,9 @@ COMPONENT_SDKCONFIG_RENAMES := $(foreach component,$(COMPONENT_PATHS),$(wildcard ifeq ($(OS),Windows_NT) # kconfiglib requires Windows-style paths for kconfig files -COMPONENT_KCONFIGS := $(shell cygpath -w $(COMPONENT_KCONFIGS)) -COMPONENT_KCONFIGS_PROJBUILD := $(shell cygpath -w $(COMPONENT_KCONFIGS_PROJBUILD)) -COMPONENT_SDKCONFIG_RENAMES := $(shell cygpath -w $(COMPONENT_SDKCONFIG_RENAMES)) +COMPONENT_KCONFIGS := $(shell cygpath -m $(COMPONENT_KCONFIGS)) +COMPONENT_KCONFIGS_PROJBUILD := $(shell cygpath -m $(COMPONENT_KCONFIGS_PROJBUILD)) +COMPONENT_SDKCONFIG_RENAMES := $(shell cygpath -m $(COMPONENT_SDKCONFIG_RENAMES)) endif #For doing make menuconfig etc From f1b5c326f413beb6adc367861975fd94506a2056 Mon Sep 17 00:00:00 2001 From: Renz Christian Bagaporo Date: Tue, 17 Sep 2019 18:07:47 +0800 Subject: [PATCH 04/30] cxx: force pthread to appear later in link line --- components/cxx/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/cxx/CMakeLists.txt b/components/cxx/CMakeLists.txt index 4e81992ad..ff2d0e438 100644 --- a/components/cxx/CMakeLists.txt +++ b/components/cxx/CMakeLists.txt @@ -1,9 +1,17 @@ idf_component_register(SRCS "cxx_exception_stubs.cpp" - "cxx_guards.cpp") + "cxx_guards.cpp" + # Make sure that pthread is in component list + PRIV_REQUIRES pthread) target_link_libraries(${COMPONENT_LIB} PUBLIC stdc++ gcc) target_link_libraries(${COMPONENT_LIB} INTERFACE "-u __cxa_guard_dummy") +# Force pthread to also appear later than stdc++ in link line +add_library(stdcpp_pthread INTERFACE) +idf_component_get_property(pthread pthread COMPONENT_LIB) +target_link_libraries(stdcpp_pthread INTERFACE stdc++ $) +target_link_libraries(${COMPONENT_LIB} PUBLIC stdcpp_pthread) + if(NOT CONFIG_COMPILER_CXX_EXCEPTIONS) target_link_libraries(${COMPONENT_LIB} INTERFACE "-u __cxx_fatal_exception") endif() From 58577db086bd3a71553f6cf785742de9f1d0d40d Mon Sep 17 00:00:00 2001 From: suda-morris <362953310@qq.com> Date: Wed, 4 Sep 2019 15:36:50 +0800 Subject: [PATCH 05/30] gh_action: fix error on new pull request 1. Disable the broken pull request sync temporarily 2. move python lint from travis to github action --- .github/workflows/issue_comment.yml | 22 +++++++++++--------- .github/workflows/issues.yml | 22 +++++++++++--------- .github/workflows/pull_request.yml | 16 --------------- .github/workflows/python_lint.yml | 32 +++++++++++++++++++++++++++++ .travis.yml | 7 ------- 5 files changed, 56 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/python_lint.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/issue_comment.yml b/.github/workflows/issue_comment.yml index 16069f1f1..1e268bed9 100644 --- a/.github/workflows/issue_comment.yml +++ b/.github/workflows/issue_comment.yml @@ -1,16 +1,18 @@ +name: Sync issue comments to JIRA + on: issue_comment -name: Sync issue and PR comments to JIRA + jobs: syncToJIRA: name: Sync to JIRA runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Sync to JIRA - uses: espressif/github-actions/sync_issues_to_jira@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JIRA_PASS: ${{ secrets.JIRA_PASS }} - JIRA_PROJECT: IDFGH - JIRA_URL: ${{ secrets.JIRA_URL }} - JIRA_USER: ${{ secrets.JIRA_USER }} + - uses: actions/checkout@master + - name: Sync issue comments to JIRA + uses: espressif/github-actions/sync_issues_to_jira@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JIRA_PASS: ${{ secrets.JIRA_PASS }} + JIRA_PROJECT: IDFGH + JIRA_URL: ${{ secrets.JIRA_URL }} + JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml index 3ead59688..b4cc06110 100644 --- a/.github/workflows/issues.yml +++ b/.github/workflows/issues.yml @@ -1,16 +1,18 @@ -on: issues name: Sync issues to JIRA + +on: issues + jobs: syncToJIRA: name: Sync to JIRA runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Sync to JIRA - uses: espressif/github-actions/sync_issues_to_jira@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JIRA_PASS: ${{ secrets.JIRA_PASS }} - JIRA_PROJECT: IDFGH - JIRA_URL: ${{ secrets.JIRA_URL }} - JIRA_USER: ${{ secrets.JIRA_USER }} + - uses: actions/checkout@master + - name: Sync issues to JIRA project + uses: espressif/github-actions/sync_issues_to_jira@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JIRA_PASS: ${{ secrets.JIRA_PASS }} + JIRA_PROJECT: IDFGH + JIRA_URL: ${{ secrets.JIRA_URL }} + JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml deleted file mode 100644 index 324639e08..000000000 --- a/.github/workflows/pull_request.yml +++ /dev/null @@ -1,16 +0,0 @@ -on: pull_request -name: Sync PRs to JIRA -jobs: - syncToJIRA: - name: Sync to JIRA - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Sync to JIRA - uses: espressif/github-actions/sync_issues_to_jira@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JIRA_PASS: ${{ secrets.JIRA_PASS }} - JIRA_PROJECT: IDFGH - JIRA_URL: ${{ secrets.JIRA_URL }} - JIRA_USER: ${{ secrets.JIRA_USER }} diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 000000000..897ef55a9 --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,32 @@ +name: Python CI + +# This workflow will be triggered when a PR modifies some python relevant files +on: + pull_request: + paths: + - "*.py" + - "requirements.txt" + +jobs: + python_lint: + name: python lint + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7] + + steps: + - name: Checkout + uses: actions/checkout@master + - name: Set up Python environment + uses: actions/setup-python@master + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + flake8 . --config=.flake8 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 74902f9f3..000000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: python -sudo: false -python: - - "3.4" -script: - - pip install flake8 - - travis_wait 20 python -m flake8 --config=.flake8 . From 70d771eb2d88b558429b663d00ff1b59b6080e76 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 18 Sep 2019 15:07:47 +0800 Subject: [PATCH 06/30] esp_flash: put adapter to legacy functions into IRAM to be back-compatible --- components/spi_flash/esp_flash_api.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/spi_flash/esp_flash_api.c b/components/spi_flash/esp_flash_api.c index b062619dc..bb5be1bb8 100644 --- a/components/spi_flash/esp_flash_api.c +++ b/components/spi_flash/esp_flash_api.c @@ -634,7 +634,7 @@ esp_err_t esp_flash_app_disable_protect(bool disable) #ifndef CONFIG_SPI_FLASH_USE_LEGACY_IMPL -static esp_err_t spi_flash_translate_rc(esp_err_t err) +static IRAM_ATTR esp_err_t spi_flash_translate_rc(esp_err_t err) { switch (err) { case ESP_OK: @@ -657,19 +657,19 @@ static esp_err_t spi_flash_translate_rc(esp_err_t err) return ESP_OK; } -esp_err_t spi_flash_erase_range(uint32_t start_addr, uint32_t size) +esp_err_t IRAM_ATTR spi_flash_erase_range(uint32_t start_addr, uint32_t size) { esp_err_t err = esp_flash_erase_region(NULL, start_addr, size); return spi_flash_translate_rc(err); } -esp_err_t spi_flash_write(size_t dst, const void *srcv, size_t size) +esp_err_t IRAM_ATTR spi_flash_write(size_t dst, const void *srcv, size_t size) { esp_err_t err = esp_flash_write(NULL, srcv, dst, size); return spi_flash_translate_rc(err); } -esp_err_t spi_flash_read(size_t src, void *dstv, size_t size) +esp_err_t IRAM_ATTR spi_flash_read(size_t src, void *dstv, size_t size) { esp_err_t err = esp_flash_read(NULL, dstv, src, size); return spi_flash_translate_rc(err); From 36ba5bc5ee944a6b5dc9b6e0cd27398eed28753c Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 18 Sep 2019 17:54:10 +1000 Subject: [PATCH 07/30] cmake: Account for IDF_PATH case mismatches on case insensitive filesystems --- tools/cmake/idf.cmake | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index e97d83fcd..2d396c886 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -1,19 +1,24 @@ get_property(__idf_env_set GLOBAL PROPERTY __IDF_ENV_SET) if(NOT __idf_env_set) # Infer an IDF_PATH relative to the tools/cmake directory - get_filename_component(_idf_path "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) + get_filename_component(_idf_path "${CMAKE_CURRENT_LIST_DIR}/../.." REALPATH) file(TO_CMAKE_PATH "${_idf_path}" _idf_path) # Get the path set in environment set(idf_path $ENV{IDF_PATH}) - file(TO_CMAKE_PATH "${idf_path}" idf_path) # Environment IDF_PATH should match the inferred IDF_PATH. If not, warn the user. + # (Note: REALPATH is needed in both above steps to account for case on case + # insensitive filesystems, or relative paths) if(idf_path) + get_filename_component(idf_path "${idf_path}" REALPATH) + file(TO_CMAKE_PATH "${idf_path}" idf_path) + if(NOT idf_path STREQUAL _idf_path) message(WARNING "IDF_PATH environment variable is different from inferred IDF_PATH. Check if your project's top-level CMakeLists.txt includes the right - CMake files. Environment IDF_PATH will be used for the build.") + CMake files. Environment IDF_PATH will be used for the build: + ${idf_path}") endif() else() message(WARNING "IDF_PATH environment variable not found. Setting IDF_PATH to '${_idf_path}'.") @@ -43,4 +48,4 @@ if(NOT __idf_env_set) __build_init("${idf_path}") set_property(GLOBAL PROPERTY __IDF_ENV_SET 1) -endif() \ No newline at end of file +endif() From dc5b8a9a8f6a6960e5b48bd270a5146ed9243bcb Mon Sep 17 00:00:00 2001 From: suda-morris <362953310@qq.com> Date: Wed, 18 Sep 2019 18:28:18 +0800 Subject: [PATCH 08/30] ethernet: test apps can build with ethernet disabled --- tools/ci/test_build_system_cmake.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/ci/test_build_system_cmake.sh b/tools/ci/test_build_system_cmake.sh index 1f7b215a4..9a8424065 100755 --- a/tools/ci/test_build_system_cmake.sh +++ b/tools/ci/test_build_system_cmake.sh @@ -371,6 +371,19 @@ function run_tests() rm sdkconfig; rm sdkconfig.defaults; + print_status "can build with ethernet component disabled" + idf.py clean > /dev/null; + idf.py fullclean > /dev/null; + rm -f sdkconfig.defaults; + rm -f sdkconfig; + echo "CONFIG_ETH_USE_SPI_ETHERNET=" >> sdkconfig.defaults; + echo "CONFIG_ETH_USE_ESP32_EMAC=" >> sdkconfig.defaults; + idf.py reconfigure > /dev/null; + idf.py build || failure "Failed to build with ethernet component disabled" + assert_built ${APP_BINS} ${BOOTLOADER_BINS} ${PARTITION_BIN} + rm sdkconfig; + rm sdkconfig.defaults; + print_status "Building a project with CMake library imported and PSRAM workaround, all files compile with workaround" # Test for libraries compiled within ESP-IDF rm -rf build @@ -517,10 +530,10 @@ endmenu\n" >> ${IDF_PATH}/Kconfig; rm -rf CMakeLists.txt mv CMakeLists.txt.bak CMakeLists.txt rm -rf CMakeLists.txt.bak - + print_status "Print all required argument deprecation warnings" idf.py -C${IDF_PATH}/tools/test_idf_py --test-0=a --test-1=b --test-2=c --test-3=d test-0 --test-sub-0=sa --test-sub-1=sb ta test-1 > out.txt - ! grep -e '"test-0" is deprecated' -e '"test_0" is deprecated' out.txt || failure "Deprecation warnings are displayed for non-deprecated option/command" + ! grep -e '"test-0" is deprecated' -e '"test_0" is deprecated' out.txt || failure "Deprecation warnings are displayed for non-deprecated option/command" grep -e 'Warning: Option "test_sub_1" is deprecated and will be removed in future versions.' \ -e 'Warning: Command "test-1" is deprecated and will be removed in future versions. Please use alternative command.' \ -e 'Warning: Option "test_1" is deprecated and will be removed in future versions.' \ From 546ac64a9e98b56d7014d33bb98ee976f9dc7d38 Mon Sep 17 00:00:00 2001 From: suda-morris <362953310@qq.com> Date: Wed, 18 Sep 2019 18:28:55 +0800 Subject: [PATCH 09/30] ethernet: update spi-ethernet api --- components/esp_eth/include/esp_eth_mac.h | 28 +++++++++++++------ components/esp_eth/src/esp_eth_mac_dm9051.c | 14 +++++----- components/esp_eth/test/test_emac.c | 4 +-- .../protocol_examples_common/connect.c | 6 ++-- .../basic/main/ethernet_example_main.c | 6 ++-- .../eth2ap/main/ethernet_example_main.c | 6 ++-- examples/ethernet/iperf/main/cmd_ethernet.c | 6 ++-- 7 files changed, 41 insertions(+), 29 deletions(-) diff --git a/components/esp_eth/include/esp_eth_mac.h b/components/esp_eth/include/esp_eth_mac.h index 07507cefe..c189acb28 100644 --- a/components/esp_eth/include/esp_eth_mac.h +++ b/components/esp_eth/include/esp_eth_mac.h @@ -247,11 +247,6 @@ typedef struct { uint32_t sw_reset_timeout_ms; /*!< Software reset timeout value (Unit: ms) */ uint32_t rx_task_stack_size; /*!< Stack size of the receive task */ uint32_t rx_task_prio; /*!< Priority of the receive task */ - uint32_t queue_len; /*!< Length of the transaction queue */ -#if CONFIG_ETH_USE_SPI_ETHERNET - spi_device_handle_t spi_hdl; /*!< Handle of spi device */ -#endif - } eth_mac_config_t; /** @@ -263,7 +258,6 @@ typedef struct { .sw_reset_timeout_ms = 100, \ .rx_task_stack_size = 4096, \ .rx_task_prio = 15, \ - .queue_len = 100, \ } #if CONFIG_ETH_USE_ESP32_EMAC @@ -280,16 +274,34 @@ esp_eth_mac_t *esp_eth_mac_new_esp32(const eth_mac_config_t *config); #endif #if CONFIG_ETH_SPI_ETHERNET_DM9051 +/** + * @brief DM9051 specific configuration + * + */ +typedef struct { + spi_device_handle_t spi_hdl; /*!< Handle of SPI device driver */ +} eth_dm9051_config_t; + +/** + * @brief Default DM9051 specific configuration + * + */ +#define ETH_DM9051_DEFAULT_CONFIG(spi_device) \ + { \ + .spi_hdl = spi_device, \ + } + /** * @brief Create DM9051 Ethernet MAC instance * -* @param config: Ethernet MAC configuration +* @param dm9051_config: DM9051 specific configuration +* @param mac_config: Ethernet MAC configuration * * @return * - instance: create MAC instance successfully * - NULL: create MAC instance failed because some error occurred */ -esp_eth_mac_t *esp_eth_mac_new_dm9051(const eth_mac_config_t *config); +esp_eth_mac_t *esp_eth_mac_new_dm9051(const eth_dm9051_config_t *dm9051_config, const eth_mac_config_t *mac_config); #endif #ifdef __cplusplus } diff --git a/components/esp_eth/src/esp_eth_mac_dm9051.c b/components/esp_eth/src/esp_eth_mac_dm9051.c index c2c4c1095..5fb9456cc 100644 --- a/components/esp_eth/src/esp_eth_mac_dm9051.c +++ b/components/esp_eth/src/esp_eth_mac_dm9051.c @@ -815,16 +815,16 @@ static esp_err_t emac_dm9051_del(esp_eth_mac_t *mac) return ESP_OK; } -esp_eth_mac_t *esp_eth_mac_new_dm9051(const eth_mac_config_t *config) +esp_eth_mac_t *esp_eth_mac_new_dm9051(const eth_dm9051_config_t *dm9051_config, const eth_mac_config_t *mac_config) { esp_eth_mac_t *ret = NULL; - MAC_CHECK(config, "can't set mac config to null", err, NULL); - MAC_CHECK(config->spi_hdl, "can't set spi handle to null", err, NULL); + MAC_CHECK(dm9051_config, "can't set dm9051 specific config to null", err, NULL); + MAC_CHECK(mac_config, "can't set mac config to null", err, NULL); emac_dm9051_t *emac = calloc(1, sizeof(emac_dm9051_t)); MAC_CHECK(emac, "calloc emac failed", err, NULL); /* bind methods and attributes */ - emac->sw_reset_timeout_ms = config->sw_reset_timeout_ms; - emac->spi_hdl = config->spi_hdl; + emac->sw_reset_timeout_ms = mac_config->sw_reset_timeout_ms; + emac->spi_hdl = dm9051_config->spi_hdl; emac->parent.set_mediator = emac_dm9051_set_mediator; emac->parent.init = emac_dm9051_init; emac->parent.deinit = emac_dm9051_deinit; @@ -843,8 +843,8 @@ esp_eth_mac_t *esp_eth_mac_new_dm9051(const eth_mac_config_t *config) emac->spi_lock = xSemaphoreCreateMutex(); MAC_CHECK(emac->spi_lock, "create lock failed", err_lock, NULL); /* create dm9051 task */ - BaseType_t xReturned = xTaskCreate(emac_dm9051_task, "dm9051_tsk", config->rx_task_stack_size, emac, - config->rx_task_prio, &emac->rx_task_hdl); + BaseType_t xReturned = xTaskCreate(emac_dm9051_task, "dm9051_tsk", mac_config->rx_task_stack_size, emac, + mac_config->rx_task_prio, &emac->rx_task_hdl); MAC_CHECK(xReturned == pdPASS, "create dm9051 task failed", err_tsk, NULL); return &(emac->parent); err_tsk: diff --git a/components/esp_eth/test/test_emac.c b/components/esp_eth/test/test_emac.c index 2ebe98b02..d38c3ac9c 100644 --- a/components/esp_eth/test/test_emac.c +++ b/components/esp_eth/test/test_emac.c @@ -119,8 +119,8 @@ TEST_CASE("dm9051 io test", "[ethernet][ignore]") TEST_ESP_OK(esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, ð_event_handler, NULL)); TEST_ESP_OK(esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, &got_ip_event_handler, NULL)); eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG(); - mac_config.spi_hdl = spi_handle; - esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&mac_config); + eth_dm9051_config_t dm9051_config = ETH_DM9051_DEFAULT_CONFIG(spi_handle); + esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&dm9051_config, &mac_config); eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); esp_eth_phy_t *phy = esp_eth_phy_new_dm9051(&phy_config); esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); diff --git a/examples/common_components/protocol_examples_common/connect.c b/examples/common_components/protocol_examples_common/connect.c index 8ee8aea9d..38d68a3a5 100644 --- a/examples/common_components/protocol_examples_common/connect.c +++ b/examples/common_components/protocol_examples_common/connect.c @@ -219,9 +219,9 @@ static void start(void) .queue_size = 20 }; ESP_ERROR_CHECK(spi_bus_add_device(CONFIG_EXAMPLE_ETH_SPI_HOST, &devcfg, &spi_handle)); - /* dm9051 ethernet driver is based on spi driver, so need to specify the spi handle */ - mac_config.spi_hdl = spi_handle; - s_mac = esp_eth_mac_new_dm9051(&mac_config); + /* dm9051 ethernet driver is based on spi driver */ + eth_dm9051_config_t dm9051_config = ETH_DM9051_DEFAULT_CONFIG(spi_handle); + s_mac = esp_eth_mac_new_dm9051(&dm9051_config, &mac_config); s_phy = esp_eth_phy_new_dm9051(&phy_config); #endif esp_eth_config_t config = ETH_DEFAULT_CONFIG(s_mac, s_phy); diff --git a/examples/ethernet/basic/main/ethernet_example_main.c b/examples/ethernet/basic/main/ethernet_example_main.c index a58bab806..2157a9132 100644 --- a/examples/ethernet/basic/main/ethernet_example_main.c +++ b/examples/ethernet/basic/main/ethernet_example_main.c @@ -104,9 +104,9 @@ void app_main(void) .queue_size = 20 }; ESP_ERROR_CHECK(spi_bus_add_device(CONFIG_EXAMPLE_ETH_SPI_HOST, &devcfg, &spi_handle)); - /* dm9051 ethernet driver is based on spi driver, so need to specify the spi handle */ - mac_config.spi_hdl = spi_handle; - esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&mac_config); + /* dm9051 ethernet driver is based on spi driver */ + eth_dm9051_config_t dm9051_config = ETH_DM9051_DEFAULT_CONFIG(spi_handle); + esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&dm9051_config, &mac_config); esp_eth_phy_t *phy = esp_eth_phy_new_dm9051(&phy_config); #endif esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); diff --git a/examples/ethernet/eth2ap/main/ethernet_example_main.c b/examples/ethernet/eth2ap/main/ethernet_example_main.c index b9b2193a2..f3df345f6 100644 --- a/examples/ethernet/eth2ap/main/ethernet_example_main.c +++ b/examples/ethernet/eth2ap/main/ethernet_example_main.c @@ -176,9 +176,9 @@ static void initialize_ethernet(void) .queue_size = 20 }; ESP_ERROR_CHECK(spi_bus_add_device(CONFIG_EXAMPLE_ETH_SPI_HOST, &devcfg, &spi_handle)); - /* dm9051 ethernet driver is based on spi driver, so need to specify the spi handle */ - mac_config.spi_hdl = spi_handle; - esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&mac_config); + /* dm9051 ethernet driver is based on spi driver */ + eth_dm9051_config_t dm9051_config = ETH_DM9051_DEFAULT_CONFIG(spi_handle); + esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&dm9051_config, &mac_config); esp_eth_phy_t *phy = esp_eth_phy_new_dm9051(&phy_config); #endif esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); diff --git a/examples/ethernet/iperf/main/cmd_ethernet.c b/examples/ethernet/iperf/main/cmd_ethernet.c index d148b5a28..075908b13 100644 --- a/examples/ethernet/iperf/main/cmd_ethernet.c +++ b/examples/ethernet/iperf/main/cmd_ethernet.c @@ -215,9 +215,9 @@ void register_ethernet(void) .queue_size = 20 }; ESP_ERROR_CHECK(spi_bus_add_device(CONFIG_EXAMPLE_ETH_SPI_HOST, &devcfg, &spi_handle)); - /* dm9051 ethernet driver is based on spi driver, so need to specify the spi handle */ - mac_config.spi_hdl = spi_handle; - esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&mac_config); + /* dm9051 ethernet driver is based on spi driver */ + eth_dm9051_config_t dm9051_config = ETH_DM9051_DEFAULT_CONFIG(spi_handle); + esp_eth_mac_t *mac = esp_eth_mac_new_dm9051(&dm9051_config, &mac_config); esp_eth_phy_t *phy = esp_eth_phy_new_dm9051(&phy_config); #endif esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); From 32adfdc307d367de338edd3f8942384f1c76e20a Mon Sep 17 00:00:00 2001 From: Wang Fang Date: Fri, 20 Sep 2019 12:32:55 +0800 Subject: [PATCH 10/30] Add Chinese translation for the 2nd batch of the files in storage folder in api-reference. 1. The translation should be first reviewed by technical reviewers; 2. and then by language reviewers. For the translation for the first batch of files, please see !MR5620 and !MR5613 --- components/nvs_flash/README.rst | 26 +- components/nvs_flash/README_CN.rst | 303 ++++++++++++++++++ components/spi_flash/README.rst | 2 + components/spi_flash/README_CN.rst | 188 +++++++++++ components/vfs/README.rst | 2 + components/vfs/README_CN.rst | 167 ++++++++++ .../zh_CN/api-reference/storage/nvs_flash.rst | 37 ++- .../zh_CN/api-reference/storage/spi_flash.rst | 48 ++- docs/zh_CN/api-reference/storage/vfs.rst | 16 +- 9 files changed, 774 insertions(+), 15 deletions(-) create mode 100644 components/nvs_flash/README_CN.rst create mode 100644 components/spi_flash/README_CN.rst create mode 100644 components/vfs/README_CN.rst diff --git a/components/nvs_flash/README.rst b/components/nvs_flash/README.rst index 4638b644b..6cbb56985 100644 --- a/components/nvs_flash/README.rst +++ b/components/nvs_flash/README.rst @@ -1,6 +1,8 @@ Non-volatile storage library ============================ +:link_to_translation:`zh_CN:[中文]` + Introduction ------------ @@ -144,7 +146,7 @@ Entry and entry state bitmap Each entry can be in one of the following three states represented with two bits in the entry state bitmap. The final four bits in the bitmap (256 - 2 * 126) are not used. Empty (2'b11) - Nothing is written into the specific entry yet. It is in an uninitialized state (all bytes are ``0xff``). + Nothing is written into the specific entry yet. It is in an uninitialized state (all bytes are ``0xff``). Written (2'b10) A key-value pair (or part of key-value pair which spans multiple entries) has been written into the entry. @@ -169,12 +171,12 @@ For values of primitive types (currently integers from 1 to 8 bytes long), entry Primitive +--------------------------------+ +--------> | Data (8) | | Types +--------------------------------+ - +-> Fixed length -- + +-> Fixed length -- | | +---------+--------------+---------------+-------+ | +--------> | Size(4) | ChunkCount(1)| ChunkStart(1) | Rsv(2)| Data format ---+ Blob Index +---------+--------------+---------------+-------+ | - | +----------+---------+-----------+ + | +----------+---------+-----------+ +-> Variable length --> | Size (2) | Rsv (2) | CRC32 (4) | (Strings, Blob Data) +----------+---------+-----------+ @@ -200,21 +202,21 @@ Key Zero-terminated ASCII string containing a key name. Maximum string length is 15 bytes, excluding a zero terminator. Data - For integer types, this field contains the value itself. If the value itself is shorter than 8 bytes, it is padded to the right, with unused bytes filled with ``0xff``. + For integer types, this field contains the value itself. If the value itself is shorter than 8 bytes, it is padded to the right, with unused bytes filled with ``0xff``. For "blob index" entry, these 8 bytes hold the following information about data-chunks: - Size (Only for blob index.) Size, in bytes, of complete blob data. - - ChunkCount - (Only for blob index.) Total number of blob-data chunks into which the blob was divided during storage. - - - ChunkStart - (Only for blob index.) ChunkIndex of the first blob-data chunk of this blob. Subsequent chunks have chunkIndex incrementally allocated (step of 1). + - ChunkCount + (Only for blob index.) Total number of blob-data chunks into which the blob was divided during storage. + + - ChunkStart + (Only for blob index.) ChunkIndex of the first blob-data chunk of this blob. Subsequent chunks have chunkIndex incrementally allocated (step of 1). For string and blob data chunks, these 8 bytes hold additional data about the value, which are described below: - + - Size (Only for strings and blobs.) Size, in bytes, of actual data. For strings, this includes zero terminators. @@ -227,7 +229,7 @@ Variable length values (strings and blobs) are written into subsequent entries, Namespaces ^^^^^^^^^^ -As mentioned above, each key-value pair belongs to one of the namespaces. Namespace identifiers (strings) are stored as keys of key-value pairs in namespace with index 0. Values corresponding to these keys are indexes of these namespaces. +As mentioned above, each key-value pair belongs to one of the namespaces. Namespace identifiers (strings) are stored as keys of key-value pairs in namespace with index 0. Values corresponding to these keys are indexes of these namespaces. :: @@ -261,7 +263,7 @@ Data stored in NVS partitions can be encrypted using AES-XTS in the manner simil NVS key partition ^^^^^^^^^^^^^^^^^ -An application requiring NVS encryption support needs to be compiled with a key-partition of the type `data` and subtype `key`. This partition should be marked as `encrypted`. Refer to :doc:`Partition Tables <../../api-guides/partition-tables>` for more details. The size of the partition should be 4096 bytes (minimum partition size). The structure of this partition is depicted below. +An application requiring NVS encryption support needs to be compiled with a key-partition of the type `data` and subtype `key`. This partition should be marked as `encrypted`. Refer to :doc:`Partition Tables <../../api-guides/partition-tables>` for more details. The size of the partition should be 4096 bytes (minimum partition size). The structure of this partition is depicted below. :: diff --git a/components/nvs_flash/README_CN.rst b/components/nvs_flash/README_CN.rst new file mode 100644 index 000000000..4025b9b5f --- /dev/null +++ b/components/nvs_flash/README_CN.rst @@ -0,0 +1,303 @@ +非易失性存储库 +============================ + +:link_to_translation:`en:[English]` + +简介 +------------ + +非易失性存储 (NVS) 库主要用于在 flash 中存储键值格式的数据。本文档将详细介绍 NVS 常用的一些概念。 + +底层存储 +^^^^^^^^^^^^^^^^^^ + +NVS 通过调用 ``spi_flash_{read|write|erase}`` API 对主 flash 的部分空间进行读、写、擦除操作，包括 ``data`` 类型和 ``nvs`` 子类型的所有分区。应用程序可调用 ``nvs_open`` API 选择使用带有 ``nvs`` 标签的分区，也可以通过调用 ``nvs_open_from_part`` API 选择使用指定名称的任意分区。 + +NVS 库后续版本可能会增加其他存储器后端，实现将数据保存至其他 flash 芯片（SPI 或 I2C 接口）、RTC 或 FRAM 中。 + +.. note:: 如果 NVS 分区被截断（例如，更改分区表布局时），则应擦除分区内容。可以使用 ESP-IDF 构建系统中的 ``idf.py erase_flash`` 命令擦除 flash 上的所有内容。 + +.. note:: NVS 最适合存储一些较小的数据，而非字符串或二进制大对象 (BLOB) 等较大的数据。如需存储较大的 BLOB 或者字符串，请考虑使用基于磨损均衡库的 FAT 文件系统。 + + +键值对 +^^^^^^^^^^^^^^^ + +NVS 的操作对象为键值对，其中键是 ASCII 字符串，当前支持最大键长为 15 个字符，值可以为以下几种类型： + +- 整数型：``uint8_t``、``int8_t``、``uint16_t``、``int16_t``、``uint32_t``、``int32_t``、``uint64_t`` 和 ``int64_t``； +- 以 ``\0`` 结尾的字符串； +- 可变长度的二进制数据 (BLOB) + +.. note:: + + 字符串值当前上限为 4000 字节，其中包括空终止符。BLOB 值上限为 508,000 字节或分区大小减去 4000 字节的 97.6%，以较低值为准。 + +后续可能会增加对 ``float`` 和 ``double`` 等其他类型数据的支持。 + +键必须唯一。为现有的键写入新的值可能产生如下结果： + +- 如果新旧值数据类型相同，则更新值； +- 如果新旧值数据类型不同，则返回错误。 + +读取值时也会执行数据类型检查。如果读取操作的数据类型与该值的数据类型不匹配，则返回错误。 + + +命名空间 +^^^^^^^^^^ + +为了减少不同组件之间键名的潜在冲突，NVS 将每个键值对分配给一个命名空间。命名空间的命名规则遵循键名的命名规则，即最多可占 15 个字符。命名空间的名称在调用 ``nvs_open`` 或 ``nvs_open_from_part`` 中指定，调用后将返回一个不透明句柄，用于后续调用 ``nvs_read_*``、``nvs_write_*`` 和 ``nvs_commit`` 函数。这样，一个句柄关联一个命名空间，键名便不会与其他命名空间中相同键名冲突。请注意，不同 NVS 分区中具有相同名称的命名空间将被视为不同的命名空间。 + + +安全性、篡改性及鲁棒性 +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +NVS 与 ESP32 flash 加密系统不直接兼容。但如果 NVS 加密与 ESP32 flash 加密一起使用时，数据仍可以加密形式存储。更多详情请参阅 :ref:`nvs_encryption`。 + +如果未启用 NVS 加密，任何对 flash 芯片有物理访问权限的人都可以修改、擦除或添加键值对。NVS 加密启用后，如果不知道相应的 NVS 加密密钥，则无法修改或添加键值对并将其识别为有效键值。但是，针对擦除操作没有相应的防篡改功能。 + +当 flash 处于不一致状态时，NVS 库会尝试恢复。在任何时间点关闭设备电源，然后重新打开电源，不会导致数据丢失；但如果关闭设备电源时正在写入新的键值对，这一键值对可能会丢失。该库还应当能对 flash 中的任意数据进行正确初始化。 + + +内部实现 +--------- + +键值对日志 +^^^^^^^^^^^^^^^^^^^^^^ + +NVS 按顺序存储键值对，新的键值对添加在最后。因此，如需更新某一键值对，实际是在日志最后增加一对新的键值对，同时将旧的键值对标记为已擦除。 + +页面和条目 +^^^^^^^^^^^^^^^^^ + +NVS 库在其操作中主要使用两个实体：页面和条目。页面是一个逻辑结构，用于存储部分的整体日志。逻辑页面对应 flash 的一个物理扇区，正在使用中的页面具有与之相关联的序列号。序列号赋予了页面顺序，较高的序列号对应较晚创建的页面。页面有以下几种状态： + +空或未初始化 + 页面对应的 flash 扇区为空白状态（所有字节均为 ``0xff``）。此时，页面未存储任何数据且没有关联的序列号。 + +活跃状态 + 此时 flash 已完成初始化，页头部写入 flash，页面已具备有效序列号。页面中存在一些空条目，可写入数据。任意时刻，至多有一个页面处于活跃状态。 + +写满状态 + Flash 已写满键值对，状态不再改变。用户无法向写满状态下的页面写入新键值对，但仍可将一些键值对标记为已擦除。 + +擦除状态 + 未擦除的键值对将移至其他页面，以便擦除当前页面。这一状态仅为暂时性状态，即 API 调用返回时，页面应脱离这一状态。如果设备突然断电，下次开机时，设备将继续把未擦除的键值对移至其他页面，并继续擦除当前页面。 + +损坏状态 + 页头部包含无效数据，无法进一步解析该页面中的数据，因此之前写入该页面的所有条目均无法访问。相应的 flash 扇区并不会被立即擦除，而是与其他处于未初始化状态的扇区一起等待后续使用。这一状态可能对调试有用。 + +Flash 扇区映射至逻辑页面并没有特定的顺序，NVS 库会检查存储在 flash 扇区的页面序列号，并根据序列号组织页面。 + +:: + + +--------+ +--------+ +--------+ +--------+ + | Page 1 | | Page 2 | | Page 3 | | Page 4 | + | Full +---> | Full +---> | Active | | Empty | <- 状态 + | #11 | | #12 | | #14 | | | <- 序列号 + +---+----+ +----+---+ +----+---+ +---+----+ + | | | | + | | | | + | | | | + +---v------+ +-----v----+ +------v---+ +------v---+ + | Sector 3 | | Sector 0 | | Sector 2 | | Sector 1 | <- 物理扇区 + +----------+ +----------+ +----------+ +----------+ + +页面结构 +^^^^^^^^^^^^^^^^^^^ + +当前，我们假设 flash 扇区大小为 4096 字节，并且 ESP32 flash 加密硬件在 32 字节块上运行。未来有可能引入一些编译时可配置项（可通过 menuconfig 进行配置），以适配具有不同扇区大小的 flash 芯片。但目前尚不清楚 SPI flash 驱动和 SPI flash cache 之类的系统组件是否支持其他扇区大小。 + +页面由头部、条目状态位图和条目三部分组成。为了实现与 ESP32 flash 加密功能兼容，条目大小设置为 32 字节。如果键值为整数型，条目则保存一个键值对；如果键值为字符串或 BLOB 类型，则条目仅保存一个键值对的部分内容（更多信息详见条目结构描述）。 + +页面结构如下图所示，括号内数字表示该部分的大小（以字节为单位）:: + + +-----------+--------------+-------------+-------------------------+ + | State (4) | Seq. no. (4) | version (1) | Unused (19) | CRC32 (4) | 页头部 (32) + +-----------+--------------+-------------+-------------------------+ + | Entry state bitmap (32) | + +------------------------------------------------------------------+ + | Entry 0 (32) | + +------------------------------------------------------------------+ + | Entry 1 (32) | + +------------------------------------------------------------------+ + / / + / / + +------------------------------------------------------------------+ + | Entry 125 (32) | + +------------------------------------------------------------------+ + +头部和条目状态位图写入 flash 时不加密。如果启用了 ESP32 flash 加密功能，则条目写入 flash 时将会加密。 + +通过将 0 写入某些位可以定义页面状态值，表示状态改变。因此，如果需要变更页面状态，并不一定要擦除页面，除非要将其变更为擦除状态。 + +头部中的 ``version`` 字段反映了所用的 NVS 格式版本。为实现向后兼容，版本升级从 0xff 开始依次递减（例如，version-1 为 0xff，version-2 为 0xfe 等）。 + +头部中 CRC32 值是由不包含状态值的条目计算所得（4 到 28 字节）。当前未使用的条目用 ``0xff`` 字节填充。 + +条目结构和条目状态位图详细信息见下文描述。 + +条目和条目状态位图 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +每个条目可处于以下三种状态之一，每个状态在条目状态位图中用两位表示。位图中的最后四位 (256 - 2 * 126) 未使用。 + +空 (2'b11) + 条目还未写入任何内容，处于未初始化状态（全部字节为 ``0xff``）。 + +写入（2'b10） + 一个键值对（或跨多个条目的键值对的部分内容）已写入条目中。 + +擦除（2'b00） + 条目中的键值对已丢弃，条目内容不再解析。 + +.. _structure_of_entry: + +条目结构 +^^^^^^^^^^^^^^^^^^ + +如果键值类型为基础类型，即 1 - 8 个字节长度的整数型，条目将保存一个键值对；如果键值类型为字符串或 BLOB 类型，条目将保存整个键值对的部分内容。另外，如果键值为字符串类型且跨多个条目，则键值所跨的所有条目均保存在同一页面。BLOB 则可以切分为多个块，实现跨多个页面。BLOB 索引是一个附加的固定长度元数据条目，用于追踪 BLOB 块。目前条目仍支持早期 BLOB 格式（可读取可修改），但这些 BLOB 一经修改，即以新格式储存至条目。 + +:: + + +--------+----------+----------+----------------+-----------+---------------+----------+ + | NS (1) | Type (1) | Span (1) | ChunkIndex (1) | CRC32 (4) | Key (16) | Data (8) | + +--------+----------+----------+----------------+-----------+---------------+----------+ + + Primitive +--------------------------------+ + +--------> | Data (8) | + | Types +--------------------------------+ + +-> Fixed length -- + | | +---------+--------------+---------------+-------+ + | +--------> | Size(4) | ChunkCount(1)| ChunkStart(1) | Rsv(2)| + Data format ---+ BLOB Index +---------+--------------+---------------+-------+ + | + | +----------+---------+-----------+ + +-> Variable length --> | Size (2) | Rsv (2) | CRC32 (4) | + (Strings, BLOB Data) +----------+---------+-----------+ + + +条目结构中各个字段含义如下： + +命名空间 (NS, NameSpace) + 该条目的命名空间索引，详细信息见命名空间实现章节。 + +类型 (Type) + 一个字节表示的值的数据类型，可能的类型见 ``nvs_types.h`` 中 ``ItemType`` 枚举。 + +跨度 (Span) + 该键值对所用的条目数量。如果键值为整数型，条目数量即为 1。如果键值为字符串或 BLOB，则条目数量取决于值的长度。 + +块索引 (ChunkIndex) + 用于存储 BLOB 类型数据块的索引。如果键值为其他数据类型，则此处索引应写入 ``0xff``。 + +CRC32 + 对条目下所有字节进行校验，所得的校验和（CRC32 字段不计算在内）。 + +键 (Key) + 即以零结尾的 ASCII 字符串，字符串最长为 15 字节，不包含最后一个字节的 NULL (``\0``) 终止符。 + +数据 (Data) + 如果键值类型为整数型，则数据字段仅包含键值。如果键值小于八个字节，使用 ``0xff`` 填充未使用的部分（右侧）。 + + 如果键值类型为 BLOB 索引条目，则该字段的八个字节将保存以下数据块信息： + + - 块大小 + 整个 BLOB 数据的大小（以字节为单位）。该字段仅用于 BLOB 索引类型条目。 + + - ChunkCount + 存储过程中 BLOB 分成的数据块数量。该字段仅用于 BLOB 索引类型条目。 + + - ChunkStart + BLOB 第一个数据块的块索引，后续数据块索引依次递增，步长为 1。该字段仅用于 BLOB 索引类型条目。 + + 如果键值类型为字符串或 BLOB 数据块，数据字段的这八个字节将保存该键值的一些附加信息，如下所示： + + - 数据大小 + 实际数据的大小（以字节为单位）。如果键值类型为字符串，此字段也应将零终止符包含在内。此字段仅用于字符串和 BLOB 类型条目。 + + - CRC32 + 数据所有字节的校验和，该字段仅用于字符串和 BLOB 类型条目。 + +可变长度值（字符串和 BLOB）写入后续条目，每个条目 32 字节。第一个条目的 span 字段将指明使用了多少条目。 + +命名空间 +^^^^^^^^^^ + +如上所述，每个键值对属于一个命名空间。命名空间标识符（字符串）也作为键值对的键，存储在索引为 0 的命名空间中。与这些键对应的值就是这些命名空间的索引。 + +:: + + +-------------------------------------------+ + | NS=0 Type=uint8_t Key="wifi" Value=1 | Entry describing namespace "wifi" + +-------------------------------------------+ + | NS=1 Type=uint32_t Key="channel" Value=6 | Key "channel" in namespace "wifi" + +-------------------------------------------+ + | NS=0 Type=uint8_t Key="pwm" Value=2 | Entry describing namespace "pwm" + +-------------------------------------------+ + | NS=2 Type=uint16_t Key="channel" Value=20 | Key "channel" in namespace "pwm" + +-------------------------------------------+ + + +条目哈希列表 +^^^^^^^^^^^^^^ + +为了减少对 flash 执行的读操作次数，Page 类对象均设有一个列表，包含一对数据：条目索引和条目哈希值。该列表可大大提高检索速度，而无需迭代所有条目并逐个从 flash 中读取。``Page::findItem`` 首先从哈希列表中检索条目哈希值，如果条目存在，则在页面内给出条目索引。由于哈希冲突，在哈希列表中检索条目哈希值可能会得到不同的条目，对 flash 中条目再次迭代可解决这一冲突。 + +哈希列表中每个节点均包含一个 24 位哈希值和 8 位条目索引。哈希值根据条目命名空间、键名和块索引由 CRC32 计算所得，计算结果保留 24 位。为减少将 32 位条目存储在链表中的开销，链表采用了数组的双向链表。每个数组占用 128 个字节，包含 29 个条目、两个链表指针和一个 32 位计数字段。因此，每页额外需要的 RAM 最少为 128 字节，最多为 640 字节。 + +.. _nvs_encryption: + +NVS 加密 +-------------- + +NVS 分区内存储的数据可使用 AES-XTS 进行加密，类似于 IEEE P1619 磁盘加密标准中提到的加密方式。为了实现加密，每个条目被均视为一个扇区，并将条目相对地址（相对于分区开头）传递给加密算法，用作扇区号。NVS 加密所需的密钥存储于其他分区，并进行了 :doc:`flash 加密 <../../security/flash-encryption>`。因此，在使用 NVS 加密前应先启用 :doc:`flash 加密 <../../security/flash-encryption>`。 + +.. _nvs_key_partition: + +NVS 密钥分区 +^^^^^^^^^^^^^^^^^ + +应用程序如果想使用 NVS 加密，则需要编译进一个类型为 ``data``，子类型为 ``key`` 的密钥分区。该分区应标记为已加密，且最小为 4096 字节，具体结构见下表。如需了解更多详细信息，请参考 :doc:`分区表 <../../api-guides/partition-tables>`。 + +:: + + +-----------+--------------+-------------+----+ + | XTS encryption key(32) | + +---------------------------------------------+ + | XTS tweak key (32) | + +---------------------------------------------+ + | CRC32(4) | + +---------------------------------------------+ + +使用 NVS 分区生成程序生成上述分区表，并烧录至设备。由于分区已标记为已加密，而且启用了 :doc:`flash 加密 <../../security/flash-encryption>`，引导程序在首次启动时将使用 flash 加密对密钥分区进行加密。您也可以在设备启动后调用 ``nvs_flash.h`` 提供的 ``nvs_flash_generate_keys`` API 生成加密密钥，然后再将密钥以加密形式写入密钥分区。 + +应用程序可以使用不同的密钥对不同的 NVS 分区进行加密，这样就会需要多个加密密钥分区。应用程序应为加解密操作提供正确的密钥或密钥分区。 + +加密读取/写入 +^^^^^^^^^^^^^^^^^^^^ + +``nvs_read_*`` 和 ``nvs_write_*`` 等 NVS API 函数同样可以对 NVS 加密分区执行读写操作。但用于初始化 NVS 非加密分区和加密分区的 API 则有所不同：初始化 NVS 非加密分区可以使用 ``nvs_flash_init`` 和 ``nvs_flash_init_partition``，但初始化 NVS 加密分区则需调用 ``nvs_flash_secure_init`` 和 ``nvs_flash_secure_init_partition``。上述 API 函数所需的 ``nvs_sec_cfg_t`` 结构可使用 ``nvs_flash_generate_keys`` 或者 ``nvs_flash_read_security_cfg`` 进行填充。 + +应用程序如需在加密状态下执行 NVS 读写操作，应遵循以下步骤： + + 1. 使用 ``esp_partition_find*`` API 查找密钥分区和 NVS 数据分区； + 2. 使用 ``nvs_flash_read_security_cfg`` 或 ``nvs_flash_generate_keys`` API 填充 ``nvs_sec_cfg_t`` 结构； + 3. 使用 ``nvs_flash_secure_init`` 或 ``nvs_flash_secure_init_partition`` API 初始化 NVS flash 分区； + 4. 使用 ``nvs_open`` 或 ``nvs_open_from_part`` API 打开命名空间； + 5. 使用 ``nvs_read_*`` 或 ``nvs_write_*`` API 执行 NVS 读取/写入操作； + 6. 使用 ``nvs_flash_deinit`` API 释放已初始化的 NVS 分区。 + +NVS 迭代器 +^^^^^^^^^^^^^ + +迭代器允许根据指定的分区名称、命名空间和数据类型轮询 NVS 中存储的键值对。 + +您可以使用以下函数，执行相关操作： + +- ``nvs_entry_find``：返回一个不透明句柄，用于后续调用 ``nvs_entry_next`` 和 ``nvs_entry_info`` 函数； +- ``nvs_entry_next``：返回指向下一个键值对的迭代器； +- ``nvs_entry_info``：返回每个键值对的信息。 + +如果未找到符合标准的键值对，``nvs_entry_find`` 和 ``nvs_entry_next`` 将返回 NULL，此时不必释放迭代器。若不再需要迭代器，可使用 ``nvs_release_iterator`` 释放迭代器。 + diff --git a/components/spi_flash/README.rst b/components/spi_flash/README.rst index f9c7ecaa0..02c26950c 100644 --- a/components/spi_flash/README.rst +++ b/components/spi_flash/README.rst @@ -1,6 +1,8 @@ SPI Flash API ============= +:link_to_translation:`zh_CN:[中文]` + Overview -------- The spi_flash component contains API functions related to reading, writing, diff --git a/components/spi_flash/README_CN.rst b/components/spi_flash/README_CN.rst new file mode 100644 index 000000000..096555cc6 --- /dev/null +++ b/components/spi_flash/README_CN.rst @@ -0,0 +1,188 @@ +SPI Flash API +================= + +:link_to_translation:`en:[English]` + +概述 +-------- + +SPI Flash 组件提供外部 flash 数据读取、写入、擦除和内存映射相关的 API 函数，同时也提供了更高层级的，面向分区的 API 函数（定义在 :doc:`分区表 ` 中）。 + +与 ESP-IDF V4.0 之前的 API 不同，这一版 API 功能并不局限于主 SPI Flash 芯片（即运行程序的 SPI Flash 芯片）。使用不同的芯片指针，您可以通过 SPI0/1 或 HSPI/VSPI 总线访问外部 flash。 + +.. note:: + + ESP-IDF V4.0 之后的 flash API 不再是原子的。因此，如果 flash 操作地址有重叠，且写操作与读操作同时执行，读操作可能会返回一部分写入之前的数据，返回一部分写入之后的数据。 + +Kconfig 选项 :ref:`CONFIG_SPI_FLASH_USE_LEGACY_IMPL` 可将 ``spi_flash_*`` 函数切换至 ESP-IDF V4.0 之前的实现。但是，如果同时使用新旧 API，代码量可能会增多。 + +即便未启用 :ref:`CONFIG_SPI_FLASH_USE_LEGACY_IMPL`，加密读取和加密写入操作也均使用旧实现。因此，仅有主 flash 芯片支持加密操作，其他不同片选（经 SPI1 访问的 flash 芯片）则不支持加密操作。 + +初始化 Flash 设备 +--------------------------- + +在使用 ``esp_flash_*`` API 之前，您需要在 SPI 总线上初始化芯片。 + +1. 调用 :cpp:func:`spi_bus_initialize` 初始化 SPI 总线，此函数将初始化总线上设备间共享的资源，如 I/O、DMA 及中断等。 + +2. 调用 :cpp:func:`spi_bus_add_flash_device` 将 flash 设备连接到总线上。然后分配内存，填充 ``esp_flash_t`` 结构体，同时初始化 CS I/O。 + +3. 调用 :cpp:func:`esp_flash_init` 与芯片进行通信。后续操作会依据芯片类型不同而有差异。 + +.. note:: 目前，多个 flash 芯片可连接到同一总线。但尚不支持在同一个 SPI 总线上使用 ``esp_flash_*`` 和 ``spi_device_*`` 设备。 + +SPI Flash 访问 API +-------------------- + +如下所示为处理 flash 中数据的函数集： + +- :cpp:func:`esp_flash_read`：将数据从 flash 读取到 RAM； +- :cpp:func:`esp_flash_write`：将数据从 RAM 写入到 flash； +- :cpp:func:`esp_flash_erase_region`：擦除 flash 中指定区域的数据； +- :cpp:func:`esp_flash_erase_chip`：擦除整个 flash； +- :cpp:func:`esp_flash_get_chip_size`：返回 menuconfig 中设置的 flash 芯片容量（以字节为单位）。 + +一般来说，请尽量避免对主 SPI flash 芯片直接使用原始 SPI flash 函数，如需对主 SPI flash 芯片进行操作，请使用 :ref:`分区专用函数 `。 + +SPI Flash 容量 +-------------- + +SPI flash 容量存储于引导程序映像头部（烧录偏移量为 0x1000）的一个字段。 + +默认情况下，引导程序写入 flash 时，esptool.py 将引导程序写入 flash 时，会自动检测 SPI flash 容量，同时使用正确容量更新引导程序的头部。您也可以在工程配置中设置 :envvar:`CONFIG_ESPTOOLPY_FLASHSIZE`，生成固定的 flash 容量。 + +如需在运行时覆盖已配置的 flash 容量，请配置 ``g_rom_flashchip`` 结构中的 ``chip_size``。``esp_flash_*`` 函数使用此容量（于软件和 ROM 中）进行边界检查。 + +SPI1 Flash 并发约束 +----------------------------------------- + +由于 SPI1 flash 也被用于执行固件（通过指令 cache 或数据 cache ），因此在执行读取、写入及擦除操作时，必须禁用这些 cache。这意味着在执行 flash 写操作时，两个 CPU 必须从 IRAM 运行代码，且只能从 DRAM 中读取数据。 + +如果您使用本文档中 API 函数，上述限制将自动生效且透明（无需您额外关注），但这些限制可能会影响系统中的其他任务的性能。 + +除 SPI0/1 以外的 SPI 总线上的其它 flash 芯片则不受这种限制。 + +请参阅 :ref:`应用程序内存分布 `，查看 IRAM、DRAM 和 flash cache 的区别。 + +为避免意外读取 flash cache，一个 CPU 在启动 flash 写入或擦除操作时，另一个 CPU 将阻塞，并且在 flash 操作完成前，两个 CPU 上的所有的非 IRAM 安全的中断都会被禁用。 + +.. _iram-safe-interrupt-handlers: + +IRAM 安全中断处理程序 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +如果您需要在 flash 操作期间运行中断处理程序（比如低延迟操作），请在 :doc:`注册中断处理程序 ` 时设置 ``ESP_INTR_FLAG_IRAM``。 + +请确保中断处理程序访问的所有数据和函数（包括其调用的数据和函数）都存储在 IRAM 或 DRAM 中。 + +为函数添加 ``IRAM_ATTR`` 属性:: + + #include "esp_attr.h" + + void IRAM_ATTR gpio_isr_handler(void* arg) + { + // ... + } + + +为常量添加 ``DRAM_ATTR`` 和 ``DRAM_STR`` 属性:: + + void IRAM_ATTR gpio_isr_handler(void* arg) + { + const static DRAM_ATTR uint8_t INDEX_DATA[] = { 45, 33, 12, 0 }; + const static char *MSG = DRAM_STR("I am a string stored in RAM"); + } + +辨别哪些数据应标记为 ``DRAM_ATTR`` 可能会比较困难，除非明确标记为 ``DRAM_ATTR``，否则编译器依然可能将某些变量或表达式当做常量（即便没有 ``const`` 标记），并将其放入 flash。 + +如果函数或符号未被正确放入 IRAM/DRAM 中，当中断处理程序在 flash 操作期间从 flash cache 中读取数据，则会产生非法指令异常（这是因为代码未被正确放入 IRAM）或读取垃圾数据（这是因为常数未被正确放入 DRAM），而导致崩溃。 + +.. _flash-partition-apis: + +分区表 API +------------------- + +ESP-IDF 工程使用分区表保存 SPI flash 各区信息，包括引导程序、各种应用程序二进制文件、数据及文件系统等。请参考 :doc:`分区表 `，查看详细信息。 + +该组件在 ``esp_partition.h`` 中声明了一些 API 函数，用以枚举在分区表中找到的分区，并对这些分区执行操作： + +- :cpp:func:`esp_partition_find`：在分区表中查找特定类型的条目，返回一个不透明迭代器； +- :cpp:func:`esp_partition_get`：返回一个结构，描述给定迭代器的分区； +- :cpp:func:`esp_partition_next`：将迭代器移至下一个找到的分区； +- :cpp:func:`esp_partition_iterator_release`：释放 ``esp_partition_find`` 中返回的迭代器； +- :cpp:func:`esp_partition_find_first`：返回一个结构，描述 ``esp_partition_find`` 中找到的第一个分区； +- :cpp:func:`esp_partition_read`、:cpp:func:`esp_partition_write` 和 :cpp:func:`esp_partition_erase_range` 在分区边界内执行，等同于 :cpp:func:`spi_flash_read`、:cpp:func:`spi_flash_write` 和 :cpp:func:`spi_flash_erase_range`。 + +.. note:: + 请在应用程序代码中使用上述 ``esp_partition_*`` API 函数，而非低层级的 ``spi_flash_*`` API 函数。分区表 API 函数根据存储在分区表中的数据，进行边界检查并计算在 flash 中的正确偏移量。 + +SPI Flash 加密 +-------------------- + +您可以对 SPI flash 内容进行加密，并在硬件层对其进行透明解密。 + +请参阅 :doc:`Flash 加密 `，查看详细信息。 + +内存映射 API +------------------ + +ESP32 内存硬件可以将 flash 部分区域映射到指令地址空间和数据地址空间，此映射仅用于读操作。不能通过写入 flash 映射的存储区域来改变 flash 中内容。 + +Flash 以 64 KB 页为单位进行地址映射。内存映射硬件最多可将 4 MB flash 映射到数据地址空间，将 16 MB flash 映射到指令地址空间。请参考《ESP32 技术参考手册》查看内存映射硬件的详细信息。 + +请注意，有些 64 KB 页还用于将应用程序映射到内存中，因此实际可用的 64 KB 页会更少一些。 + +:doc:`Flash 加密 ` 启用时，使用内存映射区域从 flash 读取数据是解密 flash 的唯一方法，解密需在硬件层进行。 + +内存映射 API 在 ``esp_spi_flash.h`` 和 ``esp_partition.h`` 中声明： + +- :cpp:func:`spi_flash_mmap`：将 flash 物理地址区域映射到 CPU 指令空间或数据空间； +- :cpp:func:`spi_flash_munmap`：取消上述区域的映射； +- :cpp:func:`esp_partition_mmap`：将分区的一部分映射至 CPU 指令空间或数据空间； + + :cpp:func:`spi_flash_mmap` 和 :cpp:func:`esp_partition_mmap` 的区别如下： + +- :cpp:func:`spi_flash_mmap`：需要给定一个 64 KB 对齐的物理地址； +- :cpp:func:`esp_partition_mmap`：给定分区内任意偏移量即可，此函数根据需要将返回的指针调整至指向映射内存。 + +内存映射在 64 KB 块中进行，如果分区已传递给 ``esp_partition_mmap``，则可读取分区外数据。 + +实现 +-------------- + +``esp_flash_t`` 结构包含芯片数据和该 API 的三个重要部分： + +1. 主机驱动，为访问芯片提供硬件支持； +2. 芯片驱动，为不同芯片提供兼容性服务； +3. OS 函数，在不同阶段（一级或二级 Boot 或者应用程序阶段）为部分 OS 函数提供支持（如一些锁、延迟）。 + +主机驱动 +^^^^^^^^^^^^^^^ + +主机驱动依赖 ``soc/include/hal`` 文件夹下 ``spi_flash_host_drv.h`` 定义的 ``spi_flash_host_driver_t`` 接口。该接口提供了一些与芯片通信常用的函数。 + +在 SPI HAL 文件中，有些函数是基于现有的 ESP32 memory-spi 来实现的。但是，由于 ESP32 速度限制，HAL 层无法提供某些读命令的高速实现（所以这些命令根本没有在 HAL 的文件中被实现）。``memspi_host_driver.h`` 和 ``.c`` 文件使用 HAL 提供的 ``common_command`` 函数实现上述读命令的高速版本，并将所有它实现的及 HAL 函数封装为 ``spi_flash_host_driver_t`` 供更上层调用。 + +您也可以实现自己的主机驱动，甚至只通过简单的 GPIO。只要实现了 ``spi_flash_host_driver_t`` 中所有函数，不管底层硬件是什么，esp_flash API 都可以访问 flash。 + +芯片驱动 +^^^^^^^^^^^ + +芯片驱动在 ``spi_flash_chip_driver.h`` 中进行定义，并将主机驱动提供的基本函数进行封装以供 API 层使用。 + +有些操作需在执行前先发送命令，或在执行后读取状态，因此有些芯片需要不同的命令或值以及通信方式。 + +``generic chip`` 芯片代表了常见的 flash 芯片，其他芯片驱动可以在通用芯片的基础上进行开发。 + +芯片驱动依赖主机驱动。 + +OS 函数 +^^^^^^^^^^^^ + +OS 函数层提供访问锁和延迟的方法。 + +该锁定用于解决 SPI Flash 芯片访问和其他函数之间的冲突。例如，经 SPI0/1 访问 flash 芯片时，应当禁用 cache（平时用于取代码和 PSRAM 数据）。另一种情况是，一些没有 CS 线或者 CS 线受软件控制的设备（如通过 SPI 接口的 SD 卡控制）需要在一段时间内独占总线。 + +延时则用于某些长时操作，需要主机处于等待状态或执行轮询。 + +顶层 API 将芯片驱动和 OS 函数封装成一个完整的组件，并提供参数检查。 diff --git a/components/vfs/README.rst b/components/vfs/README.rst index c296e81a6..f3e2a8bf7 100644 --- a/components/vfs/README.rst +++ b/components/vfs/README.rst @@ -1,6 +1,8 @@ Virtual filesystem component ============================ +:link_to_translation:`zh_CN:[中文]` + Overview -------- diff --git a/components/vfs/README_CN.rst b/components/vfs/README_CN.rst new file mode 100644 index 000000000..93118ec19 --- /dev/null +++ b/components/vfs/README_CN.rst @@ -0,0 +1,167 @@ +虚拟文件系统组件 +============================ + +:link_to_translation:`en:[English]` + +概述 +-------- + +虚拟文件系统 (VFS) 组件可为一些驱动提供一个统一接口。有了该接口，用户可像操作普通文件一样操作虚拟文件。这类驱动程序可以是 FAT、SPIFFS 等真实文件系统，也可以是有文件类接口的设备驱动程序。 + +VFS 组件支持 C 库函数（如 fopen 和 fprintf 等）与文件系统 (FS) 驱动程序协同工作。在高层级，每个 FS 驱动程序均与某些路径前缀相关联。当一个 C 库函数需要打开文件时，VFS 组件将搜索与该文件所在文件路径相关联的 FS 驱动程序，并将调用传递给该驱动程序。针对该文件的读取、写入等其他操作的调用也将传递给这个驱动程序。 + +例如，您可以使用 ``/fat`` 前缀注册 FAT 文件系统驱动，之后即可调用 ``fopen("/fat/file.txt", "w")``。之后，VFS 将调用 FAT 驱动的 ``open`` 函数，并将参数 ``/file.txt`` 和合适的打开模式传递给 ``open`` 函数；后续对返回的 ``FILE*`` 数据流调用 C 库函数也同样会传递给 FAT 驱动。 + +注册 FS 驱动程序 +--------------------- + +如需注册 FS 驱动程序，首先要定义一个 :cpp:type:`esp_vfs_t` 结构体实例，并用指向 FS API 的函数指针填充它。 + +.. highlight:: c + +:: + + esp_vfs_t myfs = { + .flags = ESP_VFS_FLAG_DEFAULT, + .write = &myfs_write, + .open = &myfs_open, + .fstat = &myfs_fstat, + .close = &myfs_close, + .read = &myfs_read, + }; + + ESP_ERROR_CHECK(esp_vfs_register("/data", &myfs, NULL)); + +在上述代码中需要用到 ``read``、 ``write`` 或 ``read_p``、 ``write_p``，具体使用哪组函数由 FS 驱动程序 API 的声明方式决定。 + +示例 1：声明 API 函数时不带额外的上下文指针参数，即 FS 驱动程序为单例模式，此时使用 ``write`` :: + + ssize_t myfs_write(int fd, const void * data, size_t size); + + // In definition of esp_vfs_t: + .flags = ESP_VFS_FLAG_DEFAULT, + .write = &myfs_write, + // ... other members initialized + + // When registering FS, context pointer (third argument) is NULL: + ESP_ERROR_CHECK(esp_vfs_register("/data", &myfs, NULL)); + +示例 2：声明 API 函数时需要一个额外的上下文指针作为参数，即可支持多个 FS 驱动程序实例，此时使用 ``write_p`` :: + + ssize_t myfs_write(myfs_t* fs, int fd, const void * data, size_t size); + + // In definition of esp_vfs_t: + .flags = ESP_VFS_FLAG_CONTEXT_PTR, + .write_p = &myfs_write, + // ... other members initialized + + // When registering FS, pass the FS context pointer into the third argument + // (hypothetical myfs_mount function is used for illustrative purposes) + myfs_t* myfs_inst1 = myfs_mount(partition1->offset, partition1->size); + ESP_ERROR_CHECK(esp_vfs_register("/data1", &myfs, myfs_inst1)); + + // Can register another instance: + myfs_t* myfs_inst2 = myfs_mount(partition2->offset, partition2->size); + ESP_ERROR_CHECK(esp_vfs_register("/data2", &myfs, myfs_inst2)); + +同步输入/输出多路复用 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +如需通过 :cpp:func:`select` 使用同步输入/输出多路复用，首先需要把 :cpp:func:`start_select` 和 :cpp:func:`end_select` 注册到 VFS，如下所示： + +.. highlight:: c + +:: + + // In definition of esp_vfs_t: + .start_select = &uart_start_select, + .end_select = &uart_end_select, + // ... other members initialized + +调用 :cpp:func:`start_select` 设置环境，用以检测某一 VFS 文件描述符的读取/写入/错误条件。调用 :cpp:func:`end_select` 终止、析构或释放 :cpp:func:`start_select` 设置的资源。请在 :component_file:`vfs/vfs_uart.c` 中查看 UART 外设参考实现、:cpp:func:`esp_vfs_dev_uart_register`、:cpp:func:`uart_start_select` 和 :cpp:func:`uart_end_select` 函数。 + +请参考以下示例，查看如何使用 VFS 文件描述符调用 :cpp:func:`select`： + +- :example:`peripherals/uart_select` +- :example:`system/select` + +如果 :cpp:func:`select` 用于套接字文件描述符，您可以启用 :envvar:`CONFIG_LWIP_USE_ONLY_LWIP_SELECT` 选项来减少代码量，提高性能。 + +路径 +----- + +已注册的 FS 驱动程序均有一个路径前缀与之关联，此路径前缀即为分区的挂载点。 + +如果挂载点中嵌套了其他挂载点，则在打开文件时使用具有最长匹配路径前缀的挂载点。例如，假设以下文件系统已在 VFS 中注册： + +- 在 /data 下注册 FS 驱动程序 1 +- 在 /data/static 下注册 FS 驱动程序 2 + +那么： + +- 打开 ``/data/log.txt`` 会调用驱动程序 FS 1； +- 打开 ``/data/static/index.html`` 需调用 FS 驱动程序 2； +- 即便 FS 驱动程序 2 中没有 ``/index.html``，也不会在 FS 驱动程序 1 中查找 ``/static/index.html``。 + +挂载点名称必须以路径分隔符 (``/``) 开头，且分隔符后至少包含一个字符。但在以下情况中，VFS 同样支持空的挂载点名称：1. 应用程序需要提供一个”最后方案“下使用的文件系统；2. 应用程序需要同时覆盖 VFS 功能。如果没有与路径匹配的前缀，就会使用到这种文件系统。 + +VFS 不会对路径中的点 (``.``) 进行特殊处理，也不会将 ``..`` 视为对父目录的引用。在上述示例中，使用 ``/data/static/../log.txt`` 路径不会调用 FS 驱动程序 1 打开 ``/log.txt``。特定的 FS 驱动程序（如 FATFS）可能以不同的方式处理文件名中的点。 + +执行打开文件操作时，FS 驱动程序仅得到文件的相对路径（挂载点前缀已经被去除）： + +1. 以 ``/data`` 为路径前缀注册 ``myfs`` 驱动； +2. 应用程序调用 ``fopen("/data/config.json", ...)``； +3. VFS 调用 ``myfs_open("/config.json", ...)``； +4. ``myfs`` 驱动打开 ``/config.json`` 文件。 + +VFS 对文件路径长度没有限制，但文件系统路径前缀受 ``ESP_VFS_PATH_MAX`` 限制，即路径前缀上限为 ``ESP_VFS_PATH_MAX``。各个文件系统驱动则可能会对自己的文件名长度设置一些限制。 + + +文件描述符 +---------------- + +文件描述符是一组很小的正整数，从 ``0`` 到 ``FD_SETSIZE - 1``，``FD_SETSIZE`` 在 newlib ``sys/types.h`` 中定义。最大文件描述符由 ``CONFIG_LWIP_MAX_SOCKETS`` 定义，且为套接字保留。VFS 中包含一个名为 ``s_fd_table`` 的查找表，用于将全局文件描述符映射至 ``s_vfs`` 数组中注册的 VFS 驱动索引。 + + +标准 IO 流 (stdin, stdout, stderr) +------------------------------------------- + +如果 menuconfig 中 ``UART for console output`` 选项没有设置为 ``None``，则 ``stdin``、 ``stdout`` 和 ``stderr`` 将默认从 UART 读取或写入。UART0 或 UART1 可用作标准 IO。默认情况下，UART0 使用 115200 波特率，TX 管脚为 GPIO1，RX 管脚为 GPIO3。您可以在 menuconfig 中更改上述参数。 + +对 ``stdout`` 或 ``stderr`` 执行写入操作将会向 UART 发送 FIFO 发送字符，对 ``stdin`` 执行读取操作则会从 UART 接收 FIFO 中取出字符。 + +默认情况下，VFS 使用简单的函数对 UART 进行读写操作。在所有数据放进 UART FIFO 之前，写操作将处于 busy-wait 状态，读操处于非阻塞状态，仅返回 FIFO 中已有数据。由于读操作为非阻塞，高层级 C 库函数调用（如 ``fscanf("%d\n", &var);``）可能获取不到所需结果。 + +如果应用程序使用 UART 驱动，则可以调用 ``esp_vfs_dev_uart_use_driver`` 函数来指导 VFS 使用驱动中断、读写阻塞功能等。您也可以调用 ``esp_vfs_dev_uart_use_nonblocking`` 来恢复非阻塞函数。 + +VFS 还为输入和输出提供换行符转换功能（可选）。多数应用程序在程序内部发送或接收以 LF (''\n'') 结尾的行，但不同的终端程序可能需要不同的换行符，比如 CR 或 CRLF。应用程序可以通过 menuconfig 或者调用 ``esp_vfs_dev_uart_set_rx_line_endings`` 和 ``esp_vfs_dev_uart_set_tx_line_endings`` 为输入输出配置换行符。 + + +标准流和 FreeRTOS 任务 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``stdin``、``stdout`` 和 ``stderr`` 的 ``FILE`` 对象在所有 FreeRTOS 任务之间共享，指向这些对象的指针分别存储在每个任务的 ``struct _reent`` 中。 + +预处理器把如下代码： + +.. highlight:: c + +:: + + fprintf(stderr, "42\n"); + +解释为： + +.. highlight:: c + +:: + + fprintf(__getreent()->_stderr, "42\n"); + +其中 ``__getreent()`` 函数将为每个任务返回一个指向 ``struct _reent`` 的指针 (:component_file:`newlib/include/sys/reent.h#L370-L417`)。每个任务的 TCB 均拥有一个 ``struct _reent`` 结构体，任务初始化后，``struct _reent`` 结构体中的 ``_stdin``、``_stdout`` 和 ``_stderr`` 将会被赋予 ``_GLOBAL_REENT`` 中 ``_stdin``、 ``_stdout`` 和 ``_stderr`` 的值，``_GLOBAL_REENT`` 即为 FreeRTOS 启动之前所用结构体。 + +这样设计带来的结果是： + +- 允许重定向给定任务的 ``stdin``、 ``stdout`` 和 ``stderr``，而不影响其他任务，例如通过 ``stdin = fopen("/dev/uart/1", "r")``； +- 但使用 ``fclose`` 关闭默认 ``stdin``、 ``stdout`` 或 ``stderr`` 将同时关闭相应的 ``FILE`` 流对象，因此会影响其他任务； +- 如需更改新任务的默认 ``stdin``、 ``stdout`` 和 ``stderr`` 流，请在创建新任务之前修改 ``_GLOBAL_REENT->_stdin`` (``_stdout``、``_stderr``)。 diff --git a/docs/zh_CN/api-reference/storage/nvs_flash.rst b/docs/zh_CN/api-reference/storage/nvs_flash.rst index 88fef50af..82ddde80a 100644 --- a/docs/zh_CN/api-reference/storage/nvs_flash.rst +++ b/docs/zh_CN/api-reference/storage/nvs_flash.rst @@ -1 +1,36 @@ -.. include:: ../../../en/api-reference/storage/nvs_flash.rst \ No newline at end of file +.. include:: ../../../../components/nvs_flash/README_CN.rst + +NVS 分区生成程序 +------------------ + +NVS 分区生成程序帮助生成 NVS 分区二进制文件，可使用烧录程序将二进制文件单独烧录至特定分区。烧录至分区上的键值对由 CSV 文件提供，详情请参考 :doc:`NVS 分区生成程序 `。 + +应用示例 +------------------- + +ESP-IDF :example:`storage` 目录下提供了两个代码示例： + +:example:`storage/nvs_rw_value` + + 演示如何读取及写入 NVS 单个整数值。 + + 此示例中的值表示 ESP32 模组重启次数。NVS 中数据不会因为模组重启而丢失，因此只有将这一值存储于 NVS 中，才能起到重启次数计数器的作用。 + + 该示例也演示了如何检测读取/写入操作是否成功，以及某个特定值是否在 NVS 中尚未初始化。诊断程序以纯文本形式提供，帮助您追踪程序流程，及时发现问题。 + +:example:`storage/nvs_rw_blob`　 + + 演示如何读取及写入 NVS 单个整数值和 Blob（二进制大对象），并在 NVS 中存储这一数值，即便 ESP32 模组重启也不会消失。 + + * value - 记录 ESP32 模组软重启次数和硬重启次数。 + * blob - 内含记录模组运行次数的表格。此表格将被从 NVS 读取至动态分配的 RAM 上。每次手动软重启后，表格内运行次数即增加一次，新加的运行次数被写入 NVS。下拉 GPIO0 即可手动软重启。 + + 该示例也演示了如何执行诊断程序以检测读取/写入操作是否成功。 + + +API 参考 +------------- + +.. include:: /_build/inc/nvs_flash.inc + +.. include:: /_build/inc/nvs.inc diff --git a/docs/zh_CN/api-reference/storage/spi_flash.rst b/docs/zh_CN/api-reference/storage/spi_flash.rst index 1856da13f..e141951f1 100644 --- a/docs/zh_CN/api-reference/storage/spi_flash.rst +++ b/docs/zh_CN/api-reference/storage/spi_flash.rst @@ -1 +1,47 @@ -.. include:: ../../../en/api-reference/storage/spi_flash.rst \ No newline at end of file +.. include:: ../../../../components/spi_flash/README_CN.rst + +另请参考 +------------ + +- :doc:`分区表 <../../api-guides/partition-tables>` +- :doc:`OTA API <../system/ota>` 提供了高层 API 用于更新存储在 flash 中的 app 固件。 +- :doc:`NVS API ` 提供了结构化 API 用于存储 SPI flash 中的碎片数据。 + +.. _spi-flash-implementation-details: + +实现细节 +------------ + +必须确保操作期间，两个 CPU 均未从 flash 运行代码，实现细节如下： + +- 单核模式下，SDK 在执行 flash 操作前将禁用中断或调度算法。 +- 双核模式下，实现细节更为复杂，SDK 需确保两个 CPU 均未运行 flash 代码。 + +如果有 SPI flash API 在 CPU A（PRO 或 APP）上调用，它使用 ``esp_ipc_call`` API 在 CPU B 上运行 ``spi_flash_op_block_func`` 函数。``esp_ipc_call`` API 在 CPU B 上唤醒一个高优先级任务，即运行 ``spi_flash_op_block_func`` 函数。运行该函数将禁用 CPU B 上的 cache，并使用 ``s_flash_op_can_start`` 旗帜来标志 cache 已禁用。然后，CPU A 上的任务也会禁用 cache 并继续执行 flash 操作。 + +执行 flash 操作时，CPU A 和 CPU B 仍然可以执行中断操作。默认中断代码均存储于 RAM 中，如果新添加了中断分配 API，则应添加一个标志位以请求在 flash 操作期间禁用该新分配的中断。 + +Flash 操作完成后，CPU A 上的函数将设置另一标志位，即 ``s_flash_op_complete``，用以通知 CPU B 上的任务可以重新启用 cache 并释放 CPU。接着，CPU A 上的函数也重新启用 cache，并将控制权返还给调用者。 + +另外，所有 API 函数均受互斥量 ``s_flash_op_mutex`` 保护。 + +在单核环境中（启用 :ref:`CONFIG_FREERTOS_UNICORE`），您需要禁用上述两个 cache 以防发生 CPU 间通信。 + +SPI Flash API 参考 +------------------------- + +.. include:: /_build/inc/esp_flash_spi_init.inc +.. include:: /_build/inc/esp_flash.inc +.. include:: /_build/inc/spi_flash_types.inc + +分区表 API 参考 +------------------------------- + +.. include:: /_build/inc/esp_partition.inc + +Flash 加密 API 参考 +----------------------------- + +.. include:: /_build/inc/esp_flash_encrypt.inc + + diff --git a/docs/zh_CN/api-reference/storage/vfs.rst b/docs/zh_CN/api-reference/storage/vfs.rst index fa1aef627..cc7644ee7 100644 --- a/docs/zh_CN/api-reference/storage/vfs.rst +++ b/docs/zh_CN/api-reference/storage/vfs.rst @@ -1 +1,15 @@ -.. include:: ../../../en/api-reference/storage/vfs.rst \ No newline at end of file +.. include:: ../../../../components/vfs/README_CN.rst + +应用示例 +------------------- + +`指南`_ （未完成） + +.. _指南: ../../template.html + +API 参考 +------------- + +.. include:: /_build/inc/esp_vfs.inc + +.. include:: /_build/inc/esp_vfs_dev.inc From a3a130d0d684575cfcd5256b8e99c43fb6aded2b Mon Sep 17 00:00:00 2001 From: Martin Vychodil Date: Tue, 20 Aug 2019 08:37:45 +0200 Subject: [PATCH 11/30] Installer: support for WD exclusions (IDF Tools) JIRA IDF-306 --- tools/windows/tool_setup/idf_setup.iss.inc | 12 + tools/windows/tool_setup/idf_tool_setup.iss | 16 +- tools/windows/tool_setup/main.iss.inc | 10 + tools/windows/tool_setup/tools_WD_clean.ps1 | 166 ++++++++++++++ tools/windows/tool_setup/tools_WD_excl.ps1 | 238 ++++++++++++++++++++ 5 files changed, 439 insertions(+), 3 deletions(-) create mode 100644 tools/windows/tool_setup/tools_WD_clean.ps1 create mode 100644 tools/windows/tool_setup/tools_WD_excl.ps1 diff --git a/tools/windows/tool_setup/idf_setup.iss.inc b/tools/windows/tool_setup/idf_setup.iss.inc index c1dd9e113..63716772f 100644 --- a/tools/windows/tool_setup/idf_setup.iss.inc +++ b/tools/windows/tool_setup/idf_setup.iss.inc @@ -255,3 +255,15 @@ begin RaiseException('Failed to create the shortcut'); end; end; + + +{ ------------------------------ WD exclusion registration ------------------------------ } + +procedure RegisterIDFToolsExecutablesInWD(); +var + CmdLine: String; +begin + CmdLine := ExpandConstant('powershell -ExecutionPolicy ByPass -File "{app}\dist\tools_WD_excl.ps1" -AddExclPath "{app}\*.exe"'); + Log('Registering IDF Tools executables in Windows Defender: ' + CmdLine); + DoCmdlineInstall('Finishing ESP-IDF installation', 'Registering IDF Tools executables in Windows Defender', CmdLine); +end; diff --git a/tools/windows/tool_setup/idf_tool_setup.iss b/tools/windows/tool_setup/idf_tool_setup.iss index 13869feae..25a997069 100644 --- a/tools/windows/tool_setup/idf_tool_setup.iss +++ b/tools/windows/tool_setup/idf_tool_setup.iss @@ -65,6 +65,8 @@ Source: "..\..\idf_tools.py"; DestDir: "{app}"; DestName: "idf_tools_fallback.py Source: "tools_fallback.json"; DestDir: "{app}"; DestName: "tools_fallback.json" Source: "idf_cmd_init.bat"; DestDir: "{app}" Source: "dist\*"; DestDir: "{app}\dist" +Source: "tools_WD_excl.ps1"; DestDir: "{app}\dist" +Source: "tools_WD_clean.ps1"; DestDir: "{app}\dist" [UninstallDelete] Type: filesandordirs; Name: "{app}\dist" @@ -72,18 +74,26 @@ Type: filesandordirs; Name: "{app}\releases" Type: filesandordirs; Name: "{app}\tools" Type: filesandordirs; Name: "{app}\python_env" +[Tasks] +Name: createlnk; Description: "Create Desktop shortcut for ESP-IDF Tools Command Line"; +Name: wdexcl; Description: "Register ESP-IDF Tools executables as Windows Defender exclusions (improves compilation speed, requires elevation)"; + [Run] Filename: "{app}\dist\{#PythonInstallerName}"; Parameters: "/passive PrependPath=1 InstallLauncherAllUsers=0 Include_dev=0 Include_tcltk=0 Include_launcher=0 Include_test=0 Include_doc=0"; Description: "Installing Python"; Check: PythonInstallRequired Filename: "{app}\dist\{#GitInstallerName}"; Parameters: "/silent /tasks="""" /norestart"; Description: "Installing Git"; Check: GitInstallRequired Filename: "{group}\{#IDFCmdExeShortcutFile}"; Flags: postinstall shellexec; Description: "Run ESP-IDF Command Prompt (cmd.exe)"; Check: InstallationSuccessful -[Registry] + +[UninstallRun] +Filename: "powershell.exe"; \ + Parameters: "-ExecutionPolicy Bypass -File ""{app}\dist\tools_WD_clean.ps1"" -RmExclPath ""{app}"""; \ + WorkingDir: {app}; Flags: runhidden + +[Registry] Root: HKCU; Subkey: "Environment"; ValueType: string; ValueName: "IDF_TOOLS_PATH"; \ ValueData: "{app}"; Flags: preservestringtype createvalueifdoesntexist uninsdeletevalue deletevalue; [Code] - - #include "utils.iss.inc" #include "choice_page.iss.inc" #include "cmdline_page.iss.inc" diff --git a/tools/windows/tool_setup/main.iss.inc b/tools/windows/tool_setup/main.iss.inc index a023f14d6..288bf6bd8 100644 --- a/tools/windows/tool_setup/main.iss.inc +++ b/tools/windows/tool_setup/main.iss.inc @@ -119,7 +119,17 @@ begin IDFToolsSetup(); + + if WizardIsTaskSelected('createlnk') then + begin CreateIDFCommandPromptShortcut(); + end; + + if WizardIsTaskSelected('wdexcl') then + begin + RegisterIDFToolsExecutablesInWD(); + end; + except SetupAborted := True; if MsgBox('Installation log has been created, it may contain more information about the problem.' + #13#10 diff --git a/tools/windows/tool_setup/tools_WD_clean.ps1 b/tools/windows/tool_setup/tools_WD_clean.ps1 new file mode 100644 index 000000000..0c8868678 --- /dev/null +++ b/tools/windows/tool_setup/tools_WD_clean.ps1 @@ -0,0 +1,166 @@ +################################################################################ +# +# Microsoft WindowsDefender exclusions cleaner +# Espressif Systems, 2019 +# +################################################################################ +# +# - cleans all Windows Defender process exclusions containing given path (both Process and Path) +# - run as Administrator, eg: PowerShell -ExecutionPolicy ByPass -File tools_WD_clean.ps1 -RmExclPath "C:\Program Files\Espressif\ESP-IDF Tools". If not running with admin privileges, the script tries to elevate itself (new process, output grabbed on exit) +# minimum requirements: Windows XP SP3, PowerShell 2.0, Windows Defender with relevant PS cmdlets +# - Returns 0 on success or -1 on failure +# +################################################################################ + + +Param +( + [String]$RmExclPath, + [String]$logFile +) + +Import-Module Defender + +function Check-Command($cmdname) +{ + return [bool](Get-Command -Name $cmdname -ErrorAction SilentlyContinue) +} + +function Log-Msg($msg, $logF = $null) +{ + if( ![string]::IsNullOrEmpty($logF) ) { Write-Output $msg *>> $logF } + else { Write-Output $msg } + [Console]::Out.Flush() +} + + +Try +{ + #self-elevation support + $myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent() + $myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID) + $adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator + + if( -not $myWindowsPrincipal.IsInRole($adminRole) ) { + + $params = "" + foreach($key in $PSBoundParameters.keys) { + $params = -join( $params, "-", $key, " `"", $PSBoundParameters[$key], "`"" ) + } + + #running elevated and logFile not set + if( [string]::IsNullOrEmpty($logFile) ) { + $tempFileName = Get-Date -UFormat "%Y%m%d%H%M%s" + $lf = Join-Path -Path $env:TEMP -ChildPath "WDEspLog$tempFileName.log" + #Write-Output "Logfile: $lf" + } + + $newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell" + $newProcess.Arguments = "-ExecutionPolicy ByPass -File " + $script:MyInvocation.MyCommand.Definition + " " + $params + " -logFile $lf" + $newProcess.Verb = "RunAs" + $newProcess.WindowStyle = [System.Diagnostics.ProcessWindowStyle]::Hidden + + $proc = [System.Diagnostics.Process]::Start($newProcess) + $proc.WaitForExit() + + if (Test-Path -Path $lf ) { + foreach($line in Get-Content $lf) { + Log-Msg -msg $line + } + Remove-Item $lf + } + + #Write-Output "Process finished with code " $proc.ExitCode + exit $proc.ExitCode + } + + Log-Msg -msg "Getting Windows Defender process exclusions..." -logF $logFile + + $Preferences = Get-MpPreference + + #ExclusionProcess + $cnt = $Preferences.ExclusionProcess.Count + $cntRemoved = 0 + $cntRemovedTotal = 0 + $cntMissed = 0 + $cntMissedTotal = 0 + + $bRmPath = ![string]::IsNullOrEmpty($RmExclPath) + if( $bRmPath ) { Log-Msg -msg "Exclusion path: $RmExclPath" -logF $logFile } + + Log-Msg -msg " Found total $cnt of ExclusionProcess items" -logF $logFile + + foreach( $pref in $Preferences.ExclusionProcess ) { + + if( $bRmPath ) { $bGoAhead = $pref.Contains($RmExclPath) } + else { $bGoAhead = $true } + + if( $bGoAhead ) { + Log-Msg -msg " removing $pref" -logF $logFile + Try + { + Remove-MpPreference -ExclusionProcess $pref + $cntRemoved++ + } + Catch + { + if( ![string]::IsNullOrEmpty($logFile) ) { Write-Error -Exception $_.Exception *>> $logFile } + Write-Error -Exception $_.Exception + $cntMissed++ + } + } + } + + if( $cntMissed -eq 0 ) { Log-Msg -msg " $cntRemoved relevant items removed from ExclusionProcess list" -logF $logFile } + else { Log-Msg -msg " WARNING: Only $cntRemoved out of $(cntRemoved+cntMissed) relevant items removed from ExclusionProcess list" -logF $logFile } + + #ExclusionPath + $cnt = $Preferences.ExclusionPath.Count + $cntRemovedTotal = $cntRemoved + $cntRemoved = 0 + $cntMissedTotal = $cntMissed + $cntMissed = 0 + + Log-Msg -msg " Found total $cnt of ExclusionPath items" -logF $logFile + + foreach( $pref in $Preferences.ExclusionPath ) { + + if( $bRmPath ) { $bGoAhead = $pref.Contains($RmExclPath) } + else { $bGoAhead = $true } + + if( $bGoAhead ) { + Log-Msg -msg " removing $pref" -logF $logFile + Try + { + Remove-MpPreference -ExclusionPath $pref + $cntRemoved++ + } + Catch + { + if( ![string]::IsNullOrEmpty($logFile) ) { Write-Error -Exception $_.Exception *>> $logFile } + Write-Error -Exception $_.Exception + $cntMissed++ + } + } + } + + if( $cntMissed -eq 0 ) { Log-Msg -msg " $cntRemoved relevant items removed from ExclusionPath list" -logF $logFile } + else { Log-Msg -msg " WARNING: Only $cntRemoved out of $(cntRemoved+cntMissed) relevant items removed from ExclusionPath list" -logF $logFile } + + #TOTAL + $cntRemovedTotal += $cntRemoved + $cntMissedTotal += $cntMissed + + Log-Msg -msg "============================" -logF $logFile + if( $cntMissedTotal -eq 0 ) { Log-Msg -msg "OK: Processed all $cntRemovedTotal items" -logF $logFile } + else { Log-Msg -msg "WARNING: Processed only $cntRemovedTotal out of $(cntRemovedTotal+cntMissedTotal) relevat items" -logF $logFile } + + Log-Msg -msg "`nDone" -logF $logFile +} +Catch +{ + if( ![string]::IsNullOrEmpty($logFile) ) { Write-Error -Exception $_.Exception *>> $logFile } + Write-Error -Exception $_.Exception + exit -1 +} + diff --git a/tools/windows/tool_setup/tools_WD_excl.ps1 b/tools/windows/tool_setup/tools_WD_excl.ps1 new file mode 100644 index 000000000..f099c81f3 --- /dev/null +++ b/tools/windows/tool_setup/tools_WD_excl.ps1 @@ -0,0 +1,238 @@ +################################################################################ +# +# Microsoft WindowsDefender exclusions handler +# Espressif Systems, 2019 +# +################################################################################ +# +# PS utility to add/remove PROCESS exceptions to/from MS WD real-time +# scanning. Files (referenced by 'path' or 'path\filemask') are expected +# to be Windows process executables, for obvious reasons. +# +# The script requires Administrator privileges to succeed -> self-elevation procedure is involved +# +# Usage: +# +# PowerShell -ExecutionPolicy ByPass -File tools_WD_excl.ps1 +# +# ARGUMENTS: +# -AddExclPath +# add all matching files in the path (recursive) to the WD exception list +# +# -AddExclFile +# adds file to the WD exception list exactly as specified by 'filepath' +# +# -RmExclPath +# remove all matching files in the path (recursive) from WD exclusions +# +# -RmExclFile +# adds file to the WD exception list exactly as specified by 'filepath' +# +# -logFile +# stdout/stderr redirection file. Used internally for elevated process (generated in tempdir, deleted after the script finishing) +# use manually at your own risk +# +# Returns 0 on success or -1 on failure +# +# +# Example: +# PowerShell -ExecutionPolicy ByPass -File tools_WD_excl.ps1 -AddExclPath "C:\Program Files\Espressif\ESP-IDF Tools\*.exe" +# +# Notes: +# - default scenario is set to the following +# -AddExclPath "$Env:ProgramFiles\Espressif\ESP-IDF Tools\*.exe" +# (eg when called with no params) +# - only named parameters are supported, any other use-cases redirect to the default +# - multiple paths/files in 1 parameter are not supported by this version +# - minimum requirements: Windows XP SP3, PowerShell 2.0, Windows Defender with relevant PS cmdlets +# +################################################################################ + + +Param +( + [String]$AddExclPath, + [String]$AddExclFile, + [String]$RmExclPath, + [String]$RmExclFile, + [String]$logFile +) + +Import-Module Defender + +function Check-Command($cmdname) +{ + return [bool](Get-Command -Name $cmdname -ErrorAction SilentlyContinue) +} + +function Log-Msg($msg, $logF = $null) +{ + if( ![string]::IsNullOrEmpty($logF) ) { Write-Output $msg *>> $logF } + else { Write-Output $msg } + [Console]::Out.Flush() +} + + +Try +{ + #self-elevation support + $myWindowsID=[System.Security.Principal.WindowsIdentity]::GetCurrent() + $myWindowsPrincipal=new-object System.Security.Principal.WindowsPrincipal($myWindowsID) + $adminRole=[System.Security.Principal.WindowsBuiltInRole]::Administrator + + if( -not $myWindowsPrincipal.IsInRole($adminRole) ) { + + $params = "" + foreach($key in $PSBoundParameters.keys) { + $params = -join( $params, "-", $key, " `"", $PSBoundParameters[$key], "`"" ) + } + + #running elevated and logFile not set + if( [string]::IsNullOrEmpty($logFile) ) { + $tempFileName = Get-Date -UFormat "%Y%m%d%H%M%s" + $lf = Join-Path -Path $env:TEMP -ChildPath "WDEspLog$tempFileName.log" + #Write-Output "Logfile: $lf" + } + + $newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell" + $newProcess.Arguments = "-ExecutionPolicy ByPass -File " + $script:MyInvocation.MyCommand.Definition + " " + $params + " -logFile $lf" + $newProcess.Verb = "RunAs" + $newProcess.WindowStyle = [System.Diagnostics.ProcessWindowStyle]::Hidden + + $proc = [System.Diagnostics.Process]::Start($newProcess) + $proc.WaitForExit() + + if (Test-Path -Path $lf ) { + foreach($line in Get-Content $lf) { + Log-Msg -msg $line + } + Remove-Item $lf + } + + #Write-Output "Process finished with code " $proc.ExitCode + exit $proc.ExitCode + } + + #parameter sanity check + if( $Args.Count -gt 0 ) { + $Exception = [ArgumentException]::new("Only named parameters are supported: $Args") + throw $Exception + } + + #check WinDefender cmdlets are available + if (!(Check-Command -cmdname 'Add-MpPreference')) { + $Exception = [NotSupportedException ]::new("Windows Defender cmdlets not available") + throw $Exception + } + + $pathsToExclude = New-Object 'System.Collections.Generic.List[String]' + $filesToExclude = New-Object 'System.Collections.Generic.List[String]' + $pathsToInclude = New-Object 'System.Collections.Generic.List[String]' + $filesToRemove = New-Object 'System.Collections.Generic.List[String]' + + if( $PSBoundParameters.Count -gt 0 ) { + + $bAddPath = ![string]::IsNullOrEmpty($AddExclPath) + $bAddFile = ![string]::IsNullOrEmpty($AddExclFile) + $bRmPath = ![string]::IsNullOrEmpty($RmExclPath) + $bRmFile = ![string]::IsNullOrEmpty($RmExclFile) + + if( !$bAddPath -And !$bAddFile -And !$bRmPath -And !$bRmFile ) { + $Exception = [ArgumentException]::new("Invalid parameter(s): $Args") + throw $Exception + } + + #ADD exclusion paths + if( $bAddPath ) { + #foreach ($wdPath in $AddExclPath) { + # $pathsToExclude.Add( $wdPath ) + #} + $pathsToExclude.Add( $AddExclPath ) + } + + #ADD exclusion files + if( $bAddFile ) { + #foreach ($wdFile in $AddExclFile) { + # $filesToExclude.Add( $wdFile ) + #} + $filesToExclude.Add( $AddExclFile ) + } + + #REMOVE exclusion paths + if( $bRmPath ) { + #foreach ($wdPath in $RmExclPath) { + # $pathsToInclude.Add( $wdPath ) + #} + $pathsToInclude.Add( $RmExclPath ) + } + + #ADD exclusion file + if( $bAddFile ) { + #foreach ($wdFile in $RmExclFile) { + # $filesToRemove.Add( $wdFile ) + #} + $filesToRemove.Add( $RmExclFile ) + } + } + #default: throw exception + else { + $Exception = [ArgumentException]::new("Mandatory parameter missing") + throw $Exception + } + + + #to exclude all files opened by a process including the process' binary, a record must be added to both Exclusions/Paths and Exclusions/Processes configurations, see + # https://docs.microsoft.com/en-us/windows/security/threat-protection/windows-defender-antivirus/configure-process-opened-file-exclusions-windows-defender-antivirus : + # "When you add a process to the process exclusion list, Windows Defender Antivirus won't scan files opened by that process, no matter where the files are located. The process itself, however, will be scanned unless it has also been added to the file exclusion list. + #The exclusions only apply to always-on real-time protection and monitoring. They don't apply to scheduled or on-demand scans." + + Log-Msg -msg "Updating Windows Defender real-time scan exclusions:" -logF $logFile + + $itemCount = 0 + + #exclusions + foreach( $exclPath in $pathsToExclude ) { + $exclFiles = Get-ChildItem -Recurse -File -Path $exclPath | % { $_.FullName } + foreach ($exfile in $exclFiles) { + Log-Msg -msg " adding $exfile" -logF $logFile + Add-MpPreference -ExclusionProcess $exfile + Add-MpPreference -ExclusionPath $exfile + $itemCount++ + } + } + + ### ! better run in separate, adding files to exclusion object array from above is very inefficient (forced reallocations) + foreach ($exfile1 in $filesToExclude) { + Log-Msg -msg " adding $exfile1" -logF $logFile + Add-MpPreference -ExclusionProcess $exfile1 + Add-MpPreference -ExclusionPath $exfile1 + $itemCount++ + } + + #inclusions + foreach( $inclPath in $pathsToInclude ) { + $inclFiles = Get-ChildItem -Recurse -File -Path $inclPath | % { $_.FullName } + foreach ($infile in $inclFiles) { + Log-Msg -msg " removing $infile" -logF $logFile + Remove-MpPreference -ExclusionProcess $infile + Remove-MpPreference -ExclusionPath $infile + $itemCount++ + } + } + + ### ! see exclusions + foreach ($infile1 in $filesToExclude) { + Log-Msg -msg " removing $infile1" -logF $logFile + Remove-MpPreference -ExclusionProcess $infile1 + Remove-MpPreference -ExclusionPath $infile1 + $itemCount++ + } + + Log-Msg -msg "Done (processed $itemCount items)" -logF $logFile +} +Catch +{ + if( ![string]::IsNullOrEmpty($logFile) ) { Write-Error -Exception $_.Exception *>> $logFile } + Write-Error -Exception $_.Exception + exit -1 +} From 833822c10bf5654b7c1c285ba21d77b758ce700b Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Mon, 23 Sep 2019 16:10:57 +0200 Subject: [PATCH 12/30] Fix Kconfig issues discovered by upstream Kconfiglib --- components/bootloader/Kconfig.projbuild | 6 +++--- components/bt/Kconfig | 2 +- components/efuse/Kconfig | 2 +- components/esp32/Kconfig | 4 ++-- components/esp_wifi/Kconfig | 6 +++--- components/freertos/Kconfig | 2 +- components/nvs_flash/Kconfig | 2 +- components/openssl/Kconfig | 2 +- components/partition_table/Kconfig.projbuild | 10 +++++----- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/components/bootloader/Kconfig.projbuild b/components/bootloader/Kconfig.projbuild index 3aaa3eaf8..c46a31077 100644 --- a/components/bootloader/Kconfig.projbuild +++ b/components/bootloader/Kconfig.projbuild @@ -388,7 +388,7 @@ menu "Security features" config SECURE_BOOT_SIGNING_KEY string "Secure boot private signing key" depends on SECURE_BOOT_BUILD_SIGNED_BINARIES - default secure_boot_signing_key.pem + default "secure_boot_signing_key.pem" help Path to the key file used to sign app images. @@ -404,7 +404,7 @@ menu "Security features" config SECURE_BOOT_VERIFICATION_KEY string "Secure boot public signature verification key" depends on SECURE_SIGNED_APPS && !SECURE_BOOT_BUILD_SIGNED_BINARIES - default signature_verification_key.bin + default "signature_verification_key.bin" help Path to a public key file used to verify signed images. This key is compiled into the bootloader and/or app, to verify app images. @@ -418,7 +418,7 @@ menu "Security features" choice SECURE_BOOTLOADER_KEY_ENCODING bool "Hardware Key Encoding" depends on SECURE_BOOTLOADER_REFLASHABLE - default SECURE_BOOTLOADER_NO_ENCODING + default SECURE_BOOTLOADER_KEY_ENCODING_256BIT help In reflashable secure bootloader mode, a hardware key is derived from the signing key (with SHA-256) and diff --git a/components/bt/Kconfig b/components/bt/Kconfig index 8553cc9f7..a1ab51c74 100644 --- a/components/bt/Kconfig +++ b/components/bt/Kconfig @@ -1,4 +1,4 @@ -menu Bluetooth +menu "Bluetooth" config BT_ENABLED bool "Bluetooth" diff --git a/components/efuse/Kconfig b/components/efuse/Kconfig index bf40e2dfc..b4f83c883 100644 --- a/components/efuse/Kconfig +++ b/components/efuse/Kconfig @@ -9,7 +9,7 @@ menu "eFuse Bit Manager" config EFUSE_CUSTOM_TABLE_FILENAME string "Custom eFuse CSV file" depends on EFUSE_CUSTOM_TABLE - default main/esp_efuse_custom_table.csv + default "main/esp_efuse_custom_table.csv" help Name of the custom eFuse CSV filename. This path is evaluated relative to the project root directory. diff --git a/components/esp32/Kconfig b/components/esp32/Kconfig index 5a512f562..acc3f073c 100644 --- a/components/esp32/Kconfig +++ b/components/esp32/Kconfig @@ -117,7 +117,7 @@ menu "ESP32-specific" choice SPIRAM_SPEED prompt "Set RAM clock speed" - default SPIRAM_CACHE_SPEED_40M + default SPIRAM_SPEED_40M help Select the speed for the SPI RAM chip. If SPI RAM is enabled, we only support three combinations of SPI speed mode we supported now: @@ -487,7 +487,7 @@ menu "ESP32-specific" choice ESP32_BROWNOUT_DET_LVL_SEL prompt "Brownout voltage level" depends on ESP32_BROWNOUT_DET - default BROWNOUT_DET_LVL_SEL_25 + default ESP32_BROWNOUT_DET_LVL_SEL_0 help The brownout detector will reset the chip when the supply voltage is approximately below this level. Note that there may be some variation of brownout voltage level diff --git a/components/esp_wifi/Kconfig b/components/esp_wifi/Kconfig index 104d12ce7..2b19873cb 100644 --- a/components/esp_wifi/Kconfig +++ b/components/esp_wifi/Kconfig @@ -1,5 +1,5 @@ -menu Wi-Fi +menu "Wi-Fi" config ESP32_WIFI_SW_COEXIST_ENABLE bool "Software controls WiFi/Bluetooth coexistence" @@ -240,7 +240,7 @@ menu Wi-Fi choice ESP32_WIFI_DEBUG_LOG_LEVEL depends on ESP32_WIFI_DEBUG_LOG_ENABLE prompt "WiFi debug log level" - default ESP32_WIFI_LOG_DEBUG + default ESP32_WIFI_DEBUG_LOG_DEBUG help The WiFi log is divided into the following levels: ERROR,WARNING,INFO,DEBUG,VERBOSE. The ERROR,WARNING,INFO levels are enabled by default, and the DEBUG,VERBOSE levels can be enabled here. @@ -318,7 +318,7 @@ menu Wi-Fi endmenu # Wi-Fi -menu PHY +menu "PHY" config ESP32_PHY_CALIBRATION_AND_DATA_STORAGE bool "Store phy calibration data in NVS" diff --git a/components/freertos/Kconfig b/components/freertos/Kconfig index bf6317d45..1341dc10c 100644 --- a/components/freertos/Kconfig +++ b/components/freertos/Kconfig @@ -19,7 +19,7 @@ menu "FreeRTOS" choice FREERTOS_CORETIMER prompt "Xtensa timer to use as the FreeRTOS tick source" - default CONFIG_FREERTOS_CORETIMER_0 + default FREERTOS_CORETIMER_0 help FreeRTOS needs a timer with an associated interrupt to use as the main tick source to increase counters, run timers and do diff --git a/components/nvs_flash/Kconfig b/components/nvs_flash/Kconfig index 0ae533d4c..e39dc9cf7 100644 --- a/components/nvs_flash/Kconfig +++ b/components/nvs_flash/Kconfig @@ -1,4 +1,4 @@ -menu NVS +menu "NVS" config NVS_ENCRYPTION bool "Enable NVS encryption" diff --git a/components/openssl/Kconfig b/components/openssl/Kconfig index 0df49080c..7bb271f28 100644 --- a/components/openssl/Kconfig +++ b/components/openssl/Kconfig @@ -33,7 +33,7 @@ menu "OpenSSL" choice OPENSSL_ASSERT prompt "Select OpenSSL assert function" - default CONFIG_OPENSSL_ASSERT_EXIT + default OPENSSL_ASSERT_EXIT help OpenSSL function needs "assert" function to check if input parameters are valid. diff --git a/components/partition_table/Kconfig.projbuild b/components/partition_table/Kconfig.projbuild index 326f7fb7c..6dfa8a562 100644 --- a/components/partition_table/Kconfig.projbuild +++ b/components/partition_table/Kconfig.projbuild @@ -22,17 +22,17 @@ menu "Partition Table" config PARTITION_TABLE_CUSTOM_FILENAME string "Custom partition CSV file" if PARTITION_TABLE_CUSTOM - default partitions.csv + default "partitions.csv" help Name of the custom partition CSV filename. This path is evaluated relative to the project root directory. config PARTITION_TABLE_FILENAME string - default partitions_singleapp.csv if PARTITION_TABLE_SINGLE_APP && !ESP32_ENABLE_COREDUMP_TO_FLASH - default partitions_singleapp_coredump.csv if PARTITION_TABLE_SINGLE_APP && ESP32_ENABLE_COREDUMP_TO_FLASH - default partitions_two_ota.csv if PARTITION_TABLE_TWO_OTA && !ESP32_ENABLE_COREDUMP_TO_FLASH - default partitions_two_ota_coredump.csv if PARTITION_TABLE_TWO_OTA && ESP32_ENABLE_COREDUMP_TO_FLASH + default "partitions_singleapp.csv" if PARTITION_TABLE_SINGLE_APP && !ESP32_ENABLE_COREDUMP_TO_FLASH + default "partitions_singleapp_coredump.csv" if PARTITION_TABLE_SINGLE_APP && ESP32_ENABLE_COREDUMP_TO_FLASH + default "partitions_two_ota.csv" if PARTITION_TABLE_TWO_OTA && !ESP32_ENABLE_COREDUMP_TO_FLASH + default "partitions_two_ota_coredump.csv" if PARTITION_TABLE_TWO_OTA && ESP32_ENABLE_COREDUMP_TO_FLASH default PARTITION_TABLE_CUSTOM_FILENAME if PARTITION_TABLE_CUSTOM config PARTITION_TABLE_OFFSET From 81ffc873711caa56dd9819089b458424050752c9 Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Wed, 4 Sep 2019 23:36:47 +0200 Subject: [PATCH 13/30] idf_tools.py: pass --work-tree instead of -C option to git Older versions of git do not support -C option. Use --work-tree option instead. Closes https://github.com/espressif/esp-idf/issues/4018 Merges https://github.com/espressif/esp-idf/pull/4019 --- tools/idf_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index 3d2341237..ef292cf77 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -810,7 +810,7 @@ def get_python_env_path(): with open(version_file_path, "r") as version_file: idf_version_str = version_file.read() else: - idf_version_str = subprocess.check_output(['git', '-C', global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode() + idf_version_str = subprocess.check_output(['git', '--work-tree=' + global_idf_path, 'describe', '--tags'], cwd=global_idf_path, env=os.environ).decode() match = re.match(r'^v([0-9]+\.[0-9]+).*', idf_version_str) idf_version = match.group(1) From 3bbb758bc5ff70e97ae0629830513f62cf4ffaa2 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 23 Sep 2019 06:01:09 +0200 Subject: [PATCH 14/30] idf_tools.py: add workaround for PermissionError in os.rename Closes https://github.com/espressif/esp-idf/issues/4063 Closes https://github.com/espressif/esp-idf/issues/3819 --- tools/idf_tools.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index ef292cf77..a04cc1c9c 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -61,6 +61,12 @@ try: except ImportError: from urllib import urlretrieve +try: + from exceptions import WindowsError +except ImportError: + class WindowsError(OSError): + pass + TOOLS_FILE = 'tools/tools.json' TOOLS_SCHEMA_FILE = 'tools/tools_schema.json' @@ -254,13 +260,34 @@ def unpack(filename, destination): archive_obj.extractall(destination) +# Sometimes renaming a directory on Windows (randomly?) causes a PermissionError. +# This is confirmed to be a workaround: +# https://github.com/espressif/esp-idf/issues/3819#issuecomment-515167118 +# https://github.com/espressif/esp-idf/issues/4063#issuecomment-531490140 +# https://stackoverflow.com/a/43046729 +def rename_with_retry(path_from, path_to): + if sys.platform.startswith('win'): + retry_count = 100 + else: + retry_count = 1 + + for retry in range(retry_count): + try: + os.rename(path_from, path_to) + return + except (OSError, WindowsError): # WindowsError until Python 3.3, then OSError + if retry == retry_count - 1: + raise + warn('Rename {} to {} failed, retrying...'.format(path_from, path_to)) + + def strip_container_dirs(path, levels): assert levels > 0 # move the original directory out of the way (add a .tmp suffix) tmp_path = path + '.tmp' if os.path.exists(tmp_path): shutil.rmtree(tmp_path) - os.rename(path, tmp_path) + rename_with_retry(path, tmp_path) os.mkdir(path) base_path = tmp_path # walk given number of levels down @@ -276,7 +303,7 @@ def strip_container_dirs(path, levels): for name in contents: move_from = os.path.join(base_path, name) move_to = os.path.join(path, name) - os.rename(move_from, move_to) + rename_with_retry(move_from, move_to) shutil.rmtree(tmp_path) @@ -544,7 +571,7 @@ class IDFTool(object): if not self.check_download_file(download_obj, local_temp_path): warn('Failed to download file {}'.format(local_temp_path)) continue - os.rename(local_temp_path, local_path) + rename_with_retry(local_temp_path, local_path) downloaded = True break if not downloaded: From c780abc6a6ae4816d52406885a8c6d67b297699b Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 23 Sep 2019 05:42:01 +0200 Subject: [PATCH 15/30] tools: update Windows installer build script Replace the outdated build_installer.sh with the steps used in CI, call build_installer.sh from CI. Move the signing part into the new script, sign_installer.sh. --- tools/ci/config/build.yml | 16 +--- tools/ci/executable-list.txt | 1 + tools/windows/tool_setup/README.md | 19 ++++- tools/windows/tool_setup/build_installer.sh | 84 ++++++++------------- tools/windows/tool_setup/sign_installer.sh | 49 ++++++++++++ 5 files changed, 98 insertions(+), 71 deletions(-) create mode 100755 tools/windows/tool_setup/sign_installer.sh diff --git a/tools/ci/config/build.yml b/tools/ci/config/build.yml index 184e8b7b2..77802271e 100644 --- a/tools/ci/config/build.yml +++ b/tools/ci/config/build.yml @@ -372,19 +372,5 @@ build_installer: - build_cmdlinerunner before_script: [] script: - - mkdir idf_tools_tmp - - export IDF_TOOLS_PATH=$PWD/idf_tools_tmp - - tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all - - tools/idf_tools.py --tools-json tools/windows/tool_setup/tools_fallback.json --non-interactive download --platform Windows-x86_64 all - - mkdir tools/windows/tool_setup/dist - - mv idf_tools_tmp/dist/* tools/windows/tool_setup/dist/ - - cd tools/windows/tool_setup/ - - mkdir unzip - - cd unzip - - wget --no-verbose https://www.7-zip.org/a/7z1900-extra.7z - - 7zr e -y 7z1900-extra.7z - - cd .. - - - wget --no-verbose https://dl.espressif.com/dl/esp-idf/idf_versions.txt - - iscc idf_tool_setup.iss + - ./build_installer.sh diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index a66c30b7a..20f34357e 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -83,3 +83,4 @@ tools/test_idf_tools/test_idf_tools.py tools/unit-test-app/unit_test.py tools/windows/eclipse_make.sh tools/windows/tool_setup/build_installer.sh +tools/windows/tool_setup/sign_installer.sh diff --git a/tools/windows/tool_setup/README.md b/tools/windows/tool_setup/README.md index f9e71e9b4..d7118f695 100644 --- a/tools/windows/tool_setup/README.md +++ b/tools/windows/tool_setup/README.md @@ -14,7 +14,17 @@ Some functionality of the installer depends on additional programs: * [cmdlinerunner](cmdlinerunner/cmdlinerunner.c) — a helper DLL used to run external command line programs from the installer, capture live console output, and get the exit code. -## Steps required to build the installer +## Building the installer + +### In Docker + +This uses `wine-innosetup` Docker image and `build_installer.sh` script. This is how the installer is built in CI. + +``` +docker run --rm -v $IDF_PATH:/idf -w /idf/tools/windows/tool_setup -it $CI_DOCKER_REGISTRY/wine-innosetup:1 /bin/bash build_installer.sh +``` + +### Manually, step by step * Build cmdlinerunner DLL. - On Linux/Mac, install mingw-w64 toolchain (`i686-w64-mingw32-gcc`). Then build the DLL using CMake: @@ -35,5 +45,10 @@ Some functionality of the installer depends on additional programs: * Build the installer using Inno Setup Compiler: `ISCC.exe idf_tools_setup.iss`. -* Obtain the signing keys, then sign `Output/esp-idf-tools-setup-unsigned.exe`. +## Signing the installer +* Obtain the signing key (e.g `key.pem`) and the certificate chain (e.g. `certchain.pem`). Set the environment variables to point to these files: + - `export KEYFILE=key.pem` + - `export CERTCHAIN=certchain.pem` + +* Run `sign_installer.sh` script. This will ask for the `key.pem` password, and produce the signed installer in the Output directory. If you plan to run the script multiple times, you may also set `KEYPASSWORD` environment variable to the `key.pem` password, to avoid the prompt. diff --git a/tools/windows/tool_setup/build_installer.sh b/tools/windows/tool_setup/build_installer.sh index 270fa8efd..b3e089cc1 100755 --- a/tools/windows/tool_setup/build_installer.sh +++ b/tools/windows/tool_setup/build_installer.sh @@ -1,67 +1,43 @@ #!/bin/bash # -# Setup script to build Windows tool installer with Inno Setup +# Script to build the IDF Tools installer for Windows with Inno Setup. +# This script should be executed inside wine-innosetup docker image. # -# Designed to be run on Linux (with wine) but could be adapted to run under MSYS2 on Windows -# pretty easily... -# -# - Downloads (if necessary) all tools to install to the "dl/" directory -# - Deletes the "input" directory contains and copies everything under there +# - Downloads all tools to install into the "dist/" directory +# - Downloads 7z and idf_versions.txt # - Runs ISCC under wine to compile the installer itself -set -e -if [ -z "${KEYPASSWORD}" ]; then - echo "KEYPASSWORD should be set" +set -e +set -u + +iscc_path=$(which iscc) +if [[ -z "$iscc_path" ]]; then + echo "Inno setup compiler (iscc) not found. Are you running wine-innosetup Docker image?" exit 1 fi -if [ "$1" != "--no-download" ]; then - - mkdir -p dl input - - cd `dirname $0` - pushd dl - wget --continue "https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip" - wget --continue "https://github.com/espressif/binutils-esp32ulp/releases/download/v2.28.51-esp32ulp-20180809/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip" - wget --continue "https://github.com/espressif/openocd-esp32/releases/download/v0.10.0-esp32-20180920/openocd-esp32-win32-0.10.0-esp32-20180920.zip" - wget --continue "https://github.com/espressif/kconfig-frontends/releases/download/v4.6.0.0-idf-20180525/mconf-v4.6.0.0-idf-20180525-win32.zip" - wget --continue "https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip" - popd - - rm -rf input/* - pushd input - unzip ../dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip - unzip ../dl/mconf-v4.6.0.0-idf-20180525-win32.zip - unzip ../dl/binutils-esp32ulp-win32-2.28.51-esp32ulp-20180809.zip - unzip ../dl/openocd-esp32-win32-0.10.0-esp32-20180920.zip - unzip ../dl/ninja-win.zip - popd +if [[ -z "${IDF_PATH:-}" ]]; then + export IDF_PATH=$(cd ../../../; pwd) + echo "Assuming IDF_PATH: ${IDF_PATH}" fi -wine "C:\Program Files\Inno Setup 5\ISCC.exe" "`winepath -w ./idf_tool_setup.iss`" +echo "Downloading IDF Tools..." +mkdir -p idf_tools_tmp +export IDF_TOOLS_PATH=$PWD/idf_tools_tmp +$IDF_PATH/tools/idf_tools.py --non-interactive download --platform Windows-x86_64 all +$IDF_PATH/tools/idf_tools.py --tools-json tools_fallback.json --non-interactive download --platform Windows-x86_64 all +mkdir -p dist +cp idf_tools_tmp/dist/* dist/ -# sign the installer with osslsigncode, parsing the version number out of the -# installer config +echo "Downloading 7z..." +mkdir -p unzip +pushd unzip +wget --no-verbose -O 7z1900-extra.7z https://www.7-zip.org/a/7z1900-extra.7z +7zr e -y 7z1900-extra.7z +popd -VERSION=`grep "^AppVersion=" idf_tool_setup.iss | cut -d'=' -f2` +echo "Downloading idf_versions.txt..." +wget --no-verbose -O idf_versions.txt https://dl.espressif.com/dl/esp-idf/idf_versions.txt -echo "Signing installer..." - -# Note: The cert chain passed to -certs needs to contain the intermediate -# cert(s) as well, appended after the code signing cert, or Windows may see -# it as "Unknown Publisher" -# -# See https://stackoverflow.com/a/52637050 for full details -# -umask 770 # for the process substitution FIFO - -osslsigncode -certs ./keys/certchain.pem -key ./keys/key.pem \ - -readpass <(echo "$KEYPASSWORD") \ - -in Output/esp-idf-tools-setup-unsigned.exe \ - -out Output/esp-idf-tools-setup-${VERSION}.exe \ - -h sha256 \ - -n "Espressif Systems (Shanghai) Pte. Ltd." \ - -i "https://www.espressif.com/" \ - -ts http://timestamp.digicert.com - -chmod 644 Output/esp-idf-tools-setup-${VERSION}.exe # make up for the umask +echo "Running ISCC..." +iscc idf_tool_setup.iss diff --git a/tools/windows/tool_setup/sign_installer.sh b/tools/windows/tool_setup/sign_installer.sh new file mode 100755 index 000000000..59298edf3 --- /dev/null +++ b/tools/windows/tool_setup/sign_installer.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Script to sign the IDF Tools installer for Windows, built with build_installer.sh. +# + +set -e +set -u + +if [[ -z "${KEYFILE:-}" || -z "${CERTCHAIN:-}" ]]; then + echo "To sign the installer, set the following environment variables:" + echo " KEYFILE - private key file" + echo " KEYPASSWORD - password for the private key file (optional, will prompt for password if not set)" + echo " CERTCHAIN - certificate chain file" + exit 1 +fi + +umask 770 # for the process substitution FIFO + +VERSION=`grep "#define MyAppVersion " idf_tool_setup.iss | cut -d ' ' -f3 | tr -d '"'` +echo "Installer version ${VERSION}" + +IN_FILE="Output/esp-idf-tools-setup-unsigned.exe" +OUT_FILE="Output/esp-idf-tools-setup-${VERSION}.exe" + +if [[ -n "${KEYPASSWORD:-}" ]]; then + PASSARG="-readpass <(echo \"$KEYPASSWORD\")" +else + PASSARG="-askpass" +fi + +echo "Signing the installer (${IN_FILE})..." +# Note: The cert chain passed to -certs needs to contain the intermediate +# cert(s) as well, appended after the code signing cert, or Windows may see +# it as "Unknown Publisher" +# +# See https://stackoverflow.com/a/52637050 for full details +# +osslsigncode -certs ${CERTCHAIN} -key ${KEYFILE} \ + ${PASSARG} \ + -in ${IN_FILE} \ + -out ${OUT_FILE} \ + -h sha256 \ + -n "Espressif Systems (Shanghai) Co., Ltd." \ + -i "https://www.espressif.com/" \ + -ts http://timestamp.digicert.com + +chmod 644 ${OUT_FILE} # make up for the umask + +echo "Generated ${OUT_FILE}" From f7cae13d2a87fd62dd15d5739e3ee2e8e87655b8 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 23 Sep 2019 05:45:49 +0200 Subject: [PATCH 16/30] tools: bump windows tools installer version to v2.1 Includes the following changes: 80ad09f23 allow changing the installation path (a fix for https://github.com/espressif/esp-idf/issues/3806) 9c5284e7b verify that IDF_PATH doesn't contain spaces aaf3dcbda fix quoting of IDF_TOOLS_PATH (a fix for https://github.com/espressif/esp-idf/issues/3807) e6e179294 tools: update idf_exe to 1.0.1 76dc87e9a idf_exe: fix NULL pointer passed to WriteFile (fixes for https://github.com/espressif/esp-idf/issues/3740) --- tools/windows/tool_setup/idf_tool_setup.iss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/windows/tool_setup/idf_tool_setup.iss b/tools/windows/tool_setup/idf_tool_setup.iss index 25a997069..b61dc8058 100644 --- a/tools/windows/tool_setup/idf_tool_setup.iss +++ b/tools/windows/tool_setup/idf_tool_setup.iss @@ -5,7 +5,7 @@ #include #define MyAppName "ESP-IDF Tools" -#define MyAppVersion "2.0" +#define MyAppVersion "2.1" #define MyAppPublisher "Espressif Systems (Shanghai) Co. Ltd." #define MyAppURL "https://github.com/espressif/esp-idf" From 5a916ce12656cb7b171ea2c1262679ae1eb3519c Mon Sep 17 00:00:00 2001 From: Roland Dobai Date: Mon, 22 Jul 2019 16:04:03 +0200 Subject: [PATCH 17/30] Support ELF files loadable with gdb --- .gitlab-ci.yml | 2 +- Kconfig | 77 +++++++++++ components/bootloader/CMakeLists.txt | 2 +- components/bootloader/project_include.cmake | 4 +- components/esp32/Kconfig | 5 + components/esp32/clk.c | 4 +- components/esp32/cpu_start.c | 45 ++++-- components/esp32/ld/esp32.ld | 16 +++ components/esp32/ld/esp32.project.ld.in | 28 +++- components/esp32/ld/esp32_fragments.lf | 8 +- components/esp32/panic.c | 2 +- .../esp_rom/esp32/ld/esp32.rom.spiflash.ld | 1 + components/esptool_py/CMakeLists.txt | 2 +- components/esptool_py/Makefile.projbuild | 4 + components/esptool_py/project_include.cmake | 28 ++-- components/soc/src/memory_layout_utils.c | 6 +- examples/get-started/hello_world/.gdbinit.ci | 13 ++ .../hello_world/loadable_elf_example_test.py | 129 ++++++++++++++++++ examples/get-started/hello_world/sdkconfig.ci | 6 + make/project.mk | 10 ++ tools/ci/config/target-test.yml | 13 ++ tools/idf.py | 4 + tools/tiny-test-fw/IDF/IDFApp.py | 25 ++-- .../components/test_utils/test_runner.c | 8 ++ 24 files changed, 387 insertions(+), 55 deletions(-) create mode 100644 examples/get-started/hello_world/.gdbinit.ci create mode 100644 examples/get-started/hello_world/loadable_elf_example_test.py create mode 100644 examples/get-started/hello_world/sdkconfig.ci diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bf113b94..5d94e6231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,7 +66,7 @@ variables: rm -rf "$CUSTOM_TOOLCHAIN_PATH" .setup_tools_unless_target_test: &setup_tools_unless_target_test | - if [ "$CI_JOB_STAGE" != "target_test" ]; then + if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then tools/idf_tools.py --non-interactive install && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1 fi diff --git a/Kconfig b/Kconfig index 129a0abc6..92c76c38d 100644 --- a/Kconfig +++ b/Kconfig @@ -68,6 +68,83 @@ mainmenu "Espressif IoT Development Framework Configuration" endmenu # SDK tool configuration + menu "Build type" + + choice APP_BUILD_TYPE + prompt "Application build type" + default APP_BUILD_TYPE_APP_2NDBOOT + help + Select the way the application is built. + + By default, the application is built as a binary file in a format compatible with + the ESP32 bootloader. In addition to this application, 2nd stage bootloader is + also built. Application and bootloader binaries can be written into flash and + loaded/executed from there. + + Another option, useful for only very small and limited applications, is to only link + the .elf file of the application, such that it can be loaded directly into RAM over + JTAG. Note that since IRAM and DRAM sizes are very limited, it is not possible to + build any complex application this way. However for kinds of testing and debugging, + this option may provide faster iterations, since the application does not need to be + written into flash. + Note that at the moment, ESP-IDF does not contain all the startup code required to + initialize the CPUs and ROM memory (data/bss). Therefore it is necessary to execute + a bit of ROM code prior to executing the application. A gdbinit file may look as follows: + + # Connect to a running instance of OpenOCD + target remote :3333 + # Reset and halt the target + mon reset halt + # Run to a specific point in ROM code, + # where most of initialization is complete. + thb *0x40007901 + c + # Load the application into RAM + load + # Run till app_main + tb app_main + c + + Execute this gdbinit file as follows: + + xtensa-esp32-elf-gdb build/app-name.elf -x gdbinit + + Recommended sdkconfig.defaults for building loadable ELF files is as follows. + CONFIG_APP_BUILD_TYPE_ELF_RAM is required, other options help reduce application + memory footprint. + + CONFIG_APP_BUILD_TYPE_ELF_RAM=y + CONFIG_VFS_SUPPORT_TERMIOS= + CONFIG_NEWLIB_NANO_FORMAT=y + CONFIG_ESP32_PANIC_PRINT_HALT=y + CONFIG_ESP32_DEBUG_STUBS_ENABLE= + CONFIG_ESP_ERR_TO_NAME_LOOKUP= + + + config APP_BUILD_TYPE_APP_2NDBOOT + bool + prompt "Default (binary application + 2nd stage bootloader)" + select APP_BUILD_GENERATE_BINARIES + select APP_BUILD_BOOTLOADER + select APP_BUILD_USE_FLASH_SECTIONS + + config APP_BUILD_TYPE_ELF_RAM + bool + prompt "ELF file, loadable into RAM (EXPERIMENTAL))" + endchoice # APP_BUILD_TYPE + + # Hidden options, set according to the choice above + config APP_BUILD_GENERATE_BINARIES + bool # Whether to generate .bin files or not + + config APP_BUILD_BOOTLOADER + bool # Whether to build the bootloader + + config APP_BUILD_USE_FLASH_SECTIONS + bool # Whether to place code/data into memory-mapped flash sections + + endmenu # Build type + source "$COMPONENT_KCONFIGS_PROJBUILD" menu "Compiler options" diff --git a/components/bootloader/CMakeLists.txt b/components/bootloader/CMakeLists.txt index afe1ad639..bc80cbbd9 100644 --- a/components/bootloader/CMakeLists.txt +++ b/components/bootloader/CMakeLists.txt @@ -1,7 +1,7 @@ idf_component_register(PRIV_REQUIRES partition_table) # Do not generate flash file when building bootloader or is in early expansion of the build -if(BOOTLOADER_BUILD) +if(BOOTLOADER_BUILD OR NOT CONFIG_APP_BUILD_BOOTLOADER) return() endif() diff --git a/components/bootloader/project_include.cmake b/components/bootloader/project_include.cmake index fa26fd097..d883a234e 100644 --- a/components/bootloader/project_include.cmake +++ b/components/bootloader/project_include.cmake @@ -1,7 +1,7 @@ set(BOOTLOADER_OFFSET 0x1000) # Do not generate flash file when building bootloader -if(BOOTLOADER_BUILD) +if(BOOTLOADER_BUILD OR NOT CONFIG_APP_BUILD_BOOTLOADER) return() endif() @@ -125,4 +125,4 @@ endif() # So for now we just have the top-level build remove the final build products... set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES - ${bootloader_binary_files}) \ No newline at end of file + ${bootloader_binary_files}) diff --git a/components/esp32/Kconfig b/components/esp32/Kconfig index 5a512f562..b58e5cfe6 100644 --- a/components/esp32/Kconfig +++ b/components/esp32/Kconfig @@ -738,6 +738,11 @@ menu "ESP32-specific" Enabling this setting adds approximately 1KB to the app's IRAM usage. + config ESP32_APP_INIT_CLK + bool + default y if ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS + default y if APP_BUILD_TYPE_ELF_RAM + config ESP32_RTCDATA_IN_FAST_MEM bool "Place RTC_DATA_ATTR and RTC_RODATA_ATTR variables into RTC fast memory segment" default n diff --git a/components/esp32/clk.c b/components/esp32/clk.c index fd5c47761..28bcbec7f 100644 --- a/components/esp32/clk.c +++ b/components/esp32/clk.c @@ -75,7 +75,7 @@ void esp_clk_init(void) rtc_config_t cfg = RTC_CONFIG_DEFAULT(); rtc_init(cfg); -#ifdef CONFIG_ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS +#if (CONFIG_ESP32_COMPATIBLE_PRE_V2_1_BOOTLOADERS || CONFIG_ESP32_APP_INIT_CLK) /* Check the bootloader set the XTAL frequency. Bootloaders pre-v2.1 don't do this. @@ -293,6 +293,8 @@ void esp_perip_clk_init(void) DPORT_I2S1_CLK_EN | DPORT_SPI_DMA_CLK_EN; + common_perip_clk &= ~DPORT_SPI01_CLK_EN; + #if CONFIG_SPIRAM_SPEED_80M //80MHz SPIRAM uses SPI2/SPI3 as well; it's initialized before this is called. Because it is used in //a weird mode where clock to the peripheral is disabled but reset is also disabled, it 'hangs' diff --git a/components/esp32/cpu_start.c b/components/esp32/cpu_start.c index e3a4bc11e..ebea01b62 100644 --- a/components/esp32/cpu_start.c +++ b/components/esp32/cpu_start.c @@ -72,6 +72,11 @@ #include "esp_efuse.h" #include "bootloader_flash_config.h" +#ifdef CONFIG_APP_BUILD_TYPE_ELF_RAM +#include "esp32/rom/efuse.h" +#include "esp32/rom/spi_flash.h" +#endif // CONFIG_APP_BUILD_TYPE_ELF_RAM + #define STRINGIFY(s) STRINGIFY2(s) #define STRINGIFY2(s) #s @@ -391,6 +396,32 @@ void start_cpu0_default(void) #ifndef CONFIG_FREERTOS_UNICORE esp_dport_access_int_init(); #endif + + bootloader_flash_update_id(); +#if !CONFIG_SPIRAM_BOOT_INIT + // Read the application binary image header. This will also decrypt the header if the image is encrypted. + esp_image_header_t fhdr = {0}; +#ifdef CONFIG_APP_BUILD_TYPE_ELF_RAM + fhdr.spi_mode = ESP_IMAGE_SPI_MODE_DIO; + fhdr.spi_speed = ESP_IMAGE_SPI_SPEED_40M; + fhdr.spi_size = ESP_IMAGE_FLASH_SIZE_4MB; + + extern void esp_rom_spiflash_attach(uint32_t, bool); + esp_rom_spiflash_attach(ets_efuse_get_spiconfig(), false); + esp_rom_spiflash_unlock(); +#else + // This assumes that DROM is the first segment in the application binary, i.e. that we can read + // the binary header through cache by accessing SOC_DROM_LOW address. + memcpy(&fhdr, (void*) SOC_DROM_LOW, sizeof(fhdr)); +#endif // CONFIG_APP_BUILD_TYPE_ELF_RAM + + // If psram is uninitialized, we need to improve some flash configuration. + bootloader_flash_clock_config(&fhdr); + bootloader_flash_gpio_config(&fhdr); + bootloader_flash_dummy_config(&fhdr); + bootloader_flash_cs_timing_config(); +#endif //!CONFIG_SPIRAM_BOOT_INIT + spi_flash_init(); /* init default OS-aware flash access critical section */ spi_flash_guard_set(&g_flash_guard_default_ops); @@ -424,20 +455,6 @@ void start_cpu0_default(void) esp_coex_adapter_register(&g_coex_adapter_funcs); #endif - bootloader_flash_update_id(); -#if !CONFIG_SPIRAM_BOOT_INIT - // Read the application binary image header. This will also decrypt the header if the image is encrypted. - esp_image_header_t fhdr = {0}; - // This assumes that DROM is the first segment in the application binary, i.e. that we can read - // the binary header through cache by accessing SOC_DROM_LOW address. - memcpy(&fhdr, (void*) SOC_DROM_LOW, sizeof(fhdr)); - // If psram is uninitialized, we need to improve some flash configuration. - bootloader_flash_clock_config(&fhdr); - bootloader_flash_gpio_config(&fhdr); - bootloader_flash_dummy_config(&fhdr); - bootloader_flash_cs_timing_config(); -#endif - portBASE_TYPE res = xTaskCreatePinnedToCore(&main_task, "main", ESP_TASK_MAIN_STACK, NULL, ESP_TASK_MAIN_PRIO, NULL, 0); diff --git a/components/esp32/ld/esp32.ld b/components/esp32/ld/esp32.ld index a52b56f90..0400fc2ab 100644 --- a/components/esp32/ld/esp32.ld +++ b/components/esp32/ld/esp32.ld @@ -49,6 +49,7 @@ MEMORY /* IRAM for PRO cpu. Not sure if happy with this, this is MMU area... */ iram0_0_seg (RX) : org = 0x40080000, len = 0x20000 +#ifdef CONFIG_APP_BUILD_USE_FLASH_SECTIONS /* Even though the segment name is iram, it is actually mapped to flash */ iram0_2_seg (RX) : org = 0x400D0018, len = 0x330000-0x18 @@ -58,6 +59,7 @@ MEMORY which is flashed to the chip has a 0x18 byte file header. Setting this offset makes it simple to meet the flash cache MMU's constraint that (paddr % 64KB == vaddr % 64KB).) */ +#endif // CONFIG_APP_BUILD_USE_FLASH_SECTIONS /* Shared data RAM, excluding memory reserved for ROM bss/data/stack. @@ -72,10 +74,12 @@ MEMORY dram0_0_seg (RW) : org = 0x3FFB0000 + CONFIG_BT_RESERVE_DRAM, len = DRAM0_0_SEG_LEN - CONFIG_BT_RESERVE_DRAM +#ifdef CONFIG_APP_BUILD_USE_FLASH_SECTIONS /* Flash mapped constant data */ drom0_0_seg (R) : org = 0x3F400018, len = 0x400000-0x18 /* (See iram0_2_seg for meaning of 0x18 offset in the above.) */ +#endif // CONFIG_APP_BUILD_USE_FLASH_SECTIONS /* RTC fast memory (executable). Persists over deep sleep. */ @@ -116,3 +120,15 @@ REGION_ALIAS("rtc_data_location", rtc_slow_seg ); #else REGION_ALIAS("rtc_data_location", rtc_data_seg ); #endif + +#ifdef CONFIG_APP_BUILD_USE_FLASH_SECTIONS + REGION_ALIAS("default_code_seg", iram0_2_seg); +#else + REGION_ALIAS("default_code_seg", iram0_0_seg); +#endif // CONFIG_APP_BUILD_USE_FLASH_SECTIONS + +#ifdef CONFIG_APP_BUILD_USE_FLASH_SECTIONS + REGION_ALIAS("default_rodata_seg", drom0_0_seg); +#else + REGION_ALIAS("default_rodata_seg", dram0_0_seg); +#endif // CONFIG_APP_BUILD_USE_FLASH_SECTIONS diff --git a/components/esp32/ld/esp32.project.ld.in b/components/esp32/ld/esp32.project.ld.in index 872cecd33..b8e710e07 100644 --- a/components/esp32/ld/esp32.project.ld.in +++ b/components/esp32/ld/esp32.project.ld.in @@ -161,12 +161,8 @@ SECTIONS mapping[iram0_text] _iram_text_end = ABSOLUTE(.); - _iram_end = ABSOLUTE(.); } > iram0_0_seg - ASSERT(((_iram_text_end - ORIGIN(iram0_0_seg)) <= LENGTH(iram0_0_seg)), - "IRAM0 segment data does not fit.") - .dram0.data : { _data_start = ABSOLUTE(.); @@ -312,7 +308,7 @@ SECTIONS *(.tbss.*) _thread_local_end = ABSOLUTE(.); . = ALIGN(4); - } >drom0_0_seg + } >default_rodata_seg .flash.text : { @@ -334,5 +330,25 @@ SECTIONS the flash.text segment. */ _flash_cache_start = ABSOLUTE(0); - } >iram0_2_seg + } >default_code_seg + + /* Marks the end of IRAM code segment */ + .iram0.text_end (NOLOAD) : + { + . = ALIGN (4); + _iram_end = ABSOLUTE(.); + } > iram0_0_seg + + /* Marks the end of data, bss and possibly rodata */ + .dram0.heap_start (NOLOAD) : + { + . = ALIGN (8); + _heap_start = ABSOLUTE(.); + } > dram0_0_seg } + +ASSERT(((_iram_text_end - ORIGIN(iram0_0_seg)) <= LENGTH(iram0_0_seg)), + "IRAM0 segment data does not fit.") + +ASSERT(((_heap_start - ORIGIN(dram0_0_seg)) <= LENGTH(dram0_0_seg)), + "DRAM segment data does not fit.") diff --git a/components/esp32/ld/esp32_fragments.lf b/components/esp32/ld/esp32_fragments.lf index 932197caf..abf8fa8bf 100644 --- a/components/esp32/ld/esp32_fragments.lf +++ b/components/esp32/ld/esp32_fragments.lf @@ -50,8 +50,12 @@ entries: [scheme:default] entries: - text -> flash_text - rodata -> flash_rodata + if APP_BUILD_USE_FLASH_SECTIONS = y: + text -> flash_text + rodata -> flash_rodata + else: + text -> iram0_text + rodata -> dram0_data data -> dram0_data bss -> dram0_bss common -> dram0_bss diff --git a/components/esp32/panic.c b/components/esp32/panic.c index d84ec1ecb..5a27f26d0 100644 --- a/components/esp32/panic.c +++ b/components/esp32/panic.c @@ -450,7 +450,7 @@ static void esp_panic_dig_reset(void) ; } } -#endif +#endif // CONFIG_ESP32_PANIC_PRINT_REBOOT || CONFIG_ESP32_PANIC_SILENT_REBOOT static void putEntry(uint32_t pc, uint32_t sp) { diff --git a/components/esp_rom/esp32/ld/esp32.rom.spiflash.ld b/components/esp_rom/esp32/ld/esp32.rom.spiflash.ld index 709b35811..56497f140 100644 --- a/components/esp_rom/esp32/ld/esp32.rom.spiflash.ld +++ b/components/esp_rom/esp32/ld/esp32.rom.spiflash.ld @@ -8,6 +8,7 @@ PROVIDE ( esp_rom_spiflash_erase_area = 0x400631ac ); PROVIDE ( esp_rom_spiflash_erase_block = 0x40062c4c ); PROVIDE ( esp_rom_spiflash_erase_chip = 0x40062c14 ); PROVIDE ( esp_rom_spiflash_erase_sector = 0x40062ccc ); +PROVIDE ( esp_rom_spiflash_attach = 0x40062a6c ); PROVIDE ( esp_rom_spiflash_lock = 0x400628f0 ); PROVIDE ( esp_rom_spiflash_read = 0x40062ed8 ); PROVIDE ( esp_rom_spiflash_config_readmode = 0x40062b64 ); /* SPIMasterReadModeCnfig */ diff --git a/components/esptool_py/CMakeLists.txt b/components/esptool_py/CMakeLists.txt index d30210531..8fae3c6eb 100644 --- a/components/esptool_py/CMakeLists.txt +++ b/components/esptool_py/CMakeLists.txt @@ -1,6 +1,6 @@ idf_component_register(REQUIRES bootloader) -if(NOT BOOTLOADER_BUILD) +if(NOT BOOTLOADER_BUILD AND CONFIG_APP_BUILD_GENERATE_BINARIES) string(REPLACE ";" " " ESPTOOLPY_FLASH_PROJECT_OPTIONS "${ESPTOOLPY_FLASH_OPTIONS}") set(ESPTOOLPY_FLASH_PROJECT_OPTIONS "${ESPTOOLPY_FLASH_PROJECT_OPTIONS}" diff --git a/components/esptool_py/Makefile.projbuild b/components/esptool_py/Makefile.projbuild index a351e8282..9c00c4e3f 100644 --- a/components/esptool_py/Makefile.projbuild +++ b/components/esptool_py/Makefile.projbuild @@ -68,7 +68,11 @@ endif APP_BIN_UNSIGNED ?= $(APP_BIN) $(APP_BIN_UNSIGNED): $(APP_ELF) $(ESPTOOLPY_SRC) | check_python_dependencies +ifeq ("$(CONFIG_APP_BUILD_GENERATE_BINARIES)","y") $(ESPTOOLPY) elf2image $(ESPTOOL_FLASH_OPTIONS) $(ESPTOOL_ELF2IMAGE_OPTIONS) -o $@ $< +else + @echo "Skipping the BIN generation" +endif ifdef CONFIG_SECURE_FLASH_ENCRYPTION_MODE_DEVELOPMENT encrypted-flash: all_binaries $(ESPTOOLPY_SRC) $(call prereq_if_explicit,erase_flash) partition_table_get_info | check_python_dependencies diff --git a/components/esptool_py/project_include.cmake b/components/esptool_py/project_include.cmake index af676a626..f0433d842 100644 --- a/components/esptool_py/project_include.cmake +++ b/components/esptool_py/project_include.cmake @@ -78,24 +78,28 @@ set(PROJECT_BIN "${elf_name}.bin") # # Add 'app.bin' target - generates with elf2image # -add_custom_command(OUTPUT "${build_dir}/.bin_timestamp" - COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_FLASH_OPTIONS} ${ESPTOOLPY_ELF2IMAGE_OPTIONS} - -o "${build_dir}/${unsigned_project_binary}" "${elf}" - COMMAND ${CMAKE_COMMAND} -E echo "Generated ${build_dir}/${unsigned_project_binary}" - COMMAND ${CMAKE_COMMAND} -E md5sum "${build_dir}/${unsigned_project_binary}" > "${build_dir}/.bin_timestamp" - DEPENDS ${elf} - VERBATIM - WORKING_DIRECTORY ${build_dir} - COMMENT "Generating binary image from built executable" - ) -add_custom_target(gen_project_binary DEPENDS "${build_dir}/.bin_timestamp") +if(CONFIG_APP_BUILD_GENERATE_BINARIES) + add_custom_command(OUTPUT "${build_dir}/.bin_timestamp" + COMMAND ${ESPTOOLPY} elf2image ${ESPTOOLPY_FLASH_OPTIONS} ${ESPTOOLPY_ELF2IMAGE_OPTIONS} + -o "${build_dir}/${unsigned_project_binary}" "${elf}" + COMMAND ${CMAKE_COMMAND} -E echo "Generated ${build_dir}/${unsigned_project_binary}" + COMMAND ${CMAKE_COMMAND} -E md5sum "${build_dir}/${unsigned_project_binary}" > "${build_dir}/.bin_timestamp" + DEPENDS ${elf} + VERBATIM + WORKING_DIRECTORY ${build_dir} + COMMENT "Generating binary image from built executable" + ) + add_custom_target(gen_project_binary DEPENDS "${build_dir}/.bin_timestamp") +endif() set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${build_dir}/${unsigned_project_binary}" ) -add_custom_target(app ALL DEPENDS gen_project_binary) +if(CONFIG_APP_BUILD_GENERATE_BINARIES) + add_custom_target(app ALL DEPENDS gen_project_binary) +endif() if(NOT BOOTLOADER_BUILD AND CONFIG_SECURE_SIGNED_APPS) if(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES) diff --git a/components/soc/src/memory_layout_utils.c b/components/soc/src/memory_layout_utils.c index 29f752b9d..2f4acb5c2 100644 --- a/components/soc/src/memory_layout_utils.c +++ b/components/soc/src/memory_layout_utils.c @@ -29,7 +29,7 @@ extern soc_reserved_region_t soc_reserved_memory_region_end; These variables have the start and end of the data and static IRAM area used by the program. Defined in the linker script. */ -extern int _data_start, _static_data_end, _iram_start, _iram_end; +extern int _data_start, _heap_start, _iram_start, _iram_end; /* static DRAM & IRAM chunks */ static const size_t EXTRA_RESERVED_REGIONS = 2; @@ -67,8 +67,8 @@ static void s_prepare_reserved_regions(soc_reserved_region_t *reserved, size_t c (count - EXTRA_RESERVED_REGIONS) * sizeof(soc_reserved_region_t)); /* Add the EXTRA_RESERVED_REGIONS at the beginning */ - reserved[0].start = (intptr_t)&_data_start; /* DRAM used by data+bss */ - reserved[0].end = (intptr_t)&_static_data_end; + reserved[0].start = (intptr_t)&_data_start; /* DRAM used by data+bss and possibly rodata */ + reserved[0].end = (intptr_t)&_heap_start; reserved[1].start = (intptr_t)&_iram_start; /* IRAM used by code */ reserved[1].end = (intptr_t)&_iram_end; diff --git a/examples/get-started/hello_world/.gdbinit.ci b/examples/get-started/hello_world/.gdbinit.ci new file mode 100644 index 000000000..a7fb13506 --- /dev/null +++ b/examples/get-started/hello_world/.gdbinit.ci @@ -0,0 +1,13 @@ +# Connect to a running instance of OpenOCD +target remote 127.0.0.1:3333 +# Reset and halt the target +mon reset halt +# Run to a specific point in ROM code, +# where most of initialization is complete. +thb *0x40007901 +c +# Load the application into RAM +load +# Run till app_main +tb app_main +c diff --git a/examples/get-started/hello_world/loadable_elf_example_test.py b/examples/get-started/hello_world/loadable_elf_example_test.py new file mode 100644 index 000000000..c6f9e2462 --- /dev/null +++ b/examples/get-started/hello_world/loadable_elf_example_test.py @@ -0,0 +1,129 @@ +import os +import pexpect +import serial +import sys +import threading +import time + +try: + import IDF +except ImportError: + test_fw_path = os.getenv('TEST_FW_PATH') + if test_fw_path and test_fw_path not in sys.path: + sys.path.insert(0, test_fw_path) + import IDF + +import Utility + + +class CustomProcess(object): + def __init__(self, cmd, logfile): + self.f = open(logfile, 'wb') + self.p = pexpect.spawn(cmd, timeout=60, logfile=self.f) + + def __enter__(self): + return self + + def close(self): + self.p.terminate(force=True) + + def __exit__(self, type, value, traceback): + self.close() + self.f.close() + + +class OCDProcess(CustomProcess): + def __init__(self, proj_path): + cmd = 'openocd -f interface/ftdi/esp32_devkitj_v1.cfg -f board/esp-wroom-32.cfg' + log_file = os.path.join(proj_path, 'openocd.log') + super(OCDProcess, self).__init__(cmd, log_file) + i = self.p.expect_exact(['Info : Listening on port 3333 for gdb connections', 'Error:']) + if i == 0: + Utility.console_log('openocd is listening for gdb connections') + else: + raise RuntimeError('openocd initialization has failed') + + def close(self): + try: + self.p.sendcontrol('c') + self.p.expect_exact('shutdown command invoked') + except Exception: + Utility.console_log('openocd needs to be killed', 'O') + super(OCDProcess, self).close() + + +class GDBProcess(CustomProcess): + def __init__(self, proj_path, elf_path): + cmd = 'xtensa-esp32-elf-gdb -x {} --directory={} {}'.format(os.path.join(proj_path, '.gdbinit.ci'), + os.path.join(proj_path, 'main'), + elf_path) + log_file = os.path.join(proj_path, 'gdb.log') + super(GDBProcess, self).__init__(cmd, log_file) + self.p.sendline('') # it is for "---Type to continue, or q to quit---" + i = self.p.expect_exact(['Thread 1 hit Temporary breakpoint 2, app_main ()', + 'Load failed']) + if i == 0: + Utility.console_log('gdb is at breakpoint') + else: + raise RuntimeError('Load failed: probably the ELF file was not built for loading with gdb') + self.p.expect_exact('(gdb)') + + def close(self): + try: + self.p.sendline('q') + self.p.expect_exact('Quit anyway? (y or n)') + self.p.sendline('y') + self.p.expect_exact('Ending remote debugging.') + except Exception: + Utility.console_log('gdb needs to be killed', 'O') + super(GDBProcess, self).close() + + def break_till_end(self): + self.p.sendline('b esp_restart') + self.p.sendline('c') + self.p.expect_exact('Thread 1 hit Breakpoint 3, esp_restart ()') + + +class SerialThread(object): + def run(self, log_path, exit_event): + with serial.Serial('/dev/ttyUSB1', 115200) as ser, open(log_path, 'wb') as f: + while True: + f.write(ser.read(ser.in_waiting)) + if exit_event.is_set(): + break + time.sleep(1) + + def __init__(self, log_path): + self.exit_event = threading.Event() + self.t = threading.Thread(target=self.run, args=(log_path, self.exit_event,)) + self.t.start() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.exit_event.set() + self.t.join(60) + if self.t.is_alive(): + Utility.console_log('The pyserial thread is still alive', 'O') + + +@IDF.idf_example_test(env_tag="test_jtag_arm") +def test_examples_loadable_elf(env, extra_data): + + idf_path = os.environ['IDF_PATH'] + rel_project_path = os.path.join('examples', 'get-started', 'hello_world') + proj_path = os.path.join(idf_path, rel_project_path) + elf_path = os.path.join(IDF.Example(rel_project_path).get_binary_path(rel_project_path), 'hello-world.elf') + esp_log_path = os.path.join(proj_path, 'esp.log') + + with SerialThread(esp_log_path): + with OCDProcess(proj_path), GDBProcess(proj_path, elf_path) as gdb: + gdb.break_till_end() + + if pexpect.run('grep "Restarting now." {}'.format(esp_log_path), withexitstatus=True)[1]: + raise RuntimeError('Expected output from ESP was not received') + + +if __name__ == '__main__': + test_examples_loadable_elf() diff --git a/examples/get-started/hello_world/sdkconfig.ci b/examples/get-started/hello_world/sdkconfig.ci new file mode 100644 index 000000000..c8b9cae75 --- /dev/null +++ b/examples/get-started/hello_world/sdkconfig.ci @@ -0,0 +1,6 @@ +CONFIG_APP_BUILD_TYPE_ELF_RAM=y +CONFIG_VFS_SUPPORT_TERMIOS= +CONFIG_NEWLIB_NANO_FORMAT=y +CONFIG_ESP32_PANIC_PRINT_HALT=y +CONFIG_ESP32_DEBUG_STUBS_ENABLE= +CONFIG_ESP_ERR_TO_NAME_LOOKUP= diff --git a/make/project.mk b/make/project.mk index 4c5935b07..a8ec59f40 100644 --- a/make/project.mk +++ b/make/project.mk @@ -320,9 +320,15 @@ ifndef CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES endif @echo "To flash app & partition table, run 'make flash' or:" else +ifdef CONFIG_APP_BUILD_GENERATE_BINARIES @echo "To flash all build output, run 'make flash' or:" endif +endif +ifdef CONFIG_APP_BUILD_GENERATE_BINARIES @echo $(ESPTOOLPY_WRITE_FLASH) $(ESPTOOL_ALL_FLASH_ARGS) +else + @echo "Binary is not available for flashing" +endif # If we have `version.txt` then prefer that for extracting IDF version @@ -533,6 +539,7 @@ $(APP_ELF): $(foreach libcomp,$(COMPONENT_LIBRARIES),$(BUILD_DIR_BASE)/$(libcomp $(CC) $(LDFLAGS) -o $@ -Wl,-Map=$(APP_MAP) app: $(APP_BIN) partition_table_get_info +ifeq ("$(CONFIG_APP_BUILD_GENERATE_BINARIES)","y") ifeq ("$(CONFIG_SECURE_BOOT_ENABLED)$(CONFIG_SECURE_BOOT_BUILD_SIGNED_BINARIES)","y") # secure boot enabled, but remote sign app image @echo "App built but not signed. Signing step via espsecure.py:" @echo "espsecure.py sign_data --keyfile KEYFILE $(APP_BIN)" @@ -542,6 +549,9 @@ else @echo "App built. Default flash app command is:" @echo $(ESPTOOLPY_WRITE_FLASH) $(APP_OFFSET) $(APP_BIN) endif +else + @echo "Application in not built and cannot be flashed." +endif all_binaries: $(APP_BIN) diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 4609f10be..aeb627941 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -202,6 +202,19 @@ example_test_008: - ESP32 - Example_Flash_Encryption +example_test_009: + extends: .example_test_template + tags: + - ESP32 + - test_jtag_arm + artifacts: + when: always + paths: + - $CI_PROJECT_DIR/examples/get-started/hello_world/*.log + expire_in: 1 week + variables: + SETUP_TOOLS: "1" + UT_001: extends: .unit_test_template parallel: 50 diff --git a/tools/idf.py b/tools/idf.py index b35b60be8..406b45a3d 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -924,6 +924,10 @@ def init_cli(verbose_output=None): print("Done") return + if not os.path.exists(os.path.join(args.build_dir, "flasher_args.json")): + print("Done") + return + # Otherwise, if we built any binaries print a message about # how to flash them def print_flashing_message(title, key): diff --git a/tools/tiny-test-fw/IDF/IDFApp.py b/tools/tiny-test-fw/IDF/IDFApp.py index 82f42b0c3..6324071f5 100644 --- a/tools/tiny-test-fw/IDF/IDFApp.py +++ b/tools/tiny-test-fw/IDF/IDFApp.py @@ -35,18 +35,21 @@ class IDFApp(App.BaseApp): self.binary_path = self.get_binary_path(app_path) self.elf_file = self._get_elf_file_path(self.binary_path) assert os.path.exists(self.binary_path) - if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): - if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path): - msg = ("Neither {} nor {} exists. " - "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' " - "or 'idf.py build' " - "for resolving the issue." - "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE, - self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) - raise AssertionError(msg) + sdkconfig_dict = self.get_sdkconfig() + if "CONFIG_APP_BUILD_GENERATE_BINARIES" in sdkconfig_dict: + # There are no flashing targets available when no binaries where generated. + if self.IDF_DOWNLOAD_CONFIG_FILE not in os.listdir(self.binary_path): + if self.IDF_FLASH_ARGS_FILE not in os.listdir(self.binary_path): + msg = ("Neither {} nor {} exists. " + "Try to run 'make print_flash_cmd | tail -n 1 > {}/{}' " + "or 'idf.py build' " + "for resolving the issue." + "").format(self.IDF_DOWNLOAD_CONFIG_FILE, self.IDF_FLASH_ARGS_FILE, + self.binary_path, self.IDF_DOWNLOAD_CONFIG_FILE) + raise AssertionError(msg) - self.flash_files, self.flash_settings = self._parse_flash_download_config() - self.partition_table = self._parse_partition_table() + self.flash_files, self.flash_settings = self._parse_flash_download_config() + self.partition_table = self._parse_partition_table() @classmethod def get_sdk_path(cls): diff --git a/tools/unit-test-app/components/test_utils/test_runner.c b/tools/unit-test-app/components/test_utils/test_runner.c index 1f00a0168..fc09d5a45 100644 --- a/tools/unit-test-app/components/test_utils/test_runner.c +++ b/tools/unit-test-app/components/test_utils/test_runner.c @@ -70,7 +70,15 @@ void setUp(void) #endif printf("%s", ""); /* sneakily lazy-allocate the reent structure for this test task */ + +#ifdef CONFIG_APP_BUILD_USE_FLASH_SECTIONS + /* TODO: add sufficient startup code in case of building an ELF file, so that + * flash cache is initialized and can work in such mode. + * For now this is disabled to allow running unit tests which don't require + * flash cache related operations. + */ get_test_data_partition(); /* allocate persistent partition table structures */ +#endif // CONFIG_APP_BUILD_USE_FLASH_SECTIONS unity_reset_leak_checks(); test_utils_set_leak_level(CONFIG_UNITY_CRITICAL_LEAK_LEVEL_GENERAL, TYPE_LEAK_CRITICAL, COMP_LEAK_GENERAL); From 847537c862a1e3a920b887b84b171b36c99bbebb Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Tue, 24 Sep 2019 10:39:27 +0200 Subject: [PATCH 18/30] docs: update get-started to use Tools Installer v2.1 --- docs/en/get-started/windows-setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/get-started/windows-setup.rst b/docs/en/get-started/windows-setup.rst index 9dccad8dc..a8efd5639 100644 --- a/docs/en/get-started/windows-setup.rst +++ b/docs/en/get-started/windows-setup.rst @@ -24,7 +24,7 @@ ESP-IDF Tools Installer The easiest way to install ESP-IDF's prerequisites is to download the ESP-IDF Tools installer from this URL: -https://dl.espressif.com/dl/esp-idf-tools-setup-2.0.exe +https://dl.espressif.com/dl/esp-idf-tools-setup-2.1.exe The installer includes the cross-compilers, OpenOCD, cmake_ and Ninja_ build tool, and a configuration tool called mconf-idf_. The installer can also download and run installers for Python_ 3.7 and `Git For Windows`_ if they are not already installed on the computer. From 0792a2fde8af34e2e3153b03547f2d2c312abc51 Mon Sep 17 00:00:00 2001 From: lly Date: Wed, 25 Sep 2019 11:03:18 +0800 Subject: [PATCH 19/30] ble_mesh: check ble mesh queue and task init status --- components/bt/esp_ble_mesh/mesh_core/adv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/bt/esp_ble_mesh/mesh_core/adv.c b/components/bt/esp_ble_mesh/mesh_core/adv.c index 21fd0580c..994e28215 100644 --- a/components/bt/esp_ble_mesh/mesh_core/adv.c +++ b/components/bt/esp_ble_mesh/mesh_core/adv.c @@ -386,8 +386,10 @@ static void bt_mesh_scan_cb(const bt_mesh_addr_t *addr, s8_t rssi, void bt_mesh_adv_init(void) { xBleMeshQueue = xQueueCreate(150, sizeof(bt_mesh_msg_t)); - xTaskCreatePinnedToCore(adv_thread, "BLE_Mesh_ADV_Task", 3072, NULL, + configASSERT(xBleMeshQueue); + int ret = xTaskCreatePinnedToCore(adv_thread, "BLE_Mesh_ADV_Task", 3072, NULL, configMAX_PRIORITIES - 7, NULL, TASK_PINNED_TO_CORE); + configASSERT(ret == pdTRUE); } int bt_mesh_scan_enable(void) From 2c3648b619fe99ba1577ba7a403dd4d8b9f87180 Mon Sep 17 00:00:00 2001 From: Prasad Alatkar Date: Wed, 25 Sep 2019 11:56:47 +0800 Subject: [PATCH 20/30] BT/Bluedroid : Add support to set min encryption key requirement - Modifies `smp_utils.c` to add check on encryption key size received from peer. - Modifies `esp_ble_gap_set_security_param` API to add minimum encryption key size requirement. --- .../api/include/api/esp_gap_ble_api.h | 15 ++++++++++-- .../bt/host/bluedroid/bta/dm/bta_dm_co.c | 14 ++++++++++- .../bluedroid/bta/include/bta/bta_dm_co.h | 2 ++ .../btc/profile/std/gap/btc_gap_ble.c | 6 +++++ .../common/include/common/bte_appl.h | 3 ++- .../bt/host/bluedroid/stack/smp/smp_utils.c | 23 +++++++++++++++++-- 6 files changed, 57 insertions(+), 6 deletions(-) diff --git a/components/bt/host/bluedroid/api/include/api/esp_gap_ble_api.h b/components/bt/host/bluedroid/api/include/api/esp_gap_ble_api.h index f6097dfe7..657610a13 100644 --- a/components/bt/host/bluedroid/api/include/api/esp_gap_ble_api.h +++ b/components/bt/host/bluedroid/api/include/api/esp_gap_ble_api.h @@ -267,14 +267,25 @@ typedef enum { typedef enum { ESP_BLE_SM_PASSKEY = 0, + /* Authentication requirements of local device */ ESP_BLE_SM_AUTHEN_REQ_MODE, + /* The IO capability of local device */ ESP_BLE_SM_IOCAP_MODE, + /* Initiator Key Distribution/Generation */ ESP_BLE_SM_SET_INIT_KEY, + /* Responder Key Distribution/Generation */ ESP_BLE_SM_SET_RSP_KEY, + /* Maximum Encryption key size to support */ ESP_BLE_SM_MAX_KEY_SIZE, + /* Minimum Encryption key size requirement from Peer */ + ESP_BLE_SM_MIN_KEY_SIZE, + /* Set static Passkey */ ESP_BLE_SM_SET_STATIC_PASSKEY, + /* Reset static Passkey */ ESP_BLE_SM_CLEAR_STATIC_PASSKEY, + /* Accept only specified SMP Authentication requirement */ ESP_BLE_SM_ONLY_ACCEPT_SPECIFIED_SEC_AUTH, + /* Enable/Disable OOB support */ ESP_BLE_SM_OOB_SUPPORT, ESP_BLE_SM_MAX_PARAM, } esp_ble_sm_param_t; @@ -377,7 +388,7 @@ typedef struct { advertising reports for each packet received */ } esp_ble_scan_params_t; -/// connection parameters information +/// connection parameters information typedef struct { uint16_t interval; /*!< connection interval */ uint16_t latency; /*!< Slave latency for the connection in number of connection events. Range: 0x0000 to 0x01F3 */ @@ -598,7 +609,7 @@ typedef enum { typedef enum { ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_ADV_ADDR = 0, /*!< BLE advertising address , device info will be added into ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_ADDR_LIST */ ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_LINK_ID, /*!< BLE mesh link ID, it is for BLE mesh, device info will be added into ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_MESH_LINK_ID_LIST */ - ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_BEACON_TYPE, /*!< BLE mesh beacon AD type, the format is | Len | 0x2B | Beacon Type | Beacon Data | */ + ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_BEACON_TYPE, /*!< BLE mesh beacon AD type, the format is | Len | 0x2B | Beacon Type | Beacon Data | */ ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_PROV_SRV_ADV, /*!< BLE mesh provisioning service uuid, the format is | 0x02 | 0x01 | flags | 0x03 | 0x03 | 0x1827 | .... |` */ ESP_BLE_DUPLICATE_SCAN_EXCEPTIONAL_INFO_MESH_PROXY_SRV_ADV, /*!< BLE mesh adv with proxy service uuid, the format is | 0x02 | 0x01 | flags | 0x03 | 0x03 | 0x1828 | .... |` */ } esp_ble_duplicate_exceptional_info_type_t; diff --git a/components/bt/host/bluedroid/bta/dm/bta_dm_co.c b/components/bt/host/bluedroid/bta/dm/bta_dm_co.c index 7e6cb4808..5fdf0812a 100644 --- a/components/bt/host/bluedroid/bta/dm/bta_dm_co.c +++ b/components/bt/host/bluedroid/bta/dm/bta_dm_co.c @@ -48,6 +48,7 @@ tBTE_APPL_CFG bte_appl_cfg = { BTM_BLE_INITIATOR_KEY_SIZE, BTM_BLE_RESPONDER_KEY_SIZE, BTM_BLE_MAX_KEY_SIZE, + BTM_BLE_MIN_KEY_SIZE, BTM_BLE_ONLY_ACCEPT_SPECIFIED_SEC_AUTH_DISABLE, BTM_BLE_OOB_DISABLE, }; @@ -407,7 +408,7 @@ void bta_dm_co_ble_set_rsp_key_req(UINT8 rsp_key) void bta_dm_co_ble_set_max_key_size(UINT8 ble_key_size) { #if (SMP_INCLUDED == TRUE) - if(ble_key_size >= BTM_BLE_MIN_KEY_SIZE && ble_key_size <= BTM_BLE_MAX_KEY_SIZE) { + if(ble_key_size >= bte_appl_cfg.ble_min_key_size && ble_key_size <= BTM_BLE_MAX_KEY_SIZE) { bte_appl_cfg.ble_max_key_size = ble_key_size; } else { APPL_TRACE_ERROR("%s error:Invalid key size value, key_size =%d",__func__, ble_key_size); @@ -415,6 +416,17 @@ void bta_dm_co_ble_set_max_key_size(UINT8 ble_key_size) #endif ///SMP_INCLUDED == TRUE } +void bta_dm_co_ble_set_min_key_size(UINT8 ble_key_size) +{ +#if (SMP_INCLUDED == TRUE) + if(ble_key_size >= BTM_BLE_MIN_KEY_SIZE && ble_key_size <= bte_appl_cfg.ble_max_key_size) { + bte_appl_cfg.ble_min_key_size = ble_key_size; + } else { + APPL_TRACE_ERROR("%s error:Invalid key size value, key_size =%d",__func__, ble_key_size); + } +#endif ///SMP_INCLUDED == TRUE +} + void bta_dm_co_ble_set_accept_auth_enable(UINT8 enable) { #if (SMP_INCLUDED == TRUE) diff --git a/components/bt/host/bluedroid/bta/include/bta/bta_dm_co.h b/components/bt/host/bluedroid/bta/include/bta/bta_dm_co.h index 3ef102c5a..124711ebb 100644 --- a/components/bt/host/bluedroid/bta/include/bta/bta_dm_co.h +++ b/components/bt/host/bluedroid/bta/include/bta/bta_dm_co.h @@ -206,6 +206,8 @@ extern void bta_dm_co_ble_set_rsp_key_req(UINT8 rsp_key); extern void bta_dm_co_ble_set_max_key_size(UINT8 ble_key_size); +extern void bta_dm_co_ble_set_min_key_size(UINT8 ble_key_size); + extern void bta_dm_co_ble_set_accept_auth_enable(UINT8 enable); extern UINT8 bta_dm_co_ble_get_accept_auth_enable(void); diff --git a/components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c b/components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c index 9f02cc26f..d9a9cde7d 100644 --- a/components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c +++ b/components/bt/host/bluedroid/btc/profile/std/gap/btc_gap_ble.c @@ -1196,6 +1196,12 @@ void btc_gap_ble_call_handler(btc_msg_t *msg) bta_dm_co_ble_set_max_key_size(key_size); break; } + case ESP_BLE_SM_MIN_KEY_SIZE: { + uint8_t key_size = 0; + STREAM_TO_UINT8(key_size, value); + bta_dm_co_ble_set_min_key_size(key_size); + break; + } case ESP_BLE_SM_SET_STATIC_PASSKEY: { uint32_t passkey = 0; for(uint8_t i = 0; i < arg->set_security_param.len; i++) diff --git a/components/bt/host/bluedroid/common/include/common/bte_appl.h b/components/bt/host/bluedroid/common/include/common/bte_appl.h index 4810bd6ff..67f410835 100644 --- a/components/bt/host/bluedroid/common/include/common/bte_appl.h +++ b/components/bt/host/bluedroid/common/include/common/bte_appl.h @@ -31,6 +31,7 @@ typedef struct { UINT8 ble_init_key; UINT8 ble_resp_key; UINT8 ble_max_key_size; + UINT8 ble_min_key_size; UINT8 ble_accept_auth_enable; UINT8 oob_support; #endif @@ -48,4 +49,4 @@ typedef struct { #endif } tBTE_BT_APPL_CFG; -extern tBTE_BT_APPL_CFG bte_bt_appl_cfg; \ No newline at end of file +extern tBTE_BT_APPL_CFG bte_bt_appl_cfg; diff --git a/components/bt/host/bluedroid/stack/smp/smp_utils.c b/components/bt/host/bluedroid/stack/smp/smp_utils.c index 165b11a2a..b6e9ffce2 100644 --- a/components/bt/host/bluedroid/stack/smp/smp_utils.c +++ b/components/bt/host/bluedroid/stack/smp/smp_utils.c @@ -36,6 +36,7 @@ #include "smp_int.h" #include "device/controller.h" #include "btm_int.h" +#include "common/bte_appl.h" #define SMP_PAIRING_REQ_SIZE 7 #define SMP_CONFIRM_CMD_SIZE (BT_OCTET16_LEN + 1) @@ -1140,9 +1141,27 @@ BOOLEAN smp_pairing_request_response_parameters_are_valid(tSMP_CB *p_cb) return FALSE; } - if ((enc_size < SMP_ENCR_KEY_SIZE_MIN) || (enc_size > SMP_ENCR_KEY_SIZE_MAX)) { + /* `bte_appl_cfg.ble_min_enc_key_size` will be `SMP_ENCR_KEY_SIZE_MIN` by + * default if not set explicitly */ +#if (BLE_INCLUDED == TRUE) + if (enc_size < bte_appl_cfg.ble_min_key_size) { SMP_TRACE_WARNING("Rcvd from the peer cmd 0x%02x with Maximum Encryption \ - Key value (0x%02x) out of range).\n", + Key value (0x%02x) less than minimum required key size).\n", + p_cb->rcvd_cmd_code, enc_size); + return FALSE; + } +#else + if (enc_size < SMP_ENCR_KEY_SIZE_MIN) { + SMP_TRACE_WARNING("Rcvd from the peer cmd 0x%02x with Maximum Encryption \ + Key value (0x%02x) less than minimum required key size).\n", + p_cb->rcvd_cmd_code, enc_size); + return FALSE; + } +#endif + + if (enc_size > SMP_ENCR_KEY_SIZE_MAX) { + SMP_TRACE_WARNING("Rcvd from the peer cmd 0x%02x with Maximum Encryption \ + Key value (0x%02x) greater than supported by stack).\n", p_cb->rcvd_cmd_code, enc_size); return FALSE; } From 7c8c9cbb68cc898feaf84f549b47e65ba3593770 Mon Sep 17 00:00:00 2001 From: suda-morris <362953310@qq.com> Date: Thu, 19 Sep 2019 11:27:42 +0800 Subject: [PATCH 21/30] ethernet: enable unit test --- components/esp_eth/test/test_emac.c | 91 ++++++++++++++++++++++++----- tools/ci/config/target-test.yml | 14 +++++ 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/components/esp_eth/test/test_emac.c b/components/esp_eth/test/test_emac.c index d38c3ac9c..f5d1557c6 100644 --- a/components/esp_eth/test/test_emac.c +++ b/components/esp_eth/test/test_emac.c @@ -1,31 +1,47 @@ #include #include +#include "sdkconfig.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" +#include "freertos/event_groups.h" #include "tcpip_adapter.h" -#include "esp_event.h" #include "unity.h" #include "test_utils.h" +#include "esp_event.h" #include "esp_eth.h" #include "esp_log.h" -#include "sdkconfig.h" static const char *TAG = "esp_eth_test"; + +#define ETH_START_BIT BIT(0) +#define ETH_STOP_BIT BIT(1) +#define ETH_CONNECT_BIT BIT(2) +#define ETH_GOT_IP_BIT BIT(3) + +#define ETH_START_TIMEOUT_MS (10000) +#define ETH_CONNECT_TIMEOUT_MS (40000) +#define ETH_STOP_TIMEOUT_MS (10000) +#define ETH_GET_IP_TIMEOUT_MS (60000) + /** Event handler for Ethernet events */ static void eth_event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, void *event_data) { + EventGroupHandle_t eth_event_group = (EventGroupHandle_t)arg; switch (event_id) { case ETHERNET_EVENT_CONNECTED: + xEventGroupSetBits(eth_event_group, ETH_CONNECT_BIT); ESP_LOGI(TAG, "Ethernet Link Up"); break; case ETHERNET_EVENT_DISCONNECTED: ESP_LOGI(TAG, "Ethernet Link Down"); break; case ETHERNET_EVENT_START: + xEventGroupSetBits(eth_event_group, ETH_START_BIT); ESP_LOGI(TAG, "Ethernet Started"); break; case ETHERNET_EVENT_STOP: + xEventGroupSetBits(eth_event_group, ETH_STOP_BIT); ESP_LOGI(TAG, "Ethernet Stopped"); break; default: @@ -37,6 +53,7 @@ static void eth_event_handler(void *arg, esp_event_base_t event_base, static void got_ip_event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, void *event_data) { + EventGroupHandle_t eth_event_group = (EventGroupHandle_t)arg; ip_event_got_ip_t *event = (ip_event_got_ip_t *)event_data; const tcpip_adapter_ip_info_t *ip_info = &event->ip_info; @@ -46,27 +63,31 @@ static void got_ip_event_handler(void *arg, esp_event_base_t event_base, ESP_LOGI(TAG, "ETHMASK:" IPSTR, IP2STR(&ip_info->netmask)); ESP_LOGI(TAG, "ETHGW:" IPSTR, IP2STR(&ip_info->gw)); ESP_LOGI(TAG, "~~~~~~~~~~~"); + xEventGroupSetBits(eth_event_group, ETH_GOT_IP_BIT); } -TEST_CASE("esp32 emac io test", "[ethernet][ignore]") +TEST_CASE("esp32 ethernet io test", "[ethernet][test_env=UT_T2_Ethernet]") { TEST_ESP_OK(esp_event_loop_create_default()); eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG(); esp_eth_mac_t *mac = esp_eth_mac_new_esp32(&mac_config); eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); esp_eth_phy_t *phy = esp_eth_phy_new_ip101(&phy_config); - esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); + esp_eth_config_t eth_config = ETH_DEFAULT_CONFIG(mac, phy); esp_eth_handle_t eth_handle = NULL; - TEST_ESP_OK(esp_eth_driver_install(&config, ð_handle)); + TEST_ESP_OK(esp_eth_driver_install(ð_config, ð_handle)); vTaskDelay(pdMS_TO_TICKS(1000)); /* get MAC address */ uint8_t mac_addr[6]; memset(mac_addr, 0, sizeof(mac_addr)); TEST_ESP_OK(esp_eth_ioctl(eth_handle, ETH_CMD_G_MAC_ADDR, mac_addr)); + ESP_LOGI(TAG, "Ethernet MAC Address: %d:%d:%d:%d:%d:%d", + mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3], mac_addr[4], mac_addr[5]); TEST_ASSERT(mac_addr[0] != 0); /* get PHY address */ int phy_addr = -1; TEST_ESP_OK(esp_eth_ioctl(eth_handle, ETH_CMD_G_PHY_ADDR, &phy_addr)); + ESP_LOGI(TAG, "Ethernet PHY Address: %d", phy_addr); TEST_ASSERT(phy_addr >= 0 && phy_addr <= 31); TEST_ESP_OK(esp_eth_driver_uninstall(eth_handle)); TEST_ESP_OK(phy->del(phy)); @@ -74,21 +95,65 @@ TEST_CASE("esp32 emac io test", "[ethernet][ignore]") TEST_ESP_OK(esp_event_loop_delete_default()); } -TEST_CASE("ethernet tcpip_adapter", "[ethernet][ignore]") +TEST_CASE("esp32 ethernet event test", "[ethernet][test_env=UT_T2_Ethernet]") { - test_case_uses_tcpip(); + EventBits_t bits = 0; + EventGroupHandle_t eth_event_group = xEventGroupCreate(); + TEST_ASSERT(eth_event_group != NULL); TEST_ESP_OK(esp_event_loop_create_default()); - TEST_ESP_OK(tcpip_adapter_set_default_eth_handlers()); - TEST_ESP_OK(esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, ð_event_handler, NULL)); - TEST_ESP_OK(esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, &got_ip_event_handler, NULL)); + TEST_ESP_OK(esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, ð_event_handler, eth_event_group)); eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG(); esp_eth_mac_t *mac = esp_eth_mac_new_esp32(&mac_config); eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); esp_eth_phy_t *phy = esp_eth_phy_new_ip101(&phy_config); - esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy); + esp_eth_config_t eth_config = ETH_DEFAULT_CONFIG(mac, phy); esp_eth_handle_t eth_handle = NULL; - TEST_ESP_OK(esp_eth_driver_install(&config, ð_handle)); - vTaskDelay(portMAX_DELAY); + TEST_ESP_OK(esp_eth_driver_install(ð_config, ð_handle)); + /* wait for connection start */ + bits = xEventGroupWaitBits(eth_event_group, ETH_START_BIT, true, true, pdMS_TO_TICKS(ETH_START_TIMEOUT_MS)); + TEST_ASSERT((bits & ETH_START_BIT) == ETH_START_BIT); + /* wait for connection establish */ + bits = xEventGroupWaitBits(eth_event_group, ETH_CONNECT_BIT, true, true, pdMS_TO_TICKS(ETH_CONNECT_TIMEOUT_MS)); + TEST_ASSERT((bits & ETH_CONNECT_BIT) == ETH_CONNECT_BIT); + TEST_ESP_OK(esp_eth_driver_uninstall(eth_handle)); + /* wait for connection stop */ + bits = xEventGroupWaitBits(eth_event_group, ETH_STOP_BIT, true, true, pdMS_TO_TICKS(ETH_STOP_TIMEOUT_MS)); + TEST_ASSERT((bits & ETH_STOP_BIT) == ETH_STOP_BIT); + // "check link timer callback" might owned the reference of phy object, make sure it has release it + vTaskDelay(pdMS_TO_TICKS(2000)); + TEST_ESP_OK(phy->del(phy)); + TEST_ESP_OK(mac->del(mac)); + TEST_ESP_OK(esp_event_handler_unregister(ETH_EVENT, ESP_EVENT_ANY_ID, eth_event_handler)); + TEST_ESP_OK(esp_event_loop_delete_default()); + vEventGroupDelete(eth_event_group); +} + +TEST_CASE("esp32 ethernet dhcp test", "[ethernet][test_env=UT_T2_Ethernet]") +{ + EventBits_t bits = 0; + EventGroupHandle_t eth_event_group = xEventGroupCreate(); + TEST_ASSERT(eth_event_group != NULL); + test_case_uses_tcpip(); + TEST_ESP_OK(esp_event_loop_create_default()); + TEST_ESP_OK(tcpip_adapter_set_default_eth_handlers()); + TEST_ESP_OK(esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, &got_ip_event_handler, eth_event_group)); + eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG(); + esp_eth_mac_t *mac = esp_eth_mac_new_esp32(&mac_config); + eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); + esp_eth_phy_t *phy = esp_eth_phy_new_ip101(&phy_config); + esp_eth_config_t eth_config = ETH_DEFAULT_CONFIG(mac, phy); + esp_eth_handle_t eth_handle = NULL; + TEST_ESP_OK(esp_eth_driver_install(ð_config, ð_handle)); + /* wait for IP lease */ + bits = xEventGroupWaitBits(eth_event_group, ETH_GOT_IP_BIT, true, true, pdMS_TO_TICKS(ETH_GET_IP_TIMEOUT_MS)); + TEST_ASSERT((bits & ETH_GOT_IP_BIT) == ETH_GOT_IP_BIT); + TEST_ESP_OK(esp_eth_driver_uninstall(eth_handle)); + TEST_ESP_OK(phy->del(phy)); + TEST_ESP_OK(mac->del(mac)); + TEST_ESP_OK(esp_event_handler_unregister(IP_EVENT, IP_EVENT_ETH_GOT_IP, got_ip_event_handler)); + TEST_ESP_OK(tcpip_adapter_clear_default_eth_handlers()); + TEST_ESP_OK(esp_event_loop_delete_default()); + vEventGroupDelete(eth_event_group); } #if CONFIG_ETH_USE_SPI_ETHERNET diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 4609f10be..61cf8dd09 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -424,6 +424,20 @@ UT_031: - ESP32_IDF - UT_T1_FlashEncryption +UT_032: + extends: .unit_test_template + parallel: 4 + tags: + - ESP32_IDF + - UT_T2_Ethernet + +UT_033: + extends: .unit_test_template + tags: + - ESP32_IDF + - UT_T2_Ethernet + - psram + nvs_compatible_test: extends: .test_template artifacts: From 6fde73cdc462126f139821ed5d759c825b965dea Mon Sep 17 00:00:00 2001 From: xiehang Date: Thu, 19 Sep 2019 16:49:31 +0800 Subject: [PATCH 22/30] esp_wifi: Put g_promis_buf to IRAM --- components/esp_wifi/lib_esp32 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/esp_wifi/lib_esp32 b/components/esp_wifi/lib_esp32 index fbec23442..c3c527916 160000 --- a/components/esp_wifi/lib_esp32 +++ b/components/esp_wifi/lib_esp32 @@ -1 +1 @@ -Subproject commit fbec234429466dc25291cd89ff6f94c6f7a525f0 +Subproject commit c3c5279163d54d7a0cb969f35093207dfe1a9074 From 7a450222f5152576cc6210fbf988a1ab2d849396 Mon Sep 17 00:00:00 2001 From: lly Date: Thu, 26 Sep 2019 19:38:25 +0800 Subject: [PATCH 23/30] ble_mesh: fix client model invalid address access --- .../bt/esp_ble_mesh/btc/btc_ble_mesh_prov.c | 73 +++++++------------ .../bt/esp_ble_mesh/mesh_core/cfg_cli.c | 45 +++--------- .../bt/esp_ble_mesh/mesh_core/health_cli.c | 45 +++--------- .../bt/esp_ble_mesh/mesh_core/mesh_kernel.c | 2 +- .../mesh_models/client/client_common.c | 23 +++++- .../mesh_models/client/generic_client.c | 65 +++++------------ .../client/include/client_common.h | 2 +- .../mesh_models/client/lighting_client.c | 65 +++++------------ .../mesh_models/client/sensor_client.c | 64 +++++----------- .../mesh_models/client/time_scene_client.c | 64 +++++----------- 10 files changed, 139 insertions(+), 309 deletions(-) diff --git a/components/bt/esp_ble_mesh/btc/btc_ble_mesh_prov.c b/components/bt/esp_ble_mesh/btc/btc_ble_mesh_prov.c index c0a155bed..12080d51b 100644 --- a/components/bt/esp_ble_mesh/btc/btc_ble_mesh_prov.c +++ b/components/bt/esp_ble_mesh/btc/btc_ble_mesh_prov.c @@ -297,8 +297,6 @@ static void btc_ble_mesh_client_model_op_cb(struct bt_mesh_model *model, struct net_buf_simple *buf) { esp_ble_mesh_model_cb_param_t mesh_param = {0}; - bt_mesh_client_user_data_t *client_param = NULL; - bt_mesh_client_internal_data_t *data = NULL; bt_mesh_client_node_t *node = NULL; btc_msg_t msg = {0}; bt_status_t ret; @@ -308,15 +306,11 @@ static void btc_ble_mesh_client_model_op_cb(struct bt_mesh_model *model, return; } - client_param = (bt_mesh_client_user_data_t *)model->user_data; - data = (bt_mesh_client_internal_data_t *)client_param->internal_data; - if (!data) { - LOG_ERROR("%s, Client internal_data is NULL", __func__); - return; - } - bt_mesh_client_model_lock(); + msg.sig = BTC_SIG_API_CB; + msg.pid = BTC_PID_MODEL; + node = bt_mesh_is_client_recv_publish_msg(model, ctx, buf, false); if (node == NULL) { msg.act = ESP_BLE_MESH_CLIENT_MODEL_RECV_PUBLISH_MSG_EVT; @@ -325,6 +319,8 @@ static void btc_ble_mesh_client_model_op_cb(struct bt_mesh_model *model, mesh_param.client_recv_publish_msg.ctx = (esp_ble_mesh_msg_ctx_t *)ctx; mesh_param.client_recv_publish_msg.length = buf->len; mesh_param.client_recv_publish_msg.msg = buf->data; + ret = btc_transfer_context(&msg, &mesh_param, + sizeof(esp_ble_mesh_model_cb_param_t), btc_ble_mesh_model_copy_req_data); } else { msg.act = ESP_BLE_MESH_MODEL_OPERATION_EVT; mesh_param.model_operation.opcode = ctx->recv_op; @@ -332,19 +328,11 @@ static void btc_ble_mesh_client_model_op_cb(struct bt_mesh_model *model, mesh_param.model_operation.ctx = (esp_ble_mesh_msg_ctx_t *)ctx; mesh_param.model_operation.length = buf->len; mesh_param.model_operation.msg = buf->data; - } - - msg.sig = BTC_SIG_API_CB; - msg.pid = BTC_PID_MODEL; - if (msg.act == ESP_BLE_MESH_CLIENT_MODEL_RECV_PUBLISH_MSG_EVT) { - ret = btc_transfer_context(&msg, &mesh_param, - sizeof(esp_ble_mesh_model_cb_param_t), btc_ble_mesh_model_copy_req_data); - } else { if (!k_delayed_work_free(&node->timer)) { ret = btc_transfer_context(&msg, &mesh_param, sizeof(esp_ble_mesh_model_cb_param_t), btc_ble_mesh_model_copy_req_data); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&data->queue, node); + bt_mesh_client_free_node(node); } else { ret = BT_STATUS_SUCCESS; } @@ -603,44 +591,35 @@ static void btc_ble_mesh_prov_register_complete_cb(int err_code) static void btc_ble_mesh_client_model_timeout_cb(struct k_work *work) { esp_ble_mesh_model_cb_param_t mesh_param = {0}; - bt_mesh_client_user_data_t *client_param = NULL; - bt_mesh_client_internal_data_t *data = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; btc_msg_t msg = {0}; bt_status_t ret; - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model || !node->ctx.model->user_data) { - LOG_ERROR("%s, Invalid parameter", __func__); - return; - } - - client_param = (bt_mesh_client_user_data_t *)node->ctx.model->user_data; - data = (bt_mesh_client_internal_data_t *)client_param->internal_data; - if (!data) { - LOG_ERROR("%s, Client internal_data is NULL", __func__); - return; - } - bt_mesh_client_model_lock(); - if (!k_delayed_work_free(&node->timer)) { - mesh_param.client_send_timeout.opcode = node->opcode; - mesh_param.client_send_timeout.model = (esp_ble_mesh_model_t *)node->ctx.model; - mesh_param.client_send_timeout.ctx = (esp_ble_mesh_msg_ctx_t *)&node->ctx; + timer = CONTAINER_OF(work, struct k_delayed_work, work); - msg.sig = BTC_SIG_API_CB; - msg.pid = BTC_PID_MODEL; - msg.act = ESP_BLE_MESH_CLIENT_MODEL_SEND_TIMEOUT_EVT; + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + mesh_param.client_send_timeout.opcode = node->opcode; + mesh_param.client_send_timeout.model = (esp_ble_mesh_model_t *)node->ctx.model; + mesh_param.client_send_timeout.ctx = (esp_ble_mesh_msg_ctx_t *)&node->ctx; - ret = btc_transfer_context(&msg, &mesh_param, - sizeof(esp_ble_mesh_model_cb_param_t), btc_ble_mesh_model_copy_req_data); - if (ret != BT_STATUS_SUCCESS) { - LOG_ERROR("%s btc_transfer_context failed", __func__); + msg.sig = BTC_SIG_API_CB; + msg.pid = BTC_PID_MODEL; + msg.act = ESP_BLE_MESH_CLIENT_MODEL_SEND_TIMEOUT_EVT; + + ret = btc_transfer_context(&msg, &mesh_param, + sizeof(esp_ble_mesh_model_cb_param_t), btc_ble_mesh_model_copy_req_data); + if (ret != BT_STATUS_SUCCESS) { + LOG_ERROR("%s btc_transfer_context failed", __func__); + } + + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); } - - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&data->queue, node); } bt_mesh_client_model_unlock(); diff --git a/components/bt/esp_ble_mesh/mesh_core/cfg_cli.c b/components/bt/esp_ble_mesh/mesh_core/cfg_cli.c index 2eb843e35..c975b5600 100644 --- a/components/bt/esp_ble_mesh/mesh_core/cfg_cli.c +++ b/components/bt/esp_ble_mesh/mesh_core/cfg_cli.c @@ -107,37 +107,23 @@ static void bt_mesh_cfg_client_unlock(void) static void timeout_handler(struct k_work *work) { - config_internal_data_t *internal = NULL; - bt_mesh_config_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive configuration status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_config_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Config Client user_data is NULL", __func__); - return; - } - - internal = (config_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Config Client internal_data is NULL", __func__); - return; - } - bt_mesh_cfg_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_config_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_CONFIG_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_config_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_CONFIG_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_cfg_client_unlock(); @@ -149,7 +135,6 @@ static void cfg_client_cancel(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, void *status, size_t len) { - config_internal_data_t *data = NULL; bt_mesh_client_node_t *node = NULL; struct net_buf_simple buf = {0}; u8_t evt_type = 0xFF; @@ -159,12 +144,6 @@ static void cfg_client_cancel(struct bt_mesh_model *model, return; } - data = (config_internal_data_t *)cli->internal_data; - if (!data) { - BT_ERR("%s, Config Client internal_data is NULL", __func__); - return; - } - /* If it is a publish message, sent to the user directly. */ buf.data = (u8_t *)status; buf.len = (u16_t)len; @@ -235,7 +214,7 @@ static void cfg_client_cancel(struct bt_mesh_model *model, bt_mesh_config_client_cb_evt_to_btc( node->opcode, evt_type, model, ctx, (const u8_t *)status, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&data->queue, node); + bt_mesh_client_free_node(node); } } diff --git a/components/bt/esp_ble_mesh/mesh_core/health_cli.c b/components/bt/esp_ble_mesh/mesh_core/health_cli.c index e652bdda5..363e95c55 100644 --- a/components/bt/esp_ble_mesh/mesh_core/health_cli.c +++ b/components/bt/esp_ble_mesh/mesh_core/health_cli.c @@ -63,37 +63,23 @@ static void bt_mesh_health_client_unlock(void) static void timeout_handler(struct k_work *work) { - health_internal_data_t *internal = NULL; - bt_mesh_health_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive health status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_health_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Health Client user_data is NULL", __func__); - return; - } - - internal = (health_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Health Client internal_data is NULL", __func__); - return; - } - bt_mesh_health_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_health_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_HEALTH_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_health_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_HEALTH_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_health_client_unlock(); @@ -105,7 +91,6 @@ static void health_client_cancel(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, void *status, size_t len) { - health_internal_data_t *data = NULL; bt_mesh_client_node_t *node = NULL; struct net_buf_simple buf = {0}; u8_t evt_type = 0xFF; @@ -115,12 +100,6 @@ static void health_client_cancel(struct bt_mesh_model *model, return; } - data = (health_internal_data_t *)health_cli->internal_data; - if (!data) { - BT_ERR("%s, Health Client internal_data is NULL", __func__); - return; - } - /* If it is a publish message, sent to the user directly. */ buf.data = (u8_t *)status; buf.len = (u16_t)len; @@ -151,7 +130,7 @@ static void health_client_cancel(struct bt_mesh_model *model, bt_mesh_health_client_cb_evt_to_btc( node->opcode, evt_type, model, ctx, (const u8_t *)status, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&data->queue, node); + bt_mesh_client_free_node(node); } } diff --git a/components/bt/esp_ble_mesh/mesh_core/mesh_kernel.c b/components/bt/esp_ble_mesh/mesh_core/mesh_kernel.c index 7ff4f75b8..c169aa387 100644 --- a/components/bt/esp_ble_mesh/mesh_core/mesh_kernel.c +++ b/components/bt/esp_ble_mesh/mesh_core/mesh_kernel.c @@ -103,7 +103,7 @@ void k_delayed_work_init(struct k_delayed_work *work, k_work_handler_t handler) osi_mutex_lock(&bm_alarm_lock, OSI_MUTEX_MAX_TIMEOUT); if (!hash_map_has_key(bm_alarm_hash_map, (void *)work)) { - alarm = osi_alarm_new("bt_mesh", (osi_alarm_callback_t)handler, (void *)work, 0); + alarm = osi_alarm_new("bt_mesh", (osi_alarm_callback_t)handler, (void *)&work->work, 0); if (alarm == NULL) { BT_ERR("%s, Unable to create alarm", __func__); return; diff --git a/components/bt/esp_ble_mesh/mesh_models/client/client_common.c b/components/bt/esp_ble_mesh/mesh_models/client/client_common.c index 037dd98d0..7c2ac19b3 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/client_common.c +++ b/components/bt/esp_ble_mesh/mesh_models/client/client_common.c @@ -284,15 +284,30 @@ int bt_mesh_client_init(struct bt_mesh_model *model) return 0; } -int bt_mesh_client_free_node(sys_slist_t *queue, bt_mesh_client_node_t *node) +int bt_mesh_client_free_node(bt_mesh_client_node_t *node) { - if (!queue || !node) { - BT_ERR("%s, Invalid parameter", __func__); + bt_mesh_client_internal_data_t *internal = NULL; + bt_mesh_client_user_data_t *client = NULL; + + if (!node || !node->ctx.model) { + BT_ERR("%s, Client model list item is NULL", __func__); + return -EINVAL; + } + + client = (bt_mesh_client_user_data_t *)node->ctx.model->user_data; + if (!client) { + BT_ERR("%s, Client model user data is NULL", __func__); + return -EINVAL; + } + + internal = (bt_mesh_client_internal_data_t *)client->internal_data; + if (!internal) { + BT_ERR("%s, Client model internal data is NULL", __func__); return -EINVAL; } // Release the client node from the queue - sys_slist_find_and_remove(queue, &node->client_node); + sys_slist_find_and_remove(&internal->queue, &node->client_node); // Free the node osi_free(node); diff --git a/components/bt/esp_ble_mesh/mesh_models/client/generic_client.c b/components/bt/esp_ble_mesh/mesh_models/client/generic_client.c index 617083cbd..fd8986797 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/generic_client.c +++ b/components/bt/esp_ble_mesh/mesh_models/client/generic_client.c @@ -144,37 +144,23 @@ static void bt_mesh_generic_client_unlock(void) static void timeout_handler(struct k_work *work) { - generic_internal_data_t *internal = NULL; - bt_mesh_generic_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive generic status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_generic_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Generic Client user_data is NULL", __func__); - return; - } - - internal = (generic_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Generic Client internal_data is NULL", __func__); - return; - } - bt_mesh_generic_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_generic_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_GENERIC_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_generic_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_GENERIC_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_generic_client_unlock(); @@ -186,31 +172,14 @@ static void generic_status(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, struct net_buf_simple *buf) { - generic_internal_data_t *internal = NULL; - bt_mesh_generic_client_t *client = NULL; bt_mesh_client_node_t *node = NULL; - u8_t *val = NULL; - u8_t evt = 0xFF; - u32_t rsp = 0; + u8_t *val = NULL; + u8_t evt = 0xFF; size_t len = 0; BT_DBG("%s, len %d, bytes %s", __func__, buf->len, bt_hex(buf->data, buf->len)); - client = (bt_mesh_generic_client_t *)model->user_data; - if (!client) { - BT_ERR("%s, Generic Client user_data is NULL", __func__); - return; - } - - internal = (generic_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Generic Client internal_data is NULL", __func__); - return; - } - - rsp = ctx->recv_op; - - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_GEN_ONOFF_STATUS: { struct bt_mesh_gen_onoff_status *status = NULL; if (buf->len != 1 && buf->len != 3) { @@ -569,7 +538,7 @@ static void generic_status(struct bt_mesh_model *model, node = bt_mesh_is_client_recv_publish_msg(model, ctx, buf, true); if (!node) { - BT_DBG("Unexpected generic status message 0x%x", rsp); + BT_DBG("Unexpected generic status message 0x%x", ctx->recv_op); } else { switch (node->opcode) { case BLE_MESH_MODEL_OP_GEN_ONOFF_GET: @@ -615,13 +584,13 @@ static void generic_status(struct bt_mesh_model *model, if (!k_delayed_work_free(&node->timer)) { bt_mesh_generic_client_cb_evt_to_btc(node->opcode, evt, model, ctx, val, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + bt_mesh_client_free_node(node); } } bt_mesh_generic_client_unlock(); - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_GEN_USER_PROPERTIES_STATUS: { struct bt_mesh_gen_user_properties_status *status; status = (struct bt_mesh_gen_user_properties_status *)val; diff --git a/components/bt/esp_ble_mesh/mesh_models/client/include/client_common.h b/components/bt/esp_ble_mesh/mesh_models/client/include/client_common.h index d2b05e0c1..4cf2e890b 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/include/client_common.h +++ b/components/bt/esp_ble_mesh/mesh_models/client/include/client_common.h @@ -112,7 +112,7 @@ int bt_mesh_client_send_msg(struct bt_mesh_model *model, s32_t timeout, bool need_ack, const struct bt_mesh_send_cb *cb, void *cb_data); -int bt_mesh_client_free_node(sys_slist_t *queue, bt_mesh_client_node_t *node); +int bt_mesh_client_free_node(bt_mesh_client_node_t *node); enum { NODE = 0, diff --git a/components/bt/esp_ble_mesh/mesh_models/client/lighting_client.c b/components/bt/esp_ble_mesh/mesh_models/client/lighting_client.c index c52f35e00..a73f91a36 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/lighting_client.c +++ b/components/bt/esp_ble_mesh/mesh_models/client/lighting_client.c @@ -153,37 +153,23 @@ static void bt_mesh_light_client_unlock(void) static void timeout_handler(struct k_work *work) { - light_internal_data_t *internal = NULL; - bt_mesh_light_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive light status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_light_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Lighting Client user_data is NULL", __func__); - return; - } - - internal = (light_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Lighting Client internal_data is NULL", __func__); - return; - } - bt_mesh_light_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_lighting_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_LIGHTING_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_lighting_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_LIGHTING_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_light_client_unlock(); @@ -195,31 +181,14 @@ static void light_status(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, struct net_buf_simple *buf) { - light_internal_data_t *internal = NULL; - bt_mesh_light_client_t *client = NULL; bt_mesh_client_node_t *node = NULL; - u8_t *val = NULL; - u8_t evt = 0xFF; - u32_t rsp = 0; + u8_t *val = NULL; + u8_t evt = 0xFF; size_t len = 0; BT_DBG("%s, len %d, bytes %s", __func__, buf->len, bt_hex(buf->data, buf->len)); - client = (bt_mesh_light_client_t *)model->user_data; - if (!client) { - BT_ERR("%s, Lighting Client user_data is NULL", __func__); - return; - } - - internal = (light_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Lighting Client internal_data is NULL", __func__); - return; - } - - rsp = ctx->recv_op; - - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_LIGHT_LIGHTNESS_STATUS: { struct bt_mesh_light_lightness_status *status = NULL; if (buf->len != 2 && buf->len != 5) { @@ -684,7 +653,7 @@ static void light_status(struct bt_mesh_model *model, node = bt_mesh_is_client_recv_publish_msg(model, ctx, buf, true); if (!node) { - BT_DBG("Unexpected light status message 0x%x", rsp); + BT_DBG("Unexpected light status message 0x%x", ctx->recv_op); } else { switch (node->opcode) { case BLE_MESH_MODEL_OP_LIGHT_LIGHTNESS_GET: @@ -741,13 +710,13 @@ static void light_status(struct bt_mesh_model *model, if (!k_delayed_work_free(&node->timer)) { bt_mesh_lighting_client_cb_evt_to_btc(node->opcode, evt, model, ctx, val, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + bt_mesh_client_free_node(node); } } bt_mesh_light_client_unlock(); - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_LIGHT_LC_PROPERTY_STATUS: { struct bt_mesh_light_lc_property_status *status; status = (struct bt_mesh_light_lc_property_status *)val; diff --git a/components/bt/esp_ble_mesh/mesh_models/client/sensor_client.c b/components/bt/esp_ble_mesh/mesh_models/client/sensor_client.c index 3626a38dc..e3eda759a 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/sensor_client.c +++ b/components/bt/esp_ble_mesh/mesh_models/client/sensor_client.c @@ -82,37 +82,23 @@ static void bt_mesh_sensor_client_unlock(void) static void timeout_handler(struct k_work *work) { - sensor_internal_data_t *internal = NULL; - bt_mesh_sensor_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive sensor status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_sensor_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Sensor Client user_data is NULL", __func__); - return; - } - - internal = (sensor_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Sensor Client internal_data is NULL", __func__); - return; - } - bt_mesh_sensor_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_sensor_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_SENSOR_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_sensor_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_SENSOR_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_sensor_client_unlock(); @@ -124,30 +110,14 @@ static void sensor_status(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, struct net_buf_simple *buf) { - sensor_internal_data_t *internal = NULL; - bt_mesh_sensor_client_t *client = NULL; bt_mesh_client_node_t *node = NULL; - u8_t *val = NULL; - u8_t evt = 0xFF; - u32_t rsp = 0; + u8_t *val = NULL; + u8_t evt = 0xFF; size_t len = 0; BT_DBG("%s, len %d, bytes %s", __func__, buf->len, bt_hex(buf->data, buf->len)); - client = (bt_mesh_sensor_client_t *)model->user_data; - if (!client) { - BT_ERR("%s, Sensor Client user_data is NULL", __func__); - return; - } - - internal = (sensor_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Sensor Client internal_data is NULL", __func__); - return; - } - - rsp = ctx->recv_op; - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_SENSOR_DESCRIPTOR_STATUS: { struct bt_mesh_sensor_descriptor_status *status = NULL; status = osi_calloc(sizeof(struct bt_mesh_sensor_descriptor_status)); @@ -296,7 +266,7 @@ static void sensor_status(struct bt_mesh_model *model, node = bt_mesh_is_client_recv_publish_msg(model, ctx, buf, true); if (!node) { - BT_DBG("Unexpected sensor status message 0x%x", rsp); + BT_DBG("Unexpected sensor status message 0x%x", ctx->recv_op); } else { switch (node->opcode) { case BLE_MESH_MODEL_OP_SENSOR_DESCRIPTOR_GET: @@ -319,13 +289,13 @@ static void sensor_status(struct bt_mesh_model *model, if (!k_delayed_work_free(&node->timer)) { bt_mesh_sensor_client_cb_evt_to_btc(node->opcode, evt, model, ctx, val, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + bt_mesh_client_free_node(node); } } bt_mesh_sensor_client_unlock(); - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_SENSOR_DESCRIPTOR_STATUS: { struct bt_mesh_sensor_descriptor_status *status; status = (struct bt_mesh_sensor_descriptor_status *)val; diff --git a/components/bt/esp_ble_mesh/mesh_models/client/time_scene_client.c b/components/bt/esp_ble_mesh/mesh_models/client/time_scene_client.c index 01f563994..3b05be51b 100644 --- a/components/bt/esp_ble_mesh/mesh_models/client/time_scene_client.c +++ b/components/bt/esp_ble_mesh/mesh_models/client/time_scene_client.c @@ -98,37 +98,23 @@ static void bt_mesh_time_scene_client_unlock(void) static void timeout_handler(struct k_work *work) { - time_scene_internal_data_t *internal = NULL; - bt_mesh_time_scene_client_t *client = NULL; + struct k_delayed_work *timer = NULL; bt_mesh_client_node_t *node = NULL; BT_WARN("Receive time scene status message timeout"); - node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); - if (!node || !node->ctx.model) { - BT_ERR("%s, Invalid parameter", __func__); - return; - } - - client = (bt_mesh_time_scene_client_t *)node->ctx.model->user_data; - if (!client) { - BT_ERR("%s, Time Scene Client user_data is NULL", __func__); - return; - } - - internal = (time_scene_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Time Scene Client internal_data is NULL", __func__); - return; - } - bt_mesh_time_scene_client_lock(); - if (!k_delayed_work_free(&node->timer)) { - bt_mesh_time_scene_client_cb_evt_to_btc(node->opcode, - BTC_BLE_MESH_EVT_TIME_SCENE_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); - // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + timer = CONTAINER_OF(work, struct k_delayed_work, work); + + if (timer && !k_delayed_work_free(timer)) { + node = CONTAINER_OF(work, bt_mesh_client_node_t, timer.work); + if (node) { + bt_mesh_time_scene_client_cb_evt_to_btc(node->opcode, + BTC_BLE_MESH_EVT_TIME_SCENE_CLIENT_TIMEOUT, node->ctx.model, &node->ctx, NULL, 0); + // Don't forget to release the node at the end. + bt_mesh_client_free_node(node); + } } bt_mesh_time_scene_client_unlock(); @@ -140,30 +126,14 @@ static void time_scene_status(struct bt_mesh_model *model, struct bt_mesh_msg_ctx *ctx, struct net_buf_simple *buf) { - time_scene_internal_data_t *internal = NULL; - bt_mesh_time_scene_client_t *client = NULL; bt_mesh_client_node_t *node = NULL; - u8_t *val = NULL; - u8_t evt = 0xFF; - u32_t rsp = 0; + u8_t *val = NULL; + u8_t evt = 0xFF; size_t len = 0; BT_DBG("%s, len %d, bytes %s", __func__, buf->len, bt_hex(buf->data, buf->len)); - client = (bt_mesh_time_scene_client_t *)model->user_data; - if (!client) { - BT_ERR("%s, Time Scene Client user_data is NULL", __func__); - return; - } - - internal = (time_scene_internal_data_t *)client->internal_data; - if (!internal) { - BT_ERR("%s, Time Scene Client internal_data is NULL", __func__); - return; - } - - rsp = ctx->recv_op; - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_TIME_STATUS: { struct bt_mesh_time_status *status = NULL; if (buf->len != 5 && buf->len != 10) { @@ -333,7 +303,7 @@ static void time_scene_status(struct bt_mesh_model *model, node = bt_mesh_is_client_recv_publish_msg(model, ctx, buf, true); if (!node) { - BT_DBG("Unexpected time scene status message 0x%x", rsp); + BT_DBG("Unexpected time scene status message 0x%x", ctx->recv_op); } else { switch (node->opcode) { case BLE_MESH_MODEL_OP_TIME_GET: @@ -363,13 +333,13 @@ static void time_scene_status(struct bt_mesh_model *model, if (!k_delayed_work_free(&node->timer)) { bt_mesh_time_scene_client_cb_evt_to_btc(node->opcode, evt, model, ctx, val, len); // Don't forget to release the node at the end. - bt_mesh_client_free_node(&internal->queue, node); + bt_mesh_client_free_node(node); } } bt_mesh_time_scene_client_unlock(); - switch (rsp) { + switch (ctx->recv_op) { case BLE_MESH_MODEL_OP_SCENE_REGISTER_STATUS: { struct bt_mesh_scene_register_status *status; status = (struct bt_mesh_scene_register_status *)val; From cbab3c34f8e3955f3d41b3a90329197ee007ef09 Mon Sep 17 00:00:00 2001 From: suda-morris <362953310@qq.com> Date: Fri, 27 Sep 2019 10:58:30 +0800 Subject: [PATCH 24/30] bootloader: shrink bin size 1. write a bootloader version of "getting chip revision" function. 2. reduce wordy log. --- .../include/bootloader_common.h | 7 +++ .../src/bootloader_common.c | 43 ++++++++++++++++--- .../bootloader_support/src/bootloader_init.c | 3 +- components/efuse/src/esp_efuse_fields.c | 2 +- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/components/bootloader_support/include/bootloader_common.h b/components/bootloader_support/include/bootloader_common.h index b3b88b50f..b0dfb126b 100644 --- a/components/bootloader_support/include/bootloader_common.h +++ b/components/bootloader_support/include/bootloader_common.h @@ -142,6 +142,13 @@ int bootloader_common_select_otadata(const esp_ota_select_entry_t *two_otadata, */ esp_err_t bootloader_common_get_partition_description(const esp_partition_pos_t *partition, esp_app_desc_t *app_desc); +/** + * @brief Get chip revision + * + * @return Chip revision number + */ +uint8_t bootloader_common_get_chip_revision(void); + /** * @brief Check if the image (bootloader and application) has valid chip ID and revision * diff --git a/components/bootloader_support/src/bootloader_common.c b/components/bootloader_support/src/bootloader_common.c index 89ff80cd3..1aa7e9fa1 100644 --- a/components/bootloader_support/src/bootloader_common.c +++ b/components/bootloader_support/src/bootloader_common.c @@ -25,10 +25,11 @@ #include "bootloader_common.h" #include "soc/gpio_periph.h" #include "soc/rtc.h" +#include "soc/efuse_reg.h" +#include "soc/apb_ctrl_reg.h" #include "esp_image_format.h" #include "bootloader_sha.h" #include "sys/param.h" -#include "esp_efuse.h" #define ESP_PARTITION_HASH_LEN 32 /* SHA-256 digest length */ @@ -272,22 +273,50 @@ void bootloader_common_vddsdio_configure(void) #endif // CONFIG_BOOTLOADER_VDDSDIO_BOOST } +#ifdef CONFIG_IDF_TARGET_ESP32 +uint8_t bootloader_common_get_chip_revision(void) +{ + uint8_t eco_bit0, eco_bit1, eco_bit2; + eco_bit0 = (REG_READ(EFUSE_BLK0_RDATA3_REG) & 0xF000) >> 15; + eco_bit1 = (REG_READ(EFUSE_BLK0_RDATA5_REG) & 0x100000) >> 20; + eco_bit2 = (REG_READ(APB_CTRL_DATE_REG) & 0x80000000) >> 31; + uint32_t combine_value = (eco_bit2 << 2) | (eco_bit1 << 1) | eco_bit0; + uint8_t chip_ver = 0; + switch (combine_value) { + case 0: + chip_ver = 0; + break; + case 1: + chip_ver = 1; + break; + case 3: + chip_ver = 2; + break; + case 7: + chip_ver = 3; + break; + default: + chip_ver = 0; + break; + } + return chip_ver; +} +#endif + esp_err_t bootloader_common_check_chip_validity(const esp_image_header_t* img_hdr) { esp_err_t err = ESP_OK; esp_chip_id_t chip_id = CONFIG_IDF_FIRMWARE_CHIP_ID; if (chip_id != img_hdr->chip_id) { - ESP_LOGE(TAG, "image has invalid chip ID, expected at least %d, found %d", chip_id, img_hdr->chip_id); + ESP_LOGE(TAG, "mismatch chip ID, expect %d, found %d", chip_id, img_hdr->chip_id); err = ESP_FAIL; } - uint8_t revision = esp_efuse_get_chip_ver(); + uint8_t revision = bootloader_common_get_chip_revision(); if (revision < img_hdr->min_chip_rev) { - ESP_LOGE(TAG, "image has invalid chip revision, expected at least %d, found %d", revision, img_hdr->min_chip_rev); + ESP_LOGE(TAG, "can't run on lower chip revision, expect %d, found %d", revision, img_hdr->min_chip_rev); err = ESP_FAIL; } else if (revision != img_hdr->min_chip_rev) { - ESP_LOGI(TAG, "This chip is revision %d but project was configured for minimum revision %d. "\ - "Suggest setting project minimum revision to %d if safe to do so.", - revision, img_hdr->min_chip_rev, revision); + ESP_LOGI(TAG, "mismatch chip revision, expect %d, found %d", revision, img_hdr->min_chip_rev); } return err; } diff --git a/components/bootloader_support/src/bootloader_init.c b/components/bootloader_support/src/bootloader_init.c index d28233193..a42ab2253 100644 --- a/components/bootloader_support/src/bootloader_init.c +++ b/components/bootloader_support/src/bootloader_init.c @@ -41,7 +41,6 @@ #include "soc/spi_periph.h" #include "sdkconfig.h" -#include "esp_efuse.h" #include "esp_image_format.h" #include "esp_secure_boot.h" #include "esp_flash_encrypt.h" @@ -131,7 +130,7 @@ static esp_err_t bootloader_main(void) } /* Check chip ID and minimum chip revision that supported by this image */ - uint8_t revision = esp_efuse_get_chip_ver(); + uint8_t revision = bootloader_common_get_chip_revision(); ESP_LOGI(TAG, "Chip Revision: %d", revision); if (bootloader_common_check_chip_validity(&fhdr) != ESP_OK) { return ESP_FAIL; diff --git a/components/efuse/src/esp_efuse_fields.c b/components/efuse/src/esp_efuse_fields.c index 313bbc611..8aa9ade46 100644 --- a/components/efuse/src/esp_efuse_fields.c +++ b/components/efuse/src/esp_efuse_fields.c @@ -35,7 +35,7 @@ uint8_t esp_efuse_get_chip_ver(void) uint8_t eco_bit0, eco_bit1, eco_bit2; esp_efuse_read_field_blob(ESP_EFUSE_CHIP_VER_REV1, &eco_bit0, 1); esp_efuse_read_field_blob(ESP_EFUSE_CHIP_VER_REV2, &eco_bit1, 1); - eco_bit2 = (REG_READ(APB_CTRL_DATE_REG) & 80000000) >> 31; + eco_bit2 = (REG_READ(APB_CTRL_DATE_REG) & 0x80000000) >> 31; uint32_t combine_value = (eco_bit2 << 2) | (eco_bit1 << 1) | eco_bit0; uint8_t chip_ver = 0; switch (combine_value) { From 53d57dd736e4a56c07bf7f763f776058bb801348 Mon Sep 17 00:00:00 2001 From: zhangyanjiao Date: Sun, 29 Sep 2019 10:16:38 +0800 Subject: [PATCH 25/30] esp_wifi: 1. Fixed smartconfig ipc crash 2. Fix the WiFi init sequence bug 3. Fixed the bug that WiFi stop leads to task watchdog --- components/esp_wifi/lib_esp32 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/esp_wifi/lib_esp32 b/components/esp_wifi/lib_esp32 index c3c527916..555fb8b49 160000 --- a/components/esp_wifi/lib_esp32 +++ b/components/esp_wifi/lib_esp32 @@ -1 +1 @@ -Subproject commit c3c5279163d54d7a0cb969f35093207dfe1a9074 +Subproject commit 555fb8b49d1f86da767353ec800e65c6148eae78 From 63019689d6f82390d745d5ac878f58fef37c935c Mon Sep 17 00:00:00 2001 From: Xia Xiaotian Date: Thu, 12 Sep 2019 14:12:13 +0800 Subject: [PATCH 26/30] Coexist: fix some coexist bugs 1. Fix high beacon and broadcast packets loss ratio of WiFi to make MDNS test pass. 2. Improve stability of WiFi performance with a little sacrifice of throughput. 3. Improve BLE advertising and connection performance with dynamic priority. It sacrifices a little WiFi throughput, but achieves balance between WiFi and Bluetooth. --- components/bt/Kconfig | 15 +++++++++++++++ components/bt/controller/bt.c | 7 +++++++ components/bt/controller/lib | 2 +- components/esp32/esp_adapter.c | 8 ++++++++ .../esp_wifi/include/esp_coexist_internal.h | 6 ++++++ .../include/esp_private/wifi_os_adapter.h | 3 ++- components/esp_wifi/lib_esp32 | 2 +- 7 files changed, 40 insertions(+), 3 deletions(-) diff --git a/components/bt/Kconfig b/components/bt/Kconfig index a1ab51c74..6d340656b 100644 --- a/components/bt/Kconfig +++ b/components/bt/Kconfig @@ -358,6 +358,21 @@ menu "Bluetooth" If you set `BTDM_BLE_ADV_REPORT_DISCARD_THRSHOLD` to a small value or printf every adv lost event, it may cause adv packets lost more. + menuconfig BTDM_COEX_BT_OPTIONS + bool "Coexistence Bluetooth Side Options" + depends on ESP32_WIFI_SW_COEXIST_ENABLE + default n + help + Options of Bluetooth Side of WiFi and bluetooth coexistence. + + config BTDM_COEX_BLE_ADV_HIGH_PRIORITY + bool "Improve BLE ADV priority for WiFi & BLE coexistence" + depends on BTDM_COEX_BT_OPTIONS + default n + help + Improve BLE ADV coexistence priority to make it better performance. + For example, BLE mesh need to enable this option to improve BLE adv performance. + endmenu choice BT_HOST diff --git a/components/bt/controller/bt.c b/components/bt/controller/bt.c index 0b29e506e..81442d227 100644 --- a/components/bt/controller/bt.c +++ b/components/bt/controller/bt.c @@ -226,6 +226,7 @@ extern int coex_bt_release_wrapper(uint32_t event); extern int coex_register_bt_cb_wrapper(coex_func_cb_t cb); extern uint32_t coex_bb_reset_lock_wrapper(void); extern void coex_bb_reset_unlock_wrapper(uint32_t restore); +extern void coex_ble_adv_priority_high_set(bool high); extern char _bss_start_btdm; extern char _bss_end_btdm; @@ -1195,6 +1196,12 @@ esp_err_t esp_bt_controller_init(esp_bt_controller_config_t *cfg) goto error; } + #ifdef CONFIG_BTDM_COEX_BLE_ADV_HIGH_PRIORITY + coex_ble_adv_priority_high_set(true); + #else + coex_ble_adv_priority_high_set(false); + #endif + btdm_controller_status = ESP_BT_CONTROLLER_STATUS_INITED; return ESP_OK; diff --git a/components/bt/controller/lib b/components/bt/controller/lib index 484307604..aaadbf2c2 160000 --- a/components/bt/controller/lib +++ b/components/bt/controller/lib @@ -1 +1 @@ -Subproject commit 484307604fad19b82afca4ff2e57fda63158c2b4 +Subproject commit aaadbf2c26002ae85c175cb0e469a3b0bf57bf02 diff --git a/components/esp32/esp_adapter.c b/components/esp32/esp_adapter.c index 4fdd5ce65..0e8ec3be7 100644 --- a/components/esp32/esp_adapter.c +++ b/components/esp32/esp_adapter.c @@ -440,6 +440,13 @@ static uint32_t coex_status_get_wrapper(void) #endif } +static void coex_condition_set_wrapper(uint32_t type, bool dissatisfy) +{ +#if CONFIG_SW_COEXIST_ENABLE + coex_condition_set(type, dissatisfy); +#endif +} + static int coex_wifi_request_wrapper(uint32_t event, uint32_t latency, uint32_t duration) { #if CONFIG_ESP32_WIFI_SW_COEXIST_ENABLE @@ -591,6 +598,7 @@ wifi_osi_funcs_t g_wifi_osi_funcs = { ._modem_sleep_register = esp_modem_sleep_register, ._modem_sleep_deregister = esp_modem_sleep_deregister, ._coex_status_get = coex_status_get_wrapper, + ._coex_condition_set = coex_condition_set_wrapper, ._coex_wifi_request = coex_wifi_request_wrapper, ._coex_wifi_release = coex_wifi_release_wrapper, ._magic = ESP_WIFI_OS_ADAPTER_MAGIC, diff --git a/components/esp_wifi/include/esp_coexist_internal.h b/components/esp_wifi/include/esp_coexist_internal.h index 1d94d6db7..63f35666d 100644 --- a/components/esp_wifi/include/esp_coexist_internal.h +++ b/components/esp_wifi/include/esp_coexist_internal.h @@ -79,6 +79,12 @@ esp_err_t coex_preference_set(coex_prefer_t prefer); */ uint32_t coex_status_get(void); +/** + * @brief Set software coexist condition. + * @return : software coexist condition + */ +void coex_condition_set(uint32_t type, bool dissatisfy); + /** * @brief WiFi requests coexistence. * diff --git a/components/esp_wifi/include/esp_private/wifi_os_adapter.h b/components/esp_wifi/include/esp_private/wifi_os_adapter.h index 912ec3a53..89422c51c 100644 --- a/components/esp_wifi/include/esp_private/wifi_os_adapter.h +++ b/components/esp_wifi/include/esp_private/wifi_os_adapter.h @@ -21,7 +21,7 @@ extern "C" { #endif -#define ESP_WIFI_OS_ADAPTER_VERSION 0x00000003 +#define ESP_WIFI_OS_ADAPTER_VERSION 0x00000004 #define ESP_WIFI_OS_ADAPTER_MAGIC 0xDEADBEAF #define OSI_FUNCS_TIME_BLOCKING 0xffffffff @@ -120,6 +120,7 @@ typedef struct { int32_t (* _modem_sleep_register)(uint32_t module); int32_t (* _modem_sleep_deregister)(uint32_t module); uint32_t (* _coex_status_get)(void); + void (* _coex_condition_set)(uint32_t type, bool dissatisfy); int32_t (* _coex_wifi_request)(uint32_t event, uint32_t latency, uint32_t duration); int32_t (* _coex_wifi_release)(uint32_t event); int32_t _magic; diff --git a/components/esp_wifi/lib_esp32 b/components/esp_wifi/lib_esp32 index 555fb8b49..1266a879b 160000 --- a/components/esp_wifi/lib_esp32 +++ b/components/esp_wifi/lib_esp32 @@ -1 +1 @@ -Subproject commit 555fb8b49d1f86da767353ec800e65c6148eae78 +Subproject commit 1266a879ba57e17c3d55a247b2a95bb9c4217505 From fe08f435dc99af6b01b67360edcfbb5153941dbb Mon Sep 17 00:00:00 2001 From: Prasad Alatkar Date: Mon, 30 Sep 2019 14:15:43 +0800 Subject: [PATCH 27/30] NimBLE: Add optional mbedTLS support to NimBLE - Additional menuconfig option to select either mbedTLS or Tinycrypt from NimBLE. - Addition of `CMAC` and `ECP_RESTARTABLE` to mbedTLS menuconfig option and `esp_config.h`. - Changes NimBLE `component.mk` and `CMakeLists.txt` to support mbedTLS option. - Minor changes to `app_mesh.c` application. --- components/bt/CMakeLists.txt | 12 ++++-- components/bt/component.mk | 13 ++++-- components/bt/host/nimble/Kconfig.in | 13 +++++- components/bt/host/nimble/nimble | 2 +- .../host/nimble/port/include/esp_nimble_cfg.h | 4 ++ components/mbedtls/Kconfig | 13 ++++++ components/mbedtls/mbedtls | 2 +- .../mbedtls/port/include/mbedtls/esp_config.h | 43 ++++++++++++++++++- .../bluetooth/nimble/blemesh/main/app_mesh.c | 5 ++- 9 files changed, 94 insertions(+), 13 deletions(-) diff --git a/components/bt/CMakeLists.txt b/components/bt/CMakeLists.txt index 2b8a96363..d9cae26f1 100644 --- a/components/bt/CMakeLists.txt +++ b/components/bt/CMakeLists.txt @@ -377,9 +377,13 @@ if(CONFIG_BT_ENABLED) host/nimble/nimble/nimble/host/store/ram/include host/nimble/nimble/nimble/host/store/config/include host/nimble/nimble/porting/npl/freertos/include - host/nimble/nimble/ext/tinycrypt/include host/nimble/esp-hci/include) + if(NOT CONFIG_BT_NIMBLE_CRYPTO_STACK_MBEDTLS) + + list(APPEND include_dirs + host/nimble/nimble/ext/tinycrypt/include) + list(APPEND srcs "host/nimble/nimble/ext/tinycrypt/src/utils.c" "host/nimble/nimble/ext/tinycrypt/src/sha256.c" "host/nimble/nimble/ext/tinycrypt/src/ecc.c" @@ -394,8 +398,10 @@ if(CONFIG_BT_ENABLED) "host/nimble/nimble/ext/tinycrypt/src/hmac_prng.c" "host/nimble/nimble/ext/tinycrypt/src/ecc_platform_specific.c" "host/nimble/nimble/ext/tinycrypt/src/hmac.c" - "host/nimble/nimble/ext/tinycrypt/src/cbc_mode.c" - "host/nimble/nimble/nimble/host/util/src/addr.c" + "host/nimble/nimble/ext/tinycrypt/src/cbc_mode.c") + endif() + + list(APPEND srcs "host/nimble/nimble/nimble/host/util/src/addr.c" "host/nimble/nimble/nimble/host/services/gatt/src/ble_svc_gatt.c" "host/nimble/nimble/nimble/host/services/tps/src/ble_svc_tps.c" "host/nimble/nimble/nimble/host/services/ias/src/ble_svc_ias.c" diff --git a/components/bt/component.mk b/components/bt/component.mk index 7fa74dcf0..da70ea45a 100644 --- a/components/bt/component.mk +++ b/components/bt/component.mk @@ -148,11 +148,12 @@ ifdef CONFIG_BLE_MESH esp_ble_mesh/mesh_models/common \ esp_ble_mesh/mesh_models/client \ esp_ble_mesh/api/core \ - esp_ble_mesh/api/models + esp_ble_mesh/api/models endif ifdef CONFIG_BT_NIMBLE_ENABLED + COMPONENT_ADD_INCLUDEDIRS += host/nimble/nimble/nimble/include \ host/nimble/nimble/nimble/host/include \ host/nimble/nimble/porting/nimble/include \ @@ -167,14 +168,16 @@ COMPONENT_ADD_INCLUDEDIRS += host/nimble/nimble/nimble/include host/nimble/nimble/nimble/host/util/include \ host/nimble/nimble/nimble/host/store/ram/include \ host/nimble/nimble/nimble/host/store/config/include \ - host/nimble/nimble/ext/tinycrypt/include \ host/nimble/esp-hci/include \ host/nimble/port/include +ifndef CONFIG_BT_NIMBLE_CRYPTO_STACK_MBEDTLS +COMPONENT_ADD_INCLUDEDIRS += host/nimble/nimble/ext/tinycrypt/include +endif + COMPONENT_SRCDIRS += host/nimble/nimble/nimble/host/src \ host/nimble/nimble/porting/nimble/src \ host/nimble/nimble/porting/npl/freertos/src \ - host/nimble/nimble/ext/tinycrypt/src \ host/nimble/nimble/nimble/host/services/ans/src \ host/nimble/nimble/nimble/host/services/bas/src \ host/nimble/nimble/nimble/host/services/gap/src \ @@ -187,6 +190,10 @@ COMPONENT_SRCDIRS += host/nimble/nimble/nimble/host/src host/nimble/nimble/nimble/host/store/config/src \ host/nimble/esp-hci/src +ifndef CONFIG_BT_NIMBLE_CRYPTO_STACK_MBEDTLS +COMPONENT_SRCDIRS += host/nimble/nimble/ext/tinycrypt/src +endif + COMPONENT_OBJEXCLUDE += host/nimble/nimble/nimble/host/store/config/src/ble_store_config_conf.o ifdef CONFIG_BT_NIMBLE_MESH diff --git a/components/bt/host/nimble/Kconfig.in b/components/bt/host/nimble/Kconfig.in index acf34a73f..edc77f7e8 100644 --- a/components/bt/host/nimble/Kconfig.in +++ b/components/bt/host/nimble/Kconfig.in @@ -99,11 +99,11 @@ config BT_NIMBLE_SM_SC Enable security manager secure connections config BT_NIMBLE_DEBUG - bool "Enable host debugging" + bool "Enable extra runtime asserts and host debugging" default n depends on BT_NIMBLE_ENABLED help - This enables extra runtime assertions + This enables extra runtime asserts and host debugging config BT_NIMBLE_SVC_GAP_DEVICE_NAME string "BLE GAP default device name" @@ -257,3 +257,12 @@ config BT_NIMBLE_MESH_DEVICE_NAME help This value defines Bluetooth Mesh device/node name +config BT_NIMBLE_CRYPTO_STACK_MBEDTLS + bool "Override TinyCrypt with mbedTLS for crypto computations" + default y + depends on BT_NIMBLE_ENABLED + select MBEDTLS_ECP_RESTARTABLE + select MBEDTLS_CMAC_C + help + Enable this option to choose mbedTLS instead of TinyCrypt for crypto + computations. diff --git a/components/bt/host/nimble/nimble b/components/bt/host/nimble/nimble index adcd94086..6c91a9a15 160000 --- a/components/bt/host/nimble/nimble +++ b/components/bt/host/nimble/nimble @@ -1 +1 @@ -Subproject commit adcd9408695cb4f873f117eb8c92007455b2c066 +Subproject commit 6c91a9a153c421231b686d30c822e53fea7510c0 diff --git a/components/bt/host/nimble/port/include/esp_nimble_cfg.h b/components/bt/host/nimble/port/include/esp_nimble_cfg.h index 7b9da7521..c0b329c46 100644 --- a/components/bt/host/nimble/port/include/esp_nimble_cfg.h +++ b/components/bt/host/nimble/port/include/esp_nimble_cfg.h @@ -567,6 +567,10 @@ #define MYNEWT_VAL_BLE_SM_THEIR_KEY_DIST (0) #endif +#ifndef MYNEWT_VAL_BLE_CRYPTO_STACK_MBEDTLS +#define MYNEWT_VAL_BLE_CRYPTO_STACK_MBEDTLS (CONFIG_BT_NIMBLE_CRYPTO_STACK_MBEDTLS) +#endif + #ifndef MYNEWT_VAL_BLE_STORE_MAX_BONDS #define MYNEWT_VAL_BLE_STORE_MAX_BONDS CONFIG_BT_NIMBLE_MAX_BONDS #endif diff --git a/components/mbedtls/Kconfig b/components/mbedtls/Kconfig index a9e7983bd..89bfa9abe 100644 --- a/components/mbedtls/Kconfig +++ b/components/mbedtls/Kconfig @@ -116,6 +116,19 @@ menu "mbedTLS" default 3 if MBEDTLS_DEBUG_LEVEL_DEBUG default 4 if MBEDTLS_DEBUG_LEVEL_VERBOSE + config MBEDTLS_ECP_RESTARTABLE + bool "Enable mbedTLS ecp restartable" + default n + help + Enable "non-blocking" ECC operations that can return early and be resumed. + + config MBEDTLS_CMAC_C + bool "Enable CMAC mode for block ciphers" + default n + help + Enable the CMAC (Cipher-based Message Authentication Code) mode for + block ciphers. + config MBEDTLS_HARDWARE_AES bool "Enable hardware AES acceleration" default y diff --git a/components/mbedtls/mbedtls b/components/mbedtls/mbedtls index 97959e779..f5f2e5926 160000 --- a/components/mbedtls/mbedtls +++ b/components/mbedtls/mbedtls @@ -1 +1 @@ -Subproject commit 97959e77912524bd8db7cbb2e00fc9f6189f7a82 +Subproject commit f5f2e5926cd294ae7cb579ff6a12ad9303caeb6e diff --git a/components/mbedtls/port/include/mbedtls/esp_config.h b/components/mbedtls/port/include/mbedtls/esp_config.h index bdb9bf61a..d971ab8db 100644 --- a/components/mbedtls/port/include/mbedtls/esp_config.h +++ b/components/mbedtls/port/include/mbedtls/esp_config.h @@ -218,7 +218,7 @@ /** * \def MBEDTLS_REMOVE_ARC4_CIPHERSUITES & MBEDTLS_ARC4_C - * + * * MBEDTLS_ARC4_C * Enable the ARCFOUR stream cipher. * @@ -253,6 +253,47 @@ #define MBEDTLS_REMOVE_ARC4_CIPHERSUITES #endif +/** + * \def MBEDTLS_ECP_RESTARTABLE + * + * Enable "non-blocking" ECC operations that can return early and be resumed. + * + * This allows various functions to pause by returning + * #MBEDTLS_ERR_ECP_IN_PROGRESS (or, for functions in the SSL module, + * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS) and then be called later again in + * order to further progress and eventually complete their operation. This is + * controlled through mbedtls_ecp_set_max_ops() which limits the maximum + * number of ECC operations a function may perform before pausing; see + * mbedtls_ecp_set_max_ops() for more information. + * + * This is useful in non-threaded environments if you want to avoid blocking + * for too long on ECC (and, hence, X.509 or SSL/TLS) operations. + * + * Uncomment this macro to enable restartable ECC computations. + * + * \note This option only works with the default software implementation of + * elliptic curve functionality. It is incompatible with + * MBEDTLS_ECP_ALT, MBEDTLS_ECDH_XXX_ALT and MBEDTLS_ECDSA_XXX_ALT. + */ +#ifdef CONFIG_MBEDTLS_ECP_RESTARTABLE +#define MBEDTLS_ECP_RESTARTABLE +#endif + +/** + * \def MBEDTLS_CMAC_C + * + * Enable the CMAC (Cipher-based Message Authentication Code) mode for block + * ciphers. + * + * Module: library/cmac.c + * + * Requires: MBEDTLS_AES_C or MBEDTLS_DES_C + * + */ +#ifdef CONFIG_MBEDTLS_CMAC_C +#define MBEDTLS_CMAC_C +#endif + /** * \def MBEDTLS_ECP_DP_SECP192R1_ENABLED * diff --git a/examples/bluetooth/nimble/blemesh/main/app_mesh.c b/examples/bluetooth/nimble/blemesh/main/app_mesh.c index 82dde7ec2..e70f11ff5 100644 --- a/examples/bluetooth/nimble/blemesh/main/app_mesh.c +++ b/examples/bluetooth/nimble/blemesh/main/app_mesh.c @@ -32,7 +32,7 @@ #include "mesh/mesh.h" static const char *tag = "NimBLE_MESH"; -void ble_store_ram_init(void); +void ble_store_config_init(void); #define BT_DBG_ENABLED (MYNEWT_VAL(BLE_MESH_DEBUG)) @@ -418,6 +418,7 @@ void blemesh_host_task(void *param) health_pub_init(); nimble_port_run(); + nimble_port_freertos_deinit(); } void app_main(void) @@ -438,7 +439,7 @@ void app_main(void) bt_mesh_register_gatt(); /* XXX Need to have template for store */ - ble_store_ram_init(); + ble_store_config_init(); nimble_port_freertos_init(blemesh_host_task); } From ccde8c7d2f02abcb25dc17a053f5db36b95e03cc Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 30 Sep 2019 16:11:09 +0200 Subject: [PATCH 28/30] spi_flash: load partition table before adding an external partition esp_partition_register_external did not call load_partitions, so if it was called before any call to esp_partition_find, then the main partition table would never be loaded. Introduce new function, ensure_partitions_loaded, and call it both from esp_partition_find and esp_partition_register_external. Closes https://github.com/espressif/esp-idf/issues/4116 --- components/spi_flash/partition.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/components/spi_flash/partition.c b/components/spi_flash/partition.c index 14de32c9f..4457d6336 100644 --- a/components/spi_flash/partition.c +++ b/components/spi_flash/partition.c @@ -55,27 +55,38 @@ typedef struct esp_partition_iterator_opaque_ { static esp_partition_iterator_opaque_t* iterator_create(esp_partition_type_t type, esp_partition_subtype_t subtype, const char* label); static esp_err_t load_partitions(void); +static esp_err_t ensure_partitions_loaded(void); +static const char* TAG = "partition"; static SLIST_HEAD(partition_list_head_, partition_list_item_) s_partition_list = SLIST_HEAD_INITIALIZER(s_partition_list); static _lock_t s_partition_list_lock; -esp_partition_iterator_t esp_partition_find(esp_partition_type_t type, - esp_partition_subtype_t subtype, const char* label) +static esp_err_t ensure_partitions_loaded(void) { + esp_err_t err = ESP_OK; if (SLIST_EMPTY(&s_partition_list)) { // only lock if list is empty (and check again after acquiring lock) _lock_acquire(&s_partition_list_lock); - esp_err_t err = ESP_OK; if (SLIST_EMPTY(&s_partition_list)) { + ESP_LOGD(TAG, "Loading the partition table"); err = load_partitions(); + if (err != ESP_OK) { + ESP_LOGE(TAG, "load_partitions returned 0x%x", err); + } } _lock_release(&s_partition_list_lock); - if (err != ESP_OK) { - return NULL; - } + } + return err; +} + +esp_partition_iterator_t esp_partition_find(esp_partition_type_t type, + esp_partition_subtype_t subtype, const char* label) +{ + if (ensure_partitions_loaded() != ESP_OK) { + return NULL; } // create an iterator pointing to the start of the list // (next item will be the first one) @@ -233,6 +244,11 @@ esp_err_t esp_partition_register_external(esp_flash_t* flash_chip, size_t offset return ESP_ERR_INVALID_SIZE; } + esp_err_t err = ensure_partitions_loaded(); + if (err != ESP_OK) { + return err; + } + partition_list_item_t* item = (partition_list_item_t*) calloc(sizeof(partition_list_item_t), 1); if (item == NULL) { return ESP_ERR_NO_MEM; From 391d7ee4c7327c1e303c92c9b2e1e7c1e1e2e894 Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 30 Sep 2019 16:26:34 +0200 Subject: [PATCH 29/30] examples/ext_flash_fatfs: print out data partitions, add CI test --- .../storage/ext_flash_fatfs/example_test.py | 27 +++++++++++++++++++ .../main/ext_flash_fatfs_example_main.c | 18 +++++++++++++ tools/ci/config/target-test.yml | 6 +++++ 3 files changed, 51 insertions(+) create mode 100644 examples/storage/ext_flash_fatfs/example_test.py diff --git a/examples/storage/ext_flash_fatfs/example_test.py b/examples/storage/ext_flash_fatfs/example_test.py new file mode 100644 index 000000000..a66ad8c3a --- /dev/null +++ b/examples/storage/ext_flash_fatfs/example_test.py @@ -0,0 +1,27 @@ +from __future__ import print_function +import os +import sys + +try: + import IDF +except ImportError: + test_fw_path = os.getenv('TEST_FW_PATH') + if test_fw_path and test_fw_path not in sys.path: + sys.path.insert(0, test_fw_path) + import IDF + + +@IDF.idf_example_test(env_tag='Example_ExtFlash') +def test_examples_storage_ext_flash_fatfs(env, extra_data): + dut = env.get_dut('ext_flash_fatfs', 'examples/storage/ext_flash_fatfs') + dut.start_app() + + dut.expect('Initialized external Flash') + dut.expect('partition \'nvs\'') + dut.expect('partition \'storage\'') + dut.expect('File written') + dut.expect('Read from file: \'Written using ESP-IDF') + + +if __name__ == '__main__': + test_examples_storage_ext_flash_fatfs() diff --git a/examples/storage/ext_flash_fatfs/main/ext_flash_fatfs_example_main.c b/examples/storage/ext_flash_fatfs/main/ext_flash_fatfs_example_main.c index 7f574fe63..a596a8119 100644 --- a/examples/storage/ext_flash_fatfs/main/ext_flash_fatfs_example_main.c +++ b/examples/storage/ext_flash_fatfs/main/ext_flash_fatfs_example_main.c @@ -30,6 +30,7 @@ const char *base_path = "/extflash"; static esp_flash_t* example_init_ext_flash(void); static const esp_partition_t* example_add_partition(esp_flash_t* ext_flash, const char* partition_label); +static void example_list_data_partitions(void); static bool example_mount_fatfs(const char* partition_label); static void example_get_fatfs_usage(size_t* out_total_bytes, size_t* out_free_bytes); @@ -45,6 +46,9 @@ void app_main(void) const char *partition_label = "storage"; example_add_partition(flash, partition_label); + // List the available partitions + example_list_data_partitions(); + // Initialize FAT FS in the partition if (!example_mount_fatfs(partition_label)) { return; @@ -139,6 +143,20 @@ static const esp_partition_t* example_add_partition(esp_flash_t* ext_flash, cons return fat_partition; } +static void example_list_data_partitions(void) +{ + ESP_LOGI(TAG, "Listing data partitions:"); + esp_partition_iterator_t it = esp_partition_find(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, NULL); + + for (; it != NULL; it = esp_partition_next(it)) { + const esp_partition_t *part = esp_partition_get(it); + ESP_LOGI(TAG, "- partition '%s', subtype %d, offset 0x%x, size %d kB", + part->label, part->subtype, part->address, part->size / 1024); + } + + esp_partition_iterator_release(it); +} + static bool example_mount_fatfs(const char* partition_label) { ESP_LOGI(TAG, "Mounting FAT filesystem"); diff --git a/tools/ci/config/target-test.yml b/tools/ci/config/target-test.yml index 963571cb4..587a370b7 100644 --- a/tools/ci/config/target-test.yml +++ b/tools/ci/config/target-test.yml @@ -215,6 +215,12 @@ example_test_009: variables: SETUP_TOOLS: "1" +example_test_010: + extends: .example_test_template + tags: + - ESP32 + - Example_ExtFlash + UT_001: extends: .unit_test_template parallel: 50 From 3f9b0d8c2a923479942efaedcc189b63cdd6c2e5 Mon Sep 17 00:00:00 2001 From: Mahavir Jain Date: Wed, 11 Sep 2019 16:12:53 +0530 Subject: [PATCH 30/30] nghttp: move config.h to private_include dirs --- components/nghttp/CMakeLists.txt | 3 ++- components/nghttp/component.mk | 2 ++ components/nghttp/{port/include => private_include}/config.h | 0 3 files changed, 4 insertions(+), 1 deletion(-) rename components/nghttp/{port/include => private_include}/config.h (100%) diff --git a/components/nghttp/CMakeLists.txt b/components/nghttp/CMakeLists.txt index 1c9ba17a2..f97cc7731 100644 --- a/components/nghttp/CMakeLists.txt +++ b/components/nghttp/CMakeLists.txt @@ -24,6 +24,7 @@ set(srcs "port/http_parser.c") idf_component_register(SRCS "${srcs}" - INCLUDE_DIRS port/include nghttp2/lib/includes) + INCLUDE_DIRS port/include nghttp2/lib/includes + PRIV_INCLUDE_DIRS private_include) target_compile_definitions(${COMPONENT_LIB} PUBLIC "-DHAVE_CONFIG_H") diff --git a/components/nghttp/component.mk b/components/nghttp/component.mk index 2a69cd5e1..10d0bdefb 100644 --- a/components/nghttp/component.mk +++ b/components/nghttp/component.mk @@ -4,6 +4,8 @@ COMPONENT_ADD_INCLUDEDIRS := port/include nghttp2/lib/includes +COMPONENT_PRIV_INCLUDEDIRS := private_include + COMPONENT_SRCDIRS := nghttp2/lib port COMPONENT_SUBMODULES := nghttp2 diff --git a/components/nghttp/port/include/config.h b/components/nghttp/private_include/config.h similarity index 100% rename from components/nghttp/port/include/config.h rename to components/nghttp/private_include/config.h