diff --git a/components/nvs_flash/README.rst b/components/nvs_flash/README.rst index ee8ee0b26..2f1c46913 100644 --- a/components/nvs_flash/README.rst +++ b/components/nvs_flash/README.rst @@ -217,3 +217,11 @@ As mentioned above, each key-value pair belongs to one of the namespaces. Namesp +-------------------------------------------+ +Item hash list +~~~~~~~~~~~~~~ + +To reduce the number of reads performed from flash memory, each member of Page class maintains a list of pairs: (item index; item hash). This list makes searches much quicker. Instead of iterating over all entries, reading them from flash one at a time, ``Page::findItem`` first performs search for item hash in the hash list. This gives the item index within the page, if such an item exists. Due to a hash collision it is possible that a different item will be found. This is handled by falling back to iteration over items in flash. + +Each node in hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace and key name. CRC32 is used for calculation, result is truncated to 24 bits. To reduce overhead of storing 32-bit entries in a linked list, list is implemented as a doubly-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and 32-bit count field. Minimal amount of extra RAM useage per page is therefore 128 bytes, maximum is 640 bytes. + + diff --git a/components/nvs_flash/include/nvs.h b/components/nvs_flash/include/nvs.h index 6fff2dabf..912ea2210 100644 --- a/components/nvs_flash/include/nvs.h +++ b/components/nvs_flash/include/nvs.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include "esp_err.h" #ifdef __cplusplus extern "C" { @@ -78,9 +78,8 @@ esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_ha * This family of functions set value for the key, given its name. Note that * actual storage will not be updated until nvs_commit function is called. * - * @param[in] handle Handle obtained from nvs_open function. If the handle was - * opened with read_only set to true, nvs_set_X functions will - * fail with ESP_ERR_NVS_READONLY. + * @param[in] handle Handle obtained from nvs_open function. + * Handles that were opened read only cannot be used. * @param[in] key Key name. Maximal length is determined by the underlying * implementation, but is guaranteed to be at least * 16 characters. Shouldn't be empty. @@ -180,6 +179,41 @@ esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value); esp_err_t nvs_get_str (nvs_handle handle, const char* key, char* out_value, size_t* length); esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size_t* length); +/** + * @brief Erase key-value pair with given key name. + * + * Note that actual storage may not be updated until nvs_commit function is called. + * + * @param[in] handle Storage handle obtained with nvs_open. + * Handles that were opened read only cannot be used. + * + * @param[in] key Key name. Maximal length is determined by the underlying + * implementation, but is guaranteed to be at least + * 16 characters. Shouldn't be empty. + * + * @return - ESP_OK if erase operation was successful + * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL + * - ESP_ERR_NVS_READ_ONLY if handle was opened as read only + * - ESP_ERR_NVS_NOT_FOUND if the requested key doesn't exist + * - other error codes from the underlying storage driver + */ +esp_err_t nvs_erase_key(nvs_handle handle, const char* key); + +/** + * @brief Erase all key-value pairs in a namespace + * + * Note that actual storage may not be updated until nvs_commit function is called. + * + * @param[in] handle Storage handle obtained with nvs_open. + * Handles that were opened read only cannot be used. + * + * @return - ESP_OK if erase operation was successful + * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL + * - ESP_ERR_NVS_READ_ONLY if handle was opened as read only + * - other error codes from the underlying storage driver + */ +esp_err_t nvs_erase_all(nvs_handle handle); + /** * @brief Write any pending changes to non-volatile storage * @@ -187,8 +221,8 @@ esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size * to non-volatile storage. Individual implementations may write to storage at other times, * but this is not guaranteed. * - * @param[in] handle Storage handle obtained with nvs_open. If handle has to be - * opened as not read only for this call to succeed. + * @param[in] handle Storage handle obtained with nvs_open. + * Handles that were opened read only cannot be used. * * @return - ESP_OK if the changes have been written successfully * - ESP_ERR_NVS_INVALID_HANDLE if handle has been closed or is NULL diff --git a/components/nvs_flash/src/nvs_api.cpp b/components/nvs_flash/src/nvs_api.cpp index f9292e680..00a279d2b 100644 --- a/components/nvs_flash/src/nvs_api.cpp +++ b/components/nvs_flash/src/nvs_api.cpp @@ -17,18 +17,27 @@ #include "intrusive_list.h" #include "nvs_platform.hpp" +#ifdef ESP_PLATFORM +// Uncomment this line to force output from this module +// #define LOG_LOCAL_LEVEL ESP_LOG_DEBUG +#include "esp_log.h" +static const char* TAG = "nvs"; +#else +#define ESP_LOGD(...) +#endif + class HandleEntry : public intrusive_list_node { public: - HandleEntry(){} - + HandleEntry() {} + HandleEntry(nvs_handle handle, bool readOnly, uint8_t nsIndex) : - mHandle(handle), - mReadOnly(readOnly), - mNsIndex(nsIndex) + mHandle(handle), + mReadOnly(readOnly), + mNsIndex(nsIndex) { } - + nvs_handle mHandle; uint8_t mReadOnly; uint8_t mNsIndex; @@ -55,7 +64,7 @@ extern "C" esp_err_t nvs_flash_init(uint32_t baseSector, uint32_t sectorCount) { Lock::init(); Lock lock; - NVS_DEBUGV("%s %d %d\r\n", __func__, baseSector, sectorCount); + ESP_LOGD(TAG, "init start=%d count=%d", baseSector, sectorCount); s_nvs_handles.clear(); return s_nvs_storage.init(baseSector, sectorCount); } @@ -75,7 +84,7 @@ static esp_err_t nvs_find_ns_handle(nvs_handle handle, HandleEntry& entry) extern "C" esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_handle) { Lock lock; - NVS_DEBUGV("%s %s %d\r\n", __func__, name, open_mode); + ESP_LOGD(TAG, "%s %s %d", __func__, name, open_mode); uint8_t nsIndex; esp_err_t err = s_nvs_storage.createOrOpenNamespace(name, open_mode == NVS_READWRITE, nsIndex); if (err != ESP_OK) { @@ -93,7 +102,7 @@ extern "C" esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_han extern "C" void nvs_close(nvs_handle handle) { Lock lock; - NVS_DEBUGV("%s %d\r\n", __func__, handle); + ESP_LOGD(TAG, "%s %d", __func__, handle); auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](HandleEntry& e) -> bool { return e.mHandle == handle; }); @@ -103,11 +112,41 @@ extern "C" void nvs_close(nvs_handle handle) s_nvs_handles.erase(it); } +extern "C" esp_err_t nvs_erase_key(nvs_handle handle, const char* key) +{ + Lock lock; + ESP_LOGD(TAG, "%s %s\r\n", __func__, key); + HandleEntry entry; + auto err = nvs_find_ns_handle(handle, entry); + if (err != ESP_OK) { + return err; + } + if (entry.mReadOnly) { + return ESP_ERR_NVS_READ_ONLY; + } + return s_nvs_storage.eraseItem(entry.mNsIndex, key); +} + +extern "C" esp_err_t nvs_erase_all(nvs_handle handle) +{ + Lock lock; + ESP_LOGD(TAG, "%s\r\n", __func__); + HandleEntry entry; + auto err = nvs_find_ns_handle(handle, entry); + if (err != ESP_OK) { + return err; + } + if (entry.mReadOnly) { + return ESP_ERR_NVS_READ_ONLY; + } + return s_nvs_storage.eraseNamespace(entry.mNsIndex); +} + template static esp_err_t nvs_set(nvs_handle handle, const char* key, T value) { Lock lock; - NVS_DEBUGV("%s %s %d %d\r\n", __func__, key, sizeof(T), (uint32_t) value); + ESP_LOGD(TAG, "%s %s %d %d", __func__, key, sizeof(T), (uint32_t) value); HandleEntry entry; auto err = nvs_find_ns_handle(handle, entry); if (err != ESP_OK) { @@ -170,7 +209,7 @@ extern "C" esp_err_t nvs_commit(nvs_handle handle) extern "C" esp_err_t nvs_set_str(nvs_handle handle, const char* key, const char* value) { Lock lock; - NVS_DEBUGV("%s %s %s\r\n", __func__, key, value); + ESP_LOGD(TAG, "%s %s %s", __func__, key, value); HandleEntry entry; auto err = nvs_find_ns_handle(handle, entry); if (err != ESP_OK) { @@ -182,7 +221,7 @@ extern "C" esp_err_t nvs_set_str(nvs_handle handle, const char* key, const char* extern "C" esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, size_t length) { Lock lock; - NVS_DEBUGV("%s %s %d\r\n", __func__, key, length); + ESP_LOGD(TAG, "%s %s %d", __func__, key, length); HandleEntry entry; auto err = nvs_find_ns_handle(handle, entry); if (err != ESP_OK) { @@ -196,7 +235,7 @@ template static esp_err_t nvs_get(nvs_handle handle, const char* key, T* out_value) { Lock lock; - NVS_DEBUGV("%s %s %d\r\n", __func__, key, sizeof(T)); + ESP_LOGD(TAG, "%s %s %d", __func__, key, sizeof(T)); HandleEntry entry; auto err = nvs_find_ns_handle(handle, entry); if (err != ESP_OK) { @@ -248,7 +287,7 @@ extern "C" esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* static esp_err_t nvs_get_str_or_blob(nvs_handle handle, nvs::ItemType type, const char* key, void* out_value, size_t* length) { Lock lock; - NVS_DEBUGV("%s %s\r\n", __func__, key); + ESP_LOGD(TAG, "%s %s", __func__, key); HandleEntry entry; auto err = nvs_find_ns_handle(handle, entry); if (err != ESP_OK) { @@ -263,12 +302,10 @@ static esp_err_t nvs_get_str_or_blob(nvs_handle handle, nvs::ItemType type, cons if (length == nullptr) { return ESP_ERR_NVS_INVALID_LENGTH; - } - else if (out_value == nullptr) { + } else if (out_value == nullptr) { *length = dataSize; return ESP_OK; - } - else if (*length < dataSize) { + } else if (*length < dataSize) { *length = dataSize; return ESP_ERR_NVS_INVALID_LENGTH; } diff --git a/components/nvs_flash/src/nvs_item_hash_list.cpp b/components/nvs_flash/src/nvs_item_hash_list.cpp new file mode 100644 index 000000000..7fa019dff --- /dev/null +++ b/components/nvs_flash/src/nvs_item_hash_list.cpp @@ -0,0 +1,96 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "nvs_item_hash_list.hpp" + +namespace nvs +{ + +HashList::~HashList() +{ + for (auto it = mBlockList.begin(); it != mBlockList.end();) { + auto tmp = it; + ++it; + mBlockList.erase(tmp); + delete static_cast(tmp); + } +} + +HashList::HashListBlock::HashListBlock() +{ + static_assert(sizeof(HashListBlock) == HashListBlock::BYTE_SIZE, + "cache block size calculation incorrect"); +} + +void HashList::insert(const Item& item, size_t index) +{ + const uint32_t hash_24 = item.calculateCrc32WithoutValue() & 0xffffff; + // add entry to the end of last block if possible + if (mBlockList.size()) { + auto& block = mBlockList.back(); + if (block.mCount < HashListBlock::ENTRY_COUNT) { + block.mNodes[block.mCount++] = HashListNode(hash_24, index); + return; + } + } + // if the above failed, create a new block and add entry to it + HashListBlock* newBlock = new HashListBlock; + mBlockList.push_back(newBlock); + newBlock->mNodes[0] = HashListNode(hash_24, index); + newBlock->mCount++; +} + +void HashList::erase(size_t index) +{ + for (auto it = std::begin(mBlockList); it != std::end(mBlockList);) { + bool haveEntries = false; + for (size_t i = 0; i < it->mCount; ++i) { + if (it->mNodes[i].mIndex == index) { + it->mNodes[i].mIndex = 0xff; + return; + } + if (it->mNodes[i].mIndex != 0xff) { + haveEntries = true; + } + } + if (!haveEntries) { + auto tmp = it; + ++it; + mBlockList.erase(tmp); + delete static_cast(tmp); + } else { + ++it; + } + } + assert(false && "item should have been present in cache"); +} + +size_t HashList::find(size_t start, const Item& item) +{ + const uint32_t hash_24 = item.calculateCrc32WithoutValue() & 0xffffff; + for (auto it = std::begin(mBlockList); it != std::end(mBlockList); ++it) { + for (size_t index = 0; index < it->mCount; ++index) { + HashListNode& e = it->mNodes[index]; + if (e.mIndex >= start && + e.mHash == hash_24 && + e.mIndex != 0xff) { + return e.mIndex; + } + } + } + return SIZE_MAX; +} + + +} // namespace nvs diff --git a/components/nvs_flash/src/nvs_item_hash_list.hpp b/components/nvs_flash/src/nvs_item_hash_list.hpp new file mode 100644 index 000000000..b40a53d61 --- /dev/null +++ b/components/nvs_flash/src/nvs_item_hash_list.hpp @@ -0,0 +1,68 @@ +// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef nvs_item_hash_list_h +#define nvs_item_hash_list_h + +#include "nvs.h" +#include "nvs_types.hpp" +#include "intrusive_list.h" + +namespace nvs +{ + +class HashList +{ +public: + ~HashList(); + void insert(const Item& item, size_t index); + void erase(const size_t index); + size_t find(size_t start, const Item& item); + +protected: + + struct HashListNode { + HashListNode() : + mIndex(0xff), mHash(0) + { + } + + HashListNode(uint32_t hash, size_t index) : + mIndex((uint32_t) index), mHash(hash) + { + } + + uint32_t mIndex : 8; + uint32_t mHash : 24; + }; + + struct HashListBlock : public intrusive_list_node { + HashListBlock(); + + static const size_t BYTE_SIZE = 128; + static const size_t ENTRY_COUNT = (BYTE_SIZE - sizeof(intrusive_list_node) - sizeof(size_t)) / 4; + + size_t mCount = 0; + HashListNode mNodes[ENTRY_COUNT]; + }; + + + typedef intrusive_list TBlockList; + TBlockList mBlockList; +}; // class HashList + +} // namespace nvs + + +#endif /* nvs_item_hash_list_h */ diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index cddb69393..4f6a198c6 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -29,7 +29,7 @@ uint32_t Page::Header::calculateCrc32() reinterpret_cast(this) + offsetof(Header, mSeqNumber), offsetof(Header, mCrc32) - offsetof(Header, mSeqNumber)); } - + esp_err_t Page::load(uint32_t sectorNumber) { mBaseAddress = sectorNumber * SEC_SIZE; @@ -59,15 +59,13 @@ esp_err_t Page::load(uint32_t sectorNumber) break; } } - } - else if (header.mCrc32 != header.calculateCrc32()) { + } else if (header.mCrc32 != header.calculateCrc32()) { header.mState = PageState::CORRUPT; - } - else { + } else { mState = header.mState; mSeqNumber = header.mSeqNumber; } - + switch (mState) { case PageState::UNINITIALIZED: break; @@ -108,6 +106,27 @@ esp_err_t Page::writeEntry(const Item& item) return ESP_OK; } + +esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) +{ + assert(size % ENTRY_SIZE == 0); + assert(mNextFreeEntry != INVALID_ENTRY); + assert(mFirstUsedEntry != INVALID_ENTRY); + const uint16_t count = size / ENTRY_SIZE; + + auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), reinterpret_cast(data), static_cast(size)); + if (rc != ESP_OK) { + mState = PageState::INVALID; + return rc; + } + auto err = alterEntryRangeState(mNextFreeEntry, mNextFreeEntry + count, EntryState::WRITTEN); + if (err != ESP_OK) { + return err; + } + mUsedEntryCount += count; + mNextFreeEntry += count; + return ESP_OK; +} esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { @@ -148,16 +167,9 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c // write first item - item.nsIndex = nsIndex; - item.datatype = datatype; - item.span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE; - item.reserved = 0xff; - - std::fill_n(reinterpret_cast(item.key), sizeof(item.key) / 4, 0xffffffff); - std::fill_n(reinterpret_cast(item.data), sizeof(item.data) / 4, 0xffffffff); - - strncpy(item.key, key, sizeof(item.key) - 1); - item.key[sizeof(item.key) - 1] = 0; + size_t span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE; + item = Item(nsIndex, datatype, span, key); + mHashList.insert(item, mNextFreeEntry); if (datatype != ItemType::SZ && datatype != ItemType::BLOB) { memcpy(item.data, data, dataSize); @@ -177,18 +189,24 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return err; } - size_t left = dataSize; - while (left != 0) { - size_t willWrite = Page::ENTRY_SIZE; - willWrite = (left < willWrite)?left:willWrite; - memcpy(item.rawData, src, willWrite); - src += willWrite; - left -= willWrite; + size_t left = dataSize / ENTRY_SIZE * ENTRY_SIZE; + if (left > 0) { + err = writeEntryData(static_cast(data), left); + if (err != ESP_OK) { + return err; + } + } + + size_t tail = dataSize - left; + if (tail > 0) { + std::fill_n(item.rawData, ENTRY_SIZE / 4, 0xffffffff); + memcpy(item.rawData, static_cast(data) + left, tail); err = writeEntry(item); if (err != ESP_OK) { return err; } } + } return ESP_OK; } @@ -260,23 +278,11 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key) return findItem(nsIndex, datatype, key, index, item); } -esp_err_t Page::eraseEntry(size_t index) -{ - auto state = mEntryTable.get(index); - assert(state == EntryState::WRITTEN || state == EntryState::EMPTY); - - auto rc = alterEntryState(index, EntryState::ERASED); - if (rc != ESP_OK) { - return rc; - } - - return ESP_OK; -} - esp_err_t Page::eraseEntryAndSpan(size_t index) { auto state = mEntryTable.get(index); assert(state == EntryState::WRITTEN || state == EntryState::EMPTY); + mHashList.erase(index); size_t span = 1; if (state == EntryState::WRITTEN) { @@ -296,15 +302,18 @@ esp_err_t Page::eraseEntryAndSpan(size_t index) if (mEntryTable.get(i) == EntryState::WRITTEN) { --mUsedEntryCount; } - rc = alterEntryState(i, EntryState::ERASED); - if (rc != ESP_OK) { - return rc; - } ++mErasedEntryCount; } + if (span == 1) { + rc = alterEntryState(index, EntryState::ERASED); + } else { + rc = alterEntryRangeState(index, index + span, EntryState::ERASED); + } + if (rc != ESP_OK) { + return rc; + } } - } - else { + } else { auto rc = alterEntryState(index, EntryState::ERASED); if (rc != ESP_OK) { return rc; @@ -314,7 +323,7 @@ esp_err_t Page::eraseEntryAndSpan(size_t index) if (index == mFirstUsedEntry) { updateFirstUsedEntry(index, span); } - + if (index + span > mNextFreeEntry) { mNextFreeEntry = index + span; } @@ -337,7 +346,7 @@ void Page::updateFirstUsedEntry(size_t index, size_t span) } } } - + esp_err_t Page::moveItem(Page& other) { if (mFirstUsedEntry == INVALID_ENTRY) { @@ -347,7 +356,7 @@ esp_err_t Page::moveItem(Page& other) if (mFindInfo.itemIndex() == mFirstUsedEntry) { invalidateCache(); } - + if (other.mState == PageState::UNINITIALIZED) { auto err = other.initialize(); if (err != ESP_OK) { @@ -360,6 +369,7 @@ esp_err_t Page::moveItem(Page& other) if (err != ESP_OK) { return err; } + other.mHashList.insert(entry, other.mNextFreeEntry); err = other.writeEntry(entry); if (err != ESP_OK) { return err; @@ -367,9 +377,9 @@ esp_err_t Page::moveItem(Page& other) size_t span = entry.span; size_t end = mFirstUsedEntry + span; - + assert(mFirstUsedEntry != INVALID_ENTRY || span == 1); - + for (size_t i = mFirstUsedEntry + 1; i < end; ++i) { readEntry(i, entry); err = other.writeEntry(entry); @@ -377,17 +387,7 @@ esp_err_t Page::moveItem(Page& other) return err; } } - for (size_t i = mFirstUsedEntry; i < end; ++i) { - err = eraseEntry(i); - if (err != ESP_OK) { - return err; - } - } - updateFirstUsedEntry(mFirstUsedEntry, span); - mErasedEntryCount += span; - mUsedEntryCount -= span; - - return ESP_OK; + return eraseEntryAndSpan(mFirstUsedEntry); } esp_err_t Page::mLoadEntryTable() @@ -432,7 +432,7 @@ esp_err_t Page::mLoadEntryTable() // but before the entry state table was altered, the entry locacted via // entry state table may actually be half-written. // this is easy to check by reading EntryHeader (i.e. first word) - if (mNextFreeEntry != INVALID_ENTRY) { + while (mNextFreeEntry < ENTRY_COUNT) { uint32_t entryAddress = getEntryAddress(mNextFreeEntry); uint32_t header; auto rc = spi_flash_read(entryAddress, &header, sizeof(header)); @@ -441,12 +441,20 @@ esp_err_t Page::mLoadEntryTable() return rc; } if (header != 0xffffffff) { + auto oldState = mEntryTable.get(mNextFreeEntry); auto err = alterEntryState(mNextFreeEntry, EntryState::ERASED); if (err != ESP_OK) { mState = PageState::INVALID; return err; } ++mNextFreeEntry; + if (oldState == EntryState::WRITTEN) { + --mUsedEntryCount; + } + ++mErasedEntryCount; + } + else { + break; } } @@ -462,7 +470,7 @@ esp_err_t Page::mLoadEntryTable() lastItemIndex = INVALID_ENTRY; continue; } - + lastItemIndex = i; auto err = readEntry(i, item); @@ -480,6 +488,8 @@ esp_err_t Page::mLoadEntryTable() continue; } + mHashList.insert(item, i); + if (item.datatype != ItemType::BLOB && item.datatype != ItemType::SZ) { continue; } @@ -498,7 +508,7 @@ esp_err_t Page::mLoadEntryTable() } i += span - 1; } - + // check that last item is not duplicate if (lastItemIndex != INVALID_ENTRY) { size_t findItemIndex = 0; @@ -513,6 +523,27 @@ esp_err_t Page::mLoadEntryTable() } } } + } else if (mState == PageState::FULL || mState == PageState::FREEING) { + // We have already filled mHashList for page in active state. + // Do the same for the case when page is in full or freeing state. + Item item; + for (size_t i = mFirstUsedEntry; i < ENTRY_COUNT; ++i) { + if (mEntryTable.get(i) != EntryState::WRITTEN) { + continue; + } + + auto err = readEntry(i, item); + if (err != ESP_OK) { + mState = PageState::INVALID; + return err; + } + + mHashList.insert(item, i); + + size_t span = item.span; + i += span - 1; + } + } return ESP_OK; @@ -527,7 +558,7 @@ esp_err_t Page::initialize() header.mState = mState; header.mSeqNumber = mSeqNumber; header.mCrc32 = header.calculateCrc32(); - + auto rc = spi_flash_write(mBaseAddress, reinterpret_cast(&header), sizeof(header)); if (rc != ESP_OK) { mState = PageState::INVALID; @@ -554,6 +585,31 @@ esp_err_t Page::alterEntryState(size_t index, EntryState state) return ESP_OK; } +esp_err_t Page::alterEntryRangeState(size_t begin, size_t end, EntryState state) +{ + assert(end <= ENTRY_COUNT); + assert(end > begin); + size_t wordIndex = mEntryTable.getWordIndex(end - 1); + for (ptrdiff_t i = end - 1; i >= static_cast(begin); --i) { + mEntryTable.set(i, state); + size_t nextWordIndex; + if (i == static_cast(begin)) { + nextWordIndex = (size_t) -1; + } else { + nextWordIndex = mEntryTable.getWordIndex(i - 1); + } + if (nextWordIndex != wordIndex) { + uint32_t word = mEntryTable.data()[wordIndex]; + auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast(wordIndex) * 4, &word, 4); + if (rc != ESP_OK) { + return rc; + } + } + wordIndex = nextWordIndex; + } + return ESP_OK; +} + esp_err_t Page::alterPageState(PageState state) { auto rc = spi_flash_write(mBaseAddress, reinterpret_cast(&state), sizeof(state)); @@ -579,7 +635,7 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si if (mState == PageState::CORRUPT || mState == PageState::INVALID || mState == PageState::UNINITIALIZED) { return ESP_ERR_NVS_NOT_FOUND; } - + if (itemIndex >= ENTRY_COUNT) { return ESP_ERR_NVS_NOT_FOUND; } @@ -593,12 +649,21 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si if (itemIndex > mFirstUsedEntry && itemIndex < ENTRY_COUNT) { start = itemIndex; } - + size_t end = mNextFreeEntry; if (end > ENTRY_COUNT) { end = ENTRY_COUNT; } + if (nsIndex != NS_ANY && datatype != ItemType::ANY && key != NULL) { + size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key)); + if (cachedIndex < ENTRY_COUNT) { + start = cachedIndex; + } else { + return ESP_ERR_NVS_NOT_FOUND; + } + } + size_t next; for (size_t i = start; i < end; i = next) { next = i + 1; @@ -671,7 +736,13 @@ esp_err_t Page::erase() mState = PageState::INVALID; return rc; } - return load(sector); + mUsedEntryCount = 0; + mErasedEntryCount = 0; + mFirstUsedEntry = INVALID_ENTRY; + mNextFreeEntry = INVALID_ENTRY; + mState = PageState::UNINITIALIZED; + mHashList = HashList(); + return ESP_OK; } esp_err_t Page::markFreeing() @@ -695,7 +766,7 @@ void Page::invalidateCache() { mFindInfo = CachedFindInfo(); } - + void Page::debugDump() const { printf("state=%x addr=%x seq=%d\nfirstUsed=%d nextFree=%d used=%d erased=%d\n", mState, mBaseAddress, mSeqNumber, static_cast(mFirstUsedEntry), static_cast(mNextFreeEntry), mUsedEntryCount, mErasedEntryCount); @@ -705,18 +776,15 @@ void Page::debugDump() const EntryState state = mEntryTable.get(i); if (state == EntryState::EMPTY) { printf("E\n"); - } - else if (state == EntryState::ERASED) { + } else if (state == EntryState::ERASED) { printf("X\n"); - } - else if (state == EntryState::WRITTEN) { + } else if (state == EntryState::WRITTEN) { Item item; readEntry(i, item); if (skip == 0) { printf("W ns=%2u type=%2u span=%3u key=\"%s\"\n", item.nsIndex, static_cast(item.datatype), item.span, item.key); skip = item.span - 1; - } - else { + } else { printf("D\n"); skip--; } diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 9afc6ff21..c1f430cae 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -23,6 +23,7 @@ #include "esp_spi_flash.h" #include "compressed_enum_table.hpp" #include "intrusive_list.h" +#include "nvs_item_hash_list.hpp" namespace nvs { @@ -161,25 +162,27 @@ public: esp_err_t erase(); void invalidateCache(); - + void debugDump() const; protected: - class Header { + class Header + { public: - Header() { + Header() + { std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT32_MAX); } - + PageState mState; // page state uint32_t mSeqNumber; // sequence number of this page uint32_t mReserved[5]; // unused, must be 0xffffffff uint32_t mCrc32; // crc of everything except mState - + uint32_t calculateCrc32(); }; - + enum class EntryState { EMPTY = 0x3, // 0b11, default state after flash erase WRITTEN = EMPTY & ~ESB_WRITTEN, // entry was written @@ -193,16 +196,18 @@ protected: esp_err_t alterEntryState(size_t index, EntryState state); + esp_err_t alterEntryRangeState(size_t begin, size_t end, EntryState state); + esp_err_t alterPageState(PageState state); esp_err_t readEntry(size_t index, Item& dst) const; esp_err_t writeEntry(const Item& item); + + esp_err_t writeEntryData(const uint8_t* data, size_t size); - esp_err_t eraseEntry(size_t index); - esp_err_t eraseEntryAndSpan(size_t index); - + void updateFirstUsedEntry(size_t index, size_t span); static constexpr size_t getAlignmentForType(ItemType type) @@ -229,6 +234,7 @@ protected: uint16_t mErasedEntryCount = 0; CachedFindInfo mFindInfo; + HashList mHashList; static const uint32_t HEADER_OFFSET = 0; static const uint32_t ENTRY_TABLE_OFFSET = HEADER_OFFSET + 32; diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index c0e904e35..790ab7e19 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -47,13 +47,12 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) if (mPageList.empty()) { mSeqNumber = 0; return activatePage(); - } - else { + } else { uint32_t lastSeqNo; assert(mPageList.back().getSeqNumber(lastSeqNo) == ESP_OK); mSeqNumber = lastSeqNo + 1; } - + // if power went out after a new item for the given key was written, // but before the old one was erased, we end up with a duplicate item Page& lastPage = back(); @@ -64,7 +63,7 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) itemIndex += item.span; lastItemIndex = itemIndex; } - + if (lastItemIndex != SIZE_MAX) { auto last = PageManager::TPageListIterator(&lastPage); for (auto it = begin(); it != last; ++it) { @@ -78,7 +77,7 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) for (auto it = begin(); it!= end(); ++it) { if (it->state() == Page::PageState::FREEING) { Page* newPage = &mPageList.back(); - if(newPage->state() != Page::PageState::ACTIVE) { + if (newPage->state() != Page::PageState::ACTIVE) { auto err = activatePage(); if (err != ESP_OK) { return err; @@ -93,12 +92,12 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) return err; } } - + auto err = it->erase(); if (err != ESP_OK) { return err; } - + Page* p = static_cast(it); mPageList.erase(it); mFreePageList.push_back(p); @@ -139,7 +138,7 @@ esp_err_t PageManager::requestNewPage() if (err != ESP_OK) { return err; } - + Page* newPage = &mPageList.back(); Page* erasedPage = maxErasedItemsPageIt; @@ -161,7 +160,7 @@ esp_err_t PageManager::requestNewPage() if (err != ESP_OK) { return err; } - + assert(usedEntries == newPage->getUsedEntryCount()); mPageList.erase(maxErasedItemsPageIt); @@ -188,5 +187,5 @@ esp_err_t PageManager::activatePage() ++mSeqNumber; return ESP_OK; } - + } // namespace nvs diff --git a/components/nvs_flash/src/nvs_pagemanager.hpp b/components/nvs_flash/src/nvs_pagemanager.hpp index 3484e70a1..10c545f0f 100644 --- a/components/nvs_flash/src/nvs_pagemanager.hpp +++ b/components/nvs_flash/src/nvs_pagemanager.hpp @@ -52,7 +52,7 @@ public: protected: friend class Iterator; - + esp_err_t activatePage(); TPageList mPageList; diff --git a/components/nvs_flash/src/nvs_platform.hpp b/components/nvs_flash/src/nvs_platform.hpp index d0bcae90c..374dbca6c 100644 --- a/components/nvs_flash/src/nvs_platform.hpp +++ b/components/nvs_flash/src/nvs_platform.hpp @@ -61,7 +61,6 @@ public: } // namespace nvs #else // ESP_PLATFORM -#define NVS_DEBUGV(...) printf(__VA_ARGS__) namespace nvs { class Lock @@ -75,10 +74,5 @@ public: } // namespace nvs #endif // ESP_PLATFORM -#ifndef CONFIG_NVS_DEBUG -#undef NVS_DEBUGV -#define NVS_DEBUGV(...) -#endif - #endif /* nvs_platform_h */ diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index 9be6580a1..4a217ebe1 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -51,7 +51,7 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) Page& p = *it; size_t itemIndex = 0; Item item; - while(p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { + while (p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { NamespaceEntry* entry = new NamespaceEntry; item.getKey(entry->mName, sizeof(entry->mName) - 1); item.getValue(entry->mIndex); @@ -69,6 +69,19 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) return ESP_OK; } +esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item) +{ + size_t itemIndex = 0; + for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + auto err = it->findItem(nsIndex, datatype, key, itemIndex, item); + if (err == ESP_OK) { + page = it; + return ESP_OK; + } + } + return ESP_ERR_NVS_NOT_FOUND; +} + esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { if (mState != StorageState::ACTIVE) { @@ -103,14 +116,13 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key if (err != ESP_OK) { return err; } - } - else if (err != ESP_OK) { + } else if (err != ESP_OK) { return err; } if (findPage) { if (findPage->state() == Page::PageState::UNINITIALIZED || - findPage->state() == Page::PageState::INVALID) { + findPage->state() == Page::PageState::INVALID) { auto err = findItem(nsIndex, datatype, key, findPage, item); assert(err == ESP_OK); } @@ -158,7 +170,7 @@ esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uin } mNamespaceUsage.set(ns, true); nsIndex = ns; - + NamespaceEntry* entry = new NamespaceEntry; entry->mIndex = ns; strncpy(entry->mName, nsName, sizeof(entry->mName) - 1); @@ -203,6 +215,27 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key return findPage->eraseItem(nsIndex, datatype, key); } +esp_err_t Storage::eraseNamespace(uint8_t nsIndex) +{ + if (mState != StorageState::ACTIVE) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + + for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + while (true) { + auto err = it->eraseItem(nsIndex, ItemType::ANY, nullptr); + if (err == ESP_ERR_NVS_NOT_FOUND) { + break; + } + else if (err != ESP_OK) { + return err; + } + } + } + return ESP_OK; + +} + esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize) { if (mState != StorageState::ACTIVE) { @@ -234,6 +267,7 @@ void Storage::debugCheck() for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { size_t itemIndex = 0; + size_t usedCount = 0; Item item; while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { std::stringstream keyrepr; @@ -246,7 +280,9 @@ void Storage::debugCheck() } keys.insert(std::make_pair(keystr, static_cast(p))); itemIndex += item.span; + usedCount += item.span; } + assert(usedCount == p->getUsedEntryCount()); } } #endif //ESP_PLATFORM diff --git a/components/nvs_flash/src/nvs_storage.hpp b/components/nvs_flash/src/nvs_storage.hpp index 3a05c3266..f8cee9f2a 100644 --- a/components/nvs_flash/src/nvs_storage.hpp +++ b/components/nvs_flash/src/nvs_storage.hpp @@ -69,13 +69,15 @@ public: return readItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value)); } - template esp_err_t eraseItem(uint8_t nsIndex, const char* key) { - return eraseItem(nsIndex, itemTypeOf(), key); + return eraseItem(nsIndex, ItemType::ANY, key); } + esp_err_t eraseNamespace(uint8_t nsIndex); + void debugDump(); + void debugCheck(); @@ -88,19 +90,7 @@ protected: void clearNamespaces(); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item) - { - size_t itemIndex = 0; - for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - auto err = it->findItem(nsIndex, datatype, key, itemIndex, item); - if (err == ESP_OK) { - page = it; - return ESP_OK; - } - } - return ESP_ERR_NVS_NOT_FOUND; - } - + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item); protected: size_t mPageCount; diff --git a/components/nvs_flash/src/nvs_types.cpp b/components/nvs_flash/src/nvs_types.cpp index 9884b276b..d44d8b29f 100644 --- a/components/nvs_flash/src/nvs_types.cpp +++ b/components/nvs_flash/src/nvs_types.cpp @@ -21,7 +21,7 @@ namespace nvs { -uint32_t Item::calculateCrc32() +uint32_t Item::calculateCrc32() const { uint32_t result = 0xffffffff; const uint8_t* p = reinterpret_cast(this); @@ -32,6 +32,16 @@ uint32_t Item::calculateCrc32() return result; } +uint32_t Item::calculateCrc32WithoutValue() const +{ + uint32_t result = 0xffffffff; + const uint8_t* p = reinterpret_cast(this); + result = crc32_le(result, p + offsetof(Item, nsIndex), + offsetof(Item, datatype) - offsetof(Item, nsIndex)); + result = crc32_le(result, p + offsetof(Item, key), sizeof(key)); + return result; +} + uint32_t Item::calculateCrc32(const uint8_t* data, size_t size) { uint32_t result = 0xffffffff; diff --git a/components/nvs_flash/src/nvs_types.hpp b/components/nvs_flash/src/nvs_types.hpp index cc1f86940..5306744b5 100644 --- a/components/nvs_flash/src/nvs_types.hpp +++ b/components/nvs_flash/src/nvs_types.hpp @@ -77,7 +77,25 @@ public: static const size_t MAX_KEY_LENGTH = sizeof(key) - 1; - uint32_t calculateCrc32(); + Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_) + : nsIndex(nsIndex), datatype(datatype), span(span), reserved(0xff) + { + std::fill_n(reinterpret_cast(key), sizeof(key) / 4, 0xffffffff); + std::fill_n(reinterpret_cast(data), sizeof(data) / 4, 0xffffffff); + if (key_) { + strncpy(key, key_, sizeof(key) - 1); + key[sizeof(key) - 1] = 0; + } else { + key[0] = 0; + } + } + + Item() + { + } + + uint32_t calculateCrc32() const; + uint32_t calculateCrc32WithoutValue() const; static uint32_t calculateCrc32(const uint8_t* data, size_t size); void getKey(char* dst, size_t dstSize) diff --git a/components/nvs_flash/test/Makefile b/components/nvs_flash/test/Makefile index d14dbaa3c..600621396 100644 --- a/components/nvs_flash/test/Makefile +++ b/components/nvs_flash/test/Makefile @@ -8,6 +8,7 @@ SOURCE_FILES = \ nvs_page.cpp \ nvs_pagemanager.cpp \ nvs_storage.cpp \ + nvs_item_hash_list.cpp \ ) \ spi_flash_emulation.cpp \ test_compressed_enum_table.cpp \ diff --git a/components/nvs_flash/test/sdkconfig.h b/components/nvs_flash/test/sdkconfig.h new file mode 100644 index 000000000..e69de29bb diff --git a/components/nvs_flash/test/test_nvs.cpp b/components/nvs_flash/test/test_nvs.cpp index 325cdcf45..540d977b4 100644 --- a/components/nvs_flash/test/test_nvs.cpp +++ b/components/nvs_flash/test/test_nvs.cpp @@ -386,6 +386,27 @@ TEST_CASE("can modify an item on a page which will be erased", "[nvs]") } +TEST_CASE("can erase items", "[nvs]") +{ + SpiFlashEmulator emu(3); + Storage storage; + CHECK(storage.init(0, 3) == ESP_OK); + for (size_t i = 0; i < Page::ENTRY_COUNT * 2 - 3; ++i) { + char name[Item::MAX_KEY_LENGTH + 1]; + snprintf(name, sizeof(name), "key%05d", static_cast(i)); + REQUIRE(storage.writeItem(3, name, static_cast(i)) == ESP_OK); + } + CHECK(storage.writeItem(1, "foo", 32) == ESP_OK); + CHECK(storage.writeItem(2, "foo", 64) == ESP_OK); + CHECK(storage.eraseItem(2, "foo") == ESP_OK); + int val; + CHECK(storage.readItem(1, "foo", val) == ESP_OK); + CHECK(val == 32); + CHECK(storage.eraseNamespace(3) == ESP_OK); + CHECK(storage.readItem(2, "foo", val) == ESP_ERR_NVS_NOT_FOUND); + CHECK(storage.readItem(3, "key00222", val) == ESP_ERR_NVS_NOT_FOUND); +} + #define TEST_ESP_ERR(rc, res) CHECK((rc) == (res)) #define TEST_ESP_OK(rc) CHECK((rc) == ESP_OK) @@ -443,18 +464,140 @@ TEST_CASE("wifi test", "[nvs]") SpiFlashEmulator emu(10); emu.randomize(10); - nvs_handle handle; + const uint32_t NVS_FLASH_SECTOR = 5; const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); TEST_ESP_OK(nvs_flash_init(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); - TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &handle)); + nvs_handle misc_handle; + TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &misc_handle)); + char log[33]; + size_t log_size = sizeof(log); + TEST_ESP_ERR(nvs_get_str(misc_handle, "log", log, &log_size), ESP_ERR_NVS_NOT_FOUND); + strcpy(log, "foobarbazfizzz"); + TEST_ESP_OK(nvs_set_str(misc_handle, "log", log)); + + nvs_handle net80211_handle; + TEST_ESP_OK(nvs_open("nvs.net80211", NVS_READWRITE, &net80211_handle)); uint8_t opmode = 2; - if (nvs_get_u8(handle, "wifi.opmode", &opmode) != ESP_OK) { - TEST_ESP_OK(nvs_set_u8(handle, "wifi.opmode", opmode)); - } + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.opmode", &opmode), ESP_ERR_NVS_NOT_FOUND); + + TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.opmode", opmode)); + + uint8_t country = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "wifi.country", &opmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "wifi.country", opmode)); + + char ssid[36]; + size_t size = sizeof(ssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(ssid, "my android AP"); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.ssid", ssid, size)); + + char mac[6]; + size = sizeof(mac); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xab, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.mac", mac, size)); + + uint8_t authmode = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.authmode", authmode)); + + char pswd[65]; + size = sizeof(pswd); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pswd", pswd, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(pswd, "`123456788990-="); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pswd", pswd, size)); + + char pmk[32]; + size = sizeof(pmk); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); + memset(pmk, 1, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.pmk", pmk, size)); + + uint8_t chan = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.chan", &chan), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.chan", chan)); + + uint8_t autoconn = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "auto.conn", &autoconn), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "auto.conn", autoconn)); + + uint8_t bssid_set = 1; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bssid.set", &bssid_set), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "bssid.set", bssid_set)); + + char bssid[6]; + size = sizeof(bssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.bssid", bssid, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xcd, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.bssid", bssid, size)); + + uint8_t phym = 3; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phym", &phym), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phym", phym)); + + uint8_t phybw = 2; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "sta.phybw", &phybw), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "sta.phybw", phybw)); + + char apsw[2]; + size = sizeof(apsw); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apsw", apsw, &size), ESP_ERR_NVS_NOT_FOUND); + memset(apsw, 0x2, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apsw", apsw, size)); + + char apinfo[700]; + size = sizeof(apinfo); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "sta.apinfo", apinfo, &size), ESP_ERR_NVS_NOT_FOUND); + memset(apinfo, 0, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "sta.apinfo", apinfo, size)); + + size = sizeof(ssid); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.ssid", ssid, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(ssid, "ESP_A2F340"); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.ssid", ssid, size)); + + size = sizeof(mac); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.mac", mac, &size), ESP_ERR_NVS_NOT_FOUND); + memset(mac, 0xac, 6); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.mac", mac, size)); + + size = sizeof(pswd); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.passwd", pswd, &size), ESP_ERR_NVS_NOT_FOUND); + strcpy(pswd, ""); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.passwd", pswd, size)); + + size = sizeof(pmk); + TEST_ESP_ERR(nvs_get_blob(net80211_handle, "ap.pmk", pmk, &size), ESP_ERR_NVS_NOT_FOUND); + memset(pmk, 1, size); + TEST_ESP_OK(nvs_set_blob(net80211_handle, "ap.pmk", pmk, size)); + + chan = 6; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.chan", &chan), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.chan", chan)); + + authmode = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.authmode", &authmode), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.authmode", authmode)); + + uint8_t hidden = 0; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.hidden", &hidden), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.hidden", hidden)); + + uint8_t max_conn = 4; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "ap.max.conn", &max_conn), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "ap.max.conn", max_conn)); + + uint8_t bcn_interval = 2; + TEST_ESP_ERR(nvs_get_u8(net80211_handle, "bcn_interval", &bcn_interval), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_set_u8(net80211_handle, "bcn_interval", bcn_interval)); + + s_perf << "Time to simulate nvs init with wifi libs: " << emu.getTotalTime() << " us (" << emu.getEraseOps() << "E " << emu.getWriteOps() << "W " << emu.getReadOps() << "R " << emu.getWriteBytes() << "Wb " << emu.getReadBytes() << "Rb)" << std::endl; + } diff --git a/components/spi_flash/Kconfig b/components/spi_flash/Kconfig new file mode 100644 index 000000000..c344a6b74 --- /dev/null +++ b/components/spi_flash/Kconfig @@ -0,0 +1,16 @@ +menu "SPI Flash driver" + +config SPI_FLASH_ENABLE_COUNTERS + bool "Enable operation counters" + default 0 + help + This option enables the following APIs: + spi_flash_reset_counters + spi_flash_dump_counters + spi_flash_get_counters + These APIs may be used to collect performance data for spi_flash APIs + and to help understand behaviour of libraries which use SPI flash. + +endmenu + + diff --git a/components/spi_flash/esp_spi_flash.c b/components/spi_flash/esp_spi_flash.c index 191dd758a..d702f3b81 100644 --- a/components/spi_flash/esp_spi_flash.c +++ b/components/spi_flash/esp_spi_flash.c @@ -14,6 +14,9 @@ #include #include +#include +#include + #include #include #include @@ -25,7 +28,7 @@ #include "esp_ipc.h" #include "esp_attr.h" #include "esp_spi_flash.h" - +#include "esp_log.h" /* Driver for SPI flash read/write/erase operations @@ -71,6 +74,19 @@ static bool s_flash_op_can_start = false; static bool s_flash_op_complete = false; #endif //CONFIG_FREERTOS_UNICORE +#if CONFIG_SPI_FLASH_ENABLE_COUNTERS +static const char* TAG = "spi_flash"; +static spi_flash_counters_t s_flash_stats; + +#define COUNTER_START() uint32_t ts_begin = xthal_get_ccount() +#define COUNTER_STOP(counter) do{ s_flash_stats.counter.count++; s_flash_stats.counter.time += (xthal_get_ccount() - ts_begin) / (XT_CLOCK_FREQ / 1000000); } while(0) +#define COUNTER_ADD_BYTES(counter, size) do { s_flash_stats.counter.bytes += size; } while (0) +#else +#define COUNTER_START() +#define COUNTER_STOP(counter) +#define COUNTER_ADD_BYTES(counter, size) + +#endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS #ifndef CONFIG_FREERTOS_UNICORE @@ -95,6 +111,10 @@ static void IRAM_ATTR spi_flash_op_block_func(void* arg) void spi_flash_init() { s_flash_op_mutex = xSemaphoreCreateMutex(); + +#if CONFIG_SPI_FLASH_ENABLE_COUNTERS + spi_flash_reset_counters(); +#endif } static void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu() @@ -160,7 +180,9 @@ static void IRAM_ATTR spi_flash_enable_interrupts_caches_and_other_cpu() void spi_flash_init() { - // No-op in single core mode +#if CONFIG_SPI_FLASH_ENABLE_COUNTERS + spi_flash_reset_counters(); +#endif } static void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu() @@ -193,6 +215,7 @@ SpiFlashOpResult IRAM_ATTR spi_flash_unlock() esp_err_t IRAM_ATTR spi_flash_erase_sector(uint16_t sec) { + COUNTER_START(); spi_flash_disable_interrupts_caches_and_other_cpu(); SpiFlashOpResult rc; rc = spi_flash_unlock(); @@ -200,27 +223,33 @@ esp_err_t IRAM_ATTR spi_flash_erase_sector(uint16_t sec) rc = SPIEraseSector(sec); } spi_flash_enable_interrupts_caches_and_other_cpu(); + COUNTER_STOP(erase); return spi_flash_translate_rc(rc); } esp_err_t IRAM_ATTR spi_flash_write(uint32_t dest_addr, const uint32_t *src, uint32_t size) { + COUNTER_START(); spi_flash_disable_interrupts_caches_and_other_cpu(); SpiFlashOpResult rc; rc = spi_flash_unlock(); if (rc == SPI_FLASH_RESULT_OK) { rc = SPIWrite(dest_addr, src, (int32_t) size); + COUNTER_ADD_BYTES(write, size); } spi_flash_enable_interrupts_caches_and_other_cpu(); + COUNTER_STOP(write); return spi_flash_translate_rc(rc); } esp_err_t IRAM_ATTR spi_flash_read(uint32_t src_addr, uint32_t *dest, uint32_t size) { + COUNTER_START(); spi_flash_disable_interrupts_caches_and_other_cpu(); - SpiFlashOpResult rc; - rc = SPIRead(src_addr, dest, (int32_t) size); + SpiFlashOpResult rc = SPIRead(src_addr, dest, (int32_t) size); + COUNTER_ADD_BYTES(read, size); spi_flash_enable_interrupts_caches_and_other_cpu(); + COUNTER_STOP(read); return spi_flash_translate_rc(rc); } @@ -270,3 +299,30 @@ static void IRAM_ATTR spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_sta SET_PERI_REG_BITS(DPORT_APP_CACHE_CTRL1_REG, cache_mask, saved_state, 0); } } + +#if CONFIG_SPI_FLASH_ENABLE_COUNTERS + +static inline void dump_counter(spi_flash_counter_t* counter, const char* name) +{ + ESP_LOGI(TAG, "%s count=%8d time=%8dms bytes=%8d\n", name, + counter->count, counter->time, counter->bytes); +} + +const spi_flash_counters_t* spi_flash_get_counters() +{ + return &s_flash_stats; +} + +void spi_flash_reset_counters() +{ + memset(&s_flash_stats, 0, sizeof(s_flash_stats)); +} + +void spi_flash_dump_counters() +{ + dump_counter(&s_flash_stats.read, "read "); + dump_counter(&s_flash_stats.write, "write"); + dump_counter(&s_flash_stats.erase, "erase"); +} + +#endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS diff --git a/components/spi_flash/include/esp_spi_flash.h b/components/spi_flash/include/esp_spi_flash.h index da6b03c0d..6d635880e 100644 --- a/components/spi_flash/include/esp_spi_flash.h +++ b/components/spi_flash/include/esp_spi_flash.h @@ -17,6 +17,7 @@ #include #include "esp_err.h" +#include "sdkconfig.h" #ifdef __cplusplus extern "C" { @@ -69,6 +70,43 @@ esp_err_t spi_flash_write(uint32_t des_addr, const uint32_t *src_addr, uint32_t esp_err_t spi_flash_read(uint32_t src_addr, uint32_t *des_addr, uint32_t size); +#if CONFIG_SPI_FLASH_ENABLE_COUNTERS + +/** + * Structure holding statistics for one type of operation + */ +typedef struct { + uint32_t count; // number of times operation was executed + uint32_t time; // total time taken, in microseconds + uint32_t bytes; // total number of bytes, for read and write operations +} spi_flash_counter_t; + +typedef struct { + spi_flash_counter_t read; + spi_flash_counter_t write; + spi_flash_counter_t erase; +} spi_flash_counters_t; + +/** + * @brief Reset SPI flash operation counters + */ +void spi_flash_reset_counters(); + +/** + * @brief Print SPI flash operation counters + */ +void spi_flash_dump_counters(); + +/** + * @brief Return current SPI flash operation counters + * + * @return pointer to the spi_flash_counters_t structure holding values + * of the operation counters + */ +const spi_flash_counters_t* spi_flash_get_counters(); + +#endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS + #ifdef __cplusplus } #endif