From 5a27a6354165bfc97811677930d25f1596845478 Mon Sep 17 00:00:00 2001 From: Sagar Bijwe Date: Mon, 2 Apr 2018 16:14:59 +0530 Subject: [PATCH] nvs-flash: Support for blobs larger than half of SPI Flash sector size This change removes the earlier limitation of 1984 bytes for storing data-blobs. Blobs larger than the sector size are split and stored on multiple sectors. For this purpose, two new datatypes (multi-page index and multi-page data) are added for entries stored in the sectors. The underlying read, write, erase and find operations are modified to support these large blobs. The change is transparent to users of the library and no special APIs need to be used to store these large blobs. --- components/nvs_flash/README.rst | 57 +- .../nvs_partition_gen.py | 173 +++++- .../part_old_blob_format.bin | Bin 0 -> 4096 bytes .../testdata/sample.bin | 2 +- components/nvs_flash/src/intrusive_list.h | 13 +- components/nvs_flash/src/nvs_page.cpp | 122 ++-- components/nvs_flash/src/nvs_page.hpp | 18 +- components/nvs_flash/src/nvs_pagemanager.cpp | 17 +- components/nvs_flash/src/nvs_pagemanager.hpp | 4 + components/nvs_flash/src/nvs_storage.cpp | 375 +++++++++++- components/nvs_flash/src/nvs_storage.hpp | 29 +- components/nvs_flash/src/nvs_types.cpp | 1 + components/nvs_flash/src/nvs_types.hpp | 33 +- components/nvs_flash/test/test_nvs.c | 6 +- .../test_nvs_host/spi_flash_emulation.h | 3 +- .../nvs_flash/test_nvs_host/test_nvs.cpp | 541 +++++++++++++++++- 16 files changed, 1224 insertions(+), 170 deletions(-) create mode 100644 components/nvs_flash/nvs_partition_generator/part_old_blob_format.bin diff --git a/components/nvs_flash/README.rst b/components/nvs_flash/README.rst index b58a0b2a4..e69d12b71 100644 --- a/components/nvs_flash/README.rst +++ b/components/nvs_flash/README.rst @@ -27,7 +27,7 @@ NVS operates on key-value pairs. Keys are ASCII strings, maximum key length is c - variable length binary data (blob) .. note:: - String and blob values are currently limited to 1984 bytes. For strings, this includes the null terminator. + String values are currently limited to 4000 bytes. This includes the null terminator. Blob values are limited to 508000 bytes or (97.6% of the partition size - 4000) bytes whichever is lower. Additional types, such as ``float`` and ``double`` may be added later. @@ -148,21 +148,25 @@ Erased (2'b00) Structure of entry ^^^^^^^^^^^^^^^^^^ -For values of primitive types (currently integers from 1 to 8 bytes long), entry holds one key-value pair. For string and blob types, entry holds part of the whole key-value pair. In case when a key-value pair spans multiple entries, all entries are stored in the same page. +For values of primitive types (currently integers from 1 to 8 bytes long), entry holds one key-value pair. For string and blob types, entry holds part of the whole key-value pair. For strings, in case when a key-value pair spans multiple entries, all entries are stored in the same page. Blobs are allowed to span over multiple pages by dividing them into smaller chunks. For the purpose tracking these chunks, an additional fixed length metadata entry is stored called "blob index" entry. Earlier format of blobs are still supported (can be read and modified). However, once the blobs are modified, they are stored using the new format. :: - +--------+----------+----------+---------+-----------+---------------+----------+ - | NS (1) | Type (1) | Span (1) | Rsv (1) | CRC32 (4) | Key (16) | Data (8) | - +--------+----------+----------+---------+-----------+---------------+----------+ + +--------+----------+----------+----------------+-----------+---------------+----------+ + | NS (1) | Type (1) | Span (1) | ChunkIndex (1) | CRC32 (4) | Key (16) | Data (8) | + +--------+----------+----------+----------------+-----------+---------------+----------+ - +--------------------------------+ - +-> Fixed length: | Data (8) | - | +--------------------------------+ - Data format ---+ - | +----------+---------+-----------+ - +-> Variable length: | Size (2) | Rsv (2) | CRC32 (4) | - +----------+---------+-----------+ + Primitive +--------------------------------+ + +--------> | Data (8) | + | Types +--------------------------------+ + +-> Fixed length -- + | | +---------+--------------+---------------+-------+ + | +--------> | Size(4) | ChunkCount(1)| ChunkStart(1) | Rsv(2)| + Data format ---+ Blob Index +---------+--------------+---------------+-------+ + | + | +----------+---------+-----------+ + +-> Variable length --> | Size (2) | Rsv (2) | CRC32 (4) | + (Strings, Blob Data) +----------+---------+-----------+ Individual fields in entry structure have the following meanings: @@ -176,8 +180,8 @@ Type Span Number of entries used by this key-value pair. For integer types, this is equal to 1. For strings and blobs this depends on value length. -Rsv - Unused field, should be ``0xff``. +ChunkIndex + Used to store index of the blob-data chunk for blob types. For other types, this should be ``0xff``. CRC32 Checksum calculated over all the bytes in this entry, except for the CRC32 field itself. @@ -186,13 +190,26 @@ Key Zero-terminated ASCII string containing key name. Maximum string length is 15 bytes, excluding zero terminator. Data - For integer types, this field contains the value itself. If the value itself is shorter than 8 bytes it is padded to the right, with unused bytes filled with ``0xff``. For string and blob values, these 8 bytes hold additional data about the value, described next: + For integer types, this field contains the value itself. If the value itself is shorter than 8 bytes it is padded to the right, with unused bytes filled with ``0xff``. -Size - (Only for strings and blobs.) Size, in bytes, of actual data. For strings, this includes zero terminator. + For "blob index" entry, these 8 bytes hold the following information about data-chunks: -CRC32 - (Only for strings and blobs.) Checksum calculated over all bytes of data. + - Size + (Only for blob index.) Size, in bytes, of complete blob data. + + - ChunkCount + (Only for blob index.) Total number of blob-data chunks into which the blob was divided during storage. + + - ChunkStart + (Only for blob index.) ChunkIndex of the first blob-data chunk of this blob. Subsequent chunks have chunkIndex incrementely allocated (step of 1). + + For string and blob data chunks, these 8 bytes hold additional data about the value, described next: + + - Size + (Only for strings and blobs.) Size, in bytes, of actual data. For strings, this includes zero terminator. + + - CRC32 + (Only for strings and blobs.) Checksum calculated over all bytes of data. Variable length values (strings and blobs) are written into subsequent entries, 32 bytes per entry. `Span` field of the first entry indicates how many entries are used. @@ -220,5 +237,5 @@ Item hash list To reduce the number of reads performed from flash memory, each member of Page class maintains a list of pairs: (item index; item hash). This list makes searches much quicker. Instead of iterating over all entries, reading them from flash one at a time, ``Page::findItem`` first performs search for item hash in the hash list. This gives the item index within the page, if such an item exists. Due to a hash collision it is possible that a different item will be found. This is handled by falling back to iteration over items in flash. -Each node in hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace and key name. CRC32 is used for calculation, result is truncated to 24 bits. To reduce overhead of storing 32-bit entries in a linked list, list is implemented as a doubly-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and 32-bit count field. Minimal amount of extra RAM useage per page is therefore 128 bytes, maximum is 640 bytes. +Each node in hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace, key name and ChunkIndex. CRC32 is used for calculation, result is truncated to 24 bits. To reduce overhead of storing 32-bit entries in a linked list, list is implemented as a doubly-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and 32-bit count field. Minimal amount of extra RAM useage per page is therefore 128 bytes, maximum is 640 bytes. diff --git a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py index 6224d3b94..6a51e54d9 100755 --- a/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py +++ b/components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py @@ -32,7 +32,7 @@ from os import path class Page(object): PAGE_PARAMS = { "max_size": 4096, - "max_blob_size": 1984, + "max_blob_size": 4000, "max_entries": 126 } @@ -45,6 +45,8 @@ class Page(object): I32 = 0x14 SZ = 0x21 BLOB = 0x41 + BLOB_DATA = 0x42 + BLOB_IDX = 0x48 # Few Page constants HEADER_SIZE = 32 @@ -52,6 +54,9 @@ class Page(object): BITMAPARRAY_SIZE_IN_BYTES = 32 FIRST_ENTRY_OFFSET = 64 SINGLE_ENTRY_SIZE = 32 + CHUNK_ANY = 0xFF + ACTIVE = 0xFFFFFFFE + FULL = 0xFFFFFFFC def __init__(self, page_num): self.entry_num = 0 @@ -61,9 +66,11 @@ class Page(object): self.set_header(page_num) def set_header(self, page_num): + global page_header + # set page state to active page_header= bytearray(b'\xff')*32 - page_state_active_seq = 0xFFFFFFFE + page_state_active_seq = Page.ACTIVE page_header[0:4] = struct.pack(' Page.PAGE_PARAMS["max_blob_size"]: - raise InputError("%s: Size exceeds max allowed length." % key) + + if encoding == "string": + if datalen > Page.PAGE_PARAMS["max_blob_size"]: + raise InputError("%s: Size exceeds max allowed length." % key) # Calculate no. of entries data will require rounded_size = (datalen + 31) & ~31 @@ -116,13 +227,20 @@ class Page(object): total_entry_count = data_entry_count + 1 # +1 for the entry header # Check if page is already full and new page is needed to be created right away - if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]: - raise PageFullError() + if encoding == "string": + if (self.entry_num + total_entry_count) >= Page.PAGE_PARAMS["max_entries"]: + raise PageFullError() # Entry header entry_struct = bytearray('\xff')*32 - entry_struct[0] = ns_index # namespace index - entry_struct[2] = data_entry_count + 1 # Span + # Set Namespace Index + entry_struct[0] = ns_index + # Set Span + if encoding == "string": + entry_struct[2] = data_entry_count + 1 + # Set Chunk Index + chunk_index = Page.CHUNK_ANY + entry_struct[3] = chunk_index # set key key_array = bytearray('\x00')*16 @@ -135,22 +253,23 @@ class Page(object): elif encoding in ["hex2bin", "binary", "base64"]: entry_struct[1] = Page.BLOB - # compute CRC of data - entry_struct[24:26] = struct.pack('qeJF7qdLk1zhz{vRjq}>wsl+xVXO25S1)Z&7~;GvRt9cBab&>g$n@W8T_G=6zej3?kyBB?i#Z1UZ zt#HfCNriczfdi=D;-l0iMrGIZPCPCE>3_JYc~279YJ?FyK>ahgbFzT$XJKVy=itO) z1}OX_?7y%TgM$vCUmB=?!7Aw{pyN|g)6z3Cv$At?^YRM{i;7E1%gQS%tAGlC0A&BO z)@cWlGV>6cfssS3{m5Z4${r1Y(GVC7fzc2c4S~@R7!85Z5Eu=C(GVC7fzc4a9RdLQ C6Sl+v literal 0 HcmV?d00001 diff --git a/components/nvs_flash/nvs_partition_generator/testdata/sample.bin b/components/nvs_flash/nvs_partition_generator/testdata/sample.bin index 5e2522d2e..fe69eca4f 100644 --- a/components/nvs_flash/nvs_partition_generator/testdata/sample.bin +++ b/components/nvs_flash/nvs_partition_generator/testdata/sample.bin @@ -1 +1 @@ - «Νο \ No newline at end of file +start0000000000000000000000start0123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef00000000000000000123456789abcdef0000000000000000end00000000000000000000000000end \ No newline at end of file diff --git a/components/nvs_flash/src/intrusive_list.h b/components/nvs_flash/src/intrusive_list.h index fc92442cd..6bae6a657 100644 --- a/components/nvs_flash/src/intrusive_list.h +++ b/components/nvs_flash/src/intrusive_list.h @@ -229,8 +229,7 @@ public: { return mSize == 0; } - - + void clear() { while (mFirst) { @@ -238,6 +237,16 @@ public: } } + void clearAndFreeNodes() + { + while (mFirst) { + auto tmp = mFirst; + erase(mFirst); + delete tmp; + } + } + + protected: T* mFirst = nullptr; T* mLast = nullptr; diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index 9fc1a6eda..fb9cf3d40 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -106,16 +106,16 @@ esp_err_t Page::writeEntry(const Item& item) return ESP_OK; } - + esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) { assert(size % ENTRY_SIZE == 0); assert(mNextFreeEntry != INVALID_ENTRY); assert(mFirstUsedEntry != INVALID_ENTRY); const uint16_t count = size / ENTRY_SIZE; - + const uint8_t* buf = data; - + #ifdef ESP_PLATFORM /* On the ESP32, data can come from DROM, which is not accessible by spi_flash_write * function. To work around this, we copy the data to heap if it came from DROM. @@ -151,15 +151,15 @@ esp_err_t Page::writeEntryData(const uint8_t* data, size_t size) return ESP_OK; } -esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) +esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx) { Item item; esp_err_t err; - + if (mState == PageState::INVALID) { return ESP_ERR_NVS_INVALID_STATE; } - + if (mState == PageState::UNINITIALIZED) { err = initialize(); if (err != ESP_OK) { @@ -175,21 +175,22 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c if (keySize > Item::MAX_KEY_LENGTH) { return ESP_ERR_NVS_KEY_TOO_LONG; } - - if (dataSize > Page::BLOB_MAX_SIZE) { + + if (dataSize > Page::CHUNK_MAX_SIZE) { return ESP_ERR_NVS_VALUE_TOO_LONG; } size_t totalSize = ENTRY_SIZE; size_t entriesCount = 1; - if (datatype == ItemType::SZ || datatype == ItemType::BLOB) { + if (isVariableLengthType(datatype)) { size_t roundedSize = (dataSize + ENTRY_SIZE - 1) & ~(ENTRY_SIZE - 1); totalSize += roundedSize; entriesCount += roundedSize / ENTRY_SIZE; } // primitive types should fit into one entry - assert(totalSize == ENTRY_SIZE || datatype == ItemType::BLOB || datatype == ItemType::SZ); + assert(totalSize == ENTRY_SIZE || + isVariableLengthType(datatype)); if (mNextFreeEntry == INVALID_ENTRY || mNextFreeEntry + entriesCount > ENTRY_COUNT) { // page will not fit this amount of data @@ -198,10 +199,10 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c // write first item size_t span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE; - item = Item(nsIndex, datatype, span, key); + item = Item(nsIndex, datatype, span, key, chunkIdx); mHashList.insert(item, mNextFreeEntry); - if (datatype != ItemType::SZ && datatype != ItemType::BLOB) { + if (!isVariableLengthType(datatype)) { memcpy(item.data, data, dataSize); item.crc32 = item.calculateCrc32(); err = writeEntry(item); @@ -212,7 +213,7 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c const uint8_t* src = reinterpret_cast(data); item.varLength.dataCrc32 = Item::calculateCrc32(src, dataSize); item.varLength.dataSize = dataSize; - item.varLength.reserved2 = 0xffff; + item.varLength.reserved = 0xffff; item.crc32 = item.calculateCrc32(); err = writeEntry(item); if (err != ESP_OK) { @@ -226,7 +227,7 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return err; } } - + size_t tail = dataSize - left; if (tail > 0) { std::fill_n(item.rawData, ENTRY_SIZE / 4, 0xffffffff); @@ -236,26 +237,26 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return err; } } - + } return ESP_OK; } -esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) +esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - + if (mState == PageState::INVALID) { return ESP_ERR_NVS_INVALID_STATE; } - - esp_err_t rc = findItem(nsIndex, datatype, key, index, item); + + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); if (rc != ESP_OK) { return rc; } - if (datatype != ItemType::SZ && datatype != ItemType::BLOB) { + if (!isVariableLengthType(datatype)) { if (dataSize != getAlignmentForType(datatype)) { return ESP_ERR_NVS_TYPE_MISMATCH; } @@ -292,22 +293,22 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo return ESP_OK; } -esp_err_t Page::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) +esp_err_t Page::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - esp_err_t rc = findItem(nsIndex, datatype, key, index, item); + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); if (rc != ESP_OK) { return rc; } return eraseEntryAndSpan(index); } -esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key) +esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; Item item; - return findItem(nsIndex, datatype, key, index, item); + return findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); } esp_err_t Page::eraseEntryAndSpan(size_t index) @@ -523,7 +524,7 @@ esp_err_t Page::mLoadEntryTable() mState = PageState::INVALID; return err; } - + if (item.crc32 != item.calculateCrc32()) { err = eraseEntryAndSpan(i); if (err != ESP_OK) { @@ -534,11 +535,11 @@ esp_err_t Page::mLoadEntryTable() } mHashList.insert(item, i); - + // search for potential duplicate item size_t duplicateIndex = mHashList.find(0, item); - - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + + if (isVariableLengthType(item.datatype)) { span = item.span; bool needErase = false; for (size_t j = i; j < i + span; ++j) { @@ -553,7 +554,11 @@ esp_err_t Page::mLoadEntryTable() continue; } } - + + /* Note that logic for duplicate detections works fine even + * when old-format blob is present along with new-format blob-index + * for same key on active page. Since datatype is not used in hash calculation, + * old-format blob will be removed.*/ if (duplicateIndex < i) { eraseEntryAndSpan(duplicateIndex); } @@ -603,7 +608,7 @@ esp_err_t Page::mLoadEntryTable() size_t span = item.span; - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + if (isVariableLengthType(item.datatype)) { for (size_t j = i + 1; j < i + span; ++j) { if (mEntryTable.get(j) != EntryState::WRITTEN) { eraseEntryAndSpan(i); @@ -703,12 +708,12 @@ esp_err_t Page::readEntry(size_t index, Item& dst) const return ESP_OK; } -esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item) +esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item, uint8_t chunkIdx, VerOffset chunkStart) { if (mState == PageState::CORRUPT || mState == PageState::INVALID || mState == PageState::UNINITIALIZED) { return ESP_ERR_NVS_NOT_FOUND; } - + size_t findBeginIndex = itemIndex; if (findBeginIndex >= ENTRY_COUNT) { return ESP_ERR_NVS_NOT_FOUND; @@ -725,7 +730,7 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si } if (nsIndex != NS_ANY && datatype != ItemType::ANY && key != NULL) { - size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key)); + size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key, chunkIdx)); if (cachedIndex < ENTRY_COUNT) { start = cachedIndex; } else { @@ -756,7 +761,7 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si continue; } - if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) { + if (isVariableLengthType(item.datatype)) { next = i + item.span; } @@ -767,8 +772,30 @@ esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, si if (key != nullptr && strncmp(key, item.key, Item::MAX_KEY_LENGTH) != 0) { continue; } + /* For blob data, chunkIndex should match*/ + if (chunkIdx != CHUNK_ANY + && datatype == ItemType::BLOB_DATA + && item.chunkIndex != chunkIdx) { + continue; + } + /* Blob-index will match the with blob data. + * Skip data chunks when searching for blob index*/ + if (datatype == ItemType::BLOB_IDX + && item.chunkIndex != CHUNK_ANY) { + continue; + } + /* Match the version for blob-index*/ + if (datatype == ItemType::BLOB_IDX + && chunkStart != VerOffset::VER_ANY + && item.blobIndex.chunkStart != chunkStart) { + continue; + } + if (datatype != ItemType::ANY && item.datatype != datatype) { + if (key == nullptr && nsIndex == NS_ANY && chunkIdx == CHUNK_ANY) { + continue; // continue for bruteforce search on blob indices. + } return ESP_ERR_NVS_TYPE_MISMATCH; } @@ -831,28 +858,39 @@ esp_err_t Page::markFull() } return alterPageState(PageState::FULL); } - + +size_t Page::getVarDataTailroom() const +{ + if (mState == PageState::UNINITIALIZED) { + return CHUNK_MAX_SIZE; + } else if (mState == PageState::FULL) { + return 0; + } + /* Skip one entry for header*/ + return ((mNextFreeEntry < (ENTRY_COUNT-1)) ? ((ENTRY_COUNT - mNextFreeEntry - 1) * ENTRY_SIZE): 0); +} + const char* Page::pageStateToName(PageState ps) { switch (ps) { case PageState::CORRUPT: return "CORRUPT"; - + case PageState::ACTIVE: return "ACTIVE"; - + case PageState::FREEING: return "FREEING"; - + case PageState::FULL: return "FULL"; - + case PageState::INVALID: return "INVALID"; - + case PageState::UNINITIALIZED: return "UNINITIALIZED"; - + default: assert(0 && "invalid state value"); return ""; @@ -874,7 +912,7 @@ void Page::debugDump() const Item item; readEntry(i, item); if (skip == 0) { - printf("W ns=%2u type=%2u span=%3u key=\"%s\" len=%d\n", item.nsIndex, static_cast(item.datatype), item.span, item.key, (item.span != 1)?((int)item.varLength.dataSize):-1); + printf("W ns=%2u type=%2u span=%3u key=\"%s\" chunkIdx=%d len=%d\n", item.nsIndex, static_cast(item.datatype), item.span, item.key, item.chunkIndex, (item.span != 1)?((int)item.varLength.dataSize):-1); if (item.span > 0 && item.span <= ENTRY_COUNT - i) { skip = item.span - 1; } else { diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 9baa76dbf..ae803dede 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -45,12 +45,14 @@ public: static const size_t ENTRY_SIZE = 32; static const size_t ENTRY_COUNT = 126; static const uint32_t INVALID_ENTRY = 0xffffffff; - - static const size_t BLOB_MAX_SIZE = ENTRY_SIZE * (ENTRY_COUNT / 2 - 1); + + static const size_t CHUNK_MAX_SIZE = ENTRY_SIZE * (ENTRY_COUNT - 1); static const uint8_t NS_INDEX = 0; static const uint8_t NS_ANY = 255; + static const uint8_t CHUNK_ANY = Item::CHUNK_ANY; + enum class PageState : uint32_t { // All bits set, default state after flash erase. Page has not been initialized yet. UNINITIALIZED = 0xffffffff, @@ -84,15 +86,15 @@ public: esp_err_t setSeqNumber(uint32_t seqNumber); - esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize); + esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY); - esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize); + esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); - esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key); + esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key); + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item); + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); template esp_err_t writeItem(uint8_t nsIndex, const char* key, const T& value) @@ -121,7 +123,7 @@ public: { return mErasedEntryCount; } - + size_t getVarDataTailroom() const ; esp_err_t markFull(); diff --git a/components/nvs_flash/src/nvs_pagemanager.cpp b/components/nvs_flash/src/nvs_pagemanager.cpp index 3f3c5f01c..58d9e47f4 100644 --- a/components/nvs_flash/src/nvs_pagemanager.cpp +++ b/components/nvs_flash/src/nvs_pagemanager.cpp @@ -66,13 +66,26 @@ esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount) if (lastItemIndex != SIZE_MAX) { auto last = PageManager::TPageListIterator(&lastPage); - for (auto it = begin(); it != last; ++it) { + TPageListIterator it; + + for (it = begin(); it != last; ++it) { if ((it->state() != Page::PageState::FREEING) && - (it->eraseItem(item.nsIndex, item.datatype, item.key) == ESP_OK)) { + (it->eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex) == ESP_OK)) { break; } } + if ((it == last) && (item.datatype == ItemType::BLOB_IDX)) { + /* Rare case in which the blob was stored using old format, but power went just after writing + * blob index during modification. Loop again and delete the old version blob*/ + for (it = begin(); it != last; ++it) { + + if ((it->state() != Page::PageState::FREEING) && + (it->eraseItem(item.nsIndex, ItemType::BLOB, item.key, item.chunkIndex) == ESP_OK)) { + break; + } + } + } } // check if power went out while page was being freed diff --git a/components/nvs_flash/src/nvs_pagemanager.hpp b/components/nvs_flash/src/nvs_pagemanager.hpp index 74305e952..1a7ffbaab 100644 --- a/components/nvs_flash/src/nvs_pagemanager.hpp +++ b/components/nvs_flash/src/nvs_pagemanager.hpp @@ -48,6 +48,10 @@ public: return mPageList.back(); } + uint32_t getPageCount() { + return mPageCount; + } + esp_err_t requestNewPage(); esp_err_t fillStats(nvs_stats_t& nvsStats); diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index eaf48cc19..611973ce0 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -28,11 +28,59 @@ Storage::~Storage() void Storage::clearNamespaces() { - for (auto it = std::begin(mNamespaces); it != std::end(mNamespaces); ) { - auto tmp = it; - ++it; - mNamespaces.erase(tmp); - delete static_cast(tmp); + mNamespaces.clearAndFreeNodes(); +} + +void Storage::populateBlobIndices(TBlobIndexList& blobIdxList) +{ + for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + Page& p = *it; + size_t itemIndex = 0; + Item item; + + /* If the power went off just after writing a blob index, the duplicate detection + * logic in pagemanager will remove the earlier index. So we should never find a + * duplicate index at this point */ + + while (p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) { + BlobIndexNode* entry = new BlobIndexNode; + + item.getKey(entry->key, sizeof(entry->key) - 1); + entry->nsIndex = item.nsIndex; + entry->chunkStart = item.blobIndex.chunkStart; + entry->chunkCount = item.blobIndex.chunkCount; + + blobIdxList.push_back(entry); + itemIndex += item.span; + } + } +} + +void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) +{ + for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + Page& p = *it; + size_t itemIndex = 0; + Item item; + /* Chunks with same and with chunkIndex in the following ranges + * belong to same family. + * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks + * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks + */ + while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { + + auto iter = std::find_if(blobIdxList.begin(), + blobIdxList.end(), + [=] (const BlobIndexNode& e) -> bool + {return (strncmp(item.key, e.key, sizeof(e.key) - 1) == 0) + && (item.nsIndex == e.nsIndex) + && (item.chunkIndex >= static_cast (e.chunkStart)) + && (item.chunkIndex < static_cast (e.chunkStart) + e.chunkCount);}); + if (iter == std::end(blobIdxList)) { + p.eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex); + } + itemIndex += item.span; + } } } @@ -63,6 +111,17 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) mNamespaceUsage.set(0, true); mNamespaceUsage.set(255, true); mState = StorageState::ACTIVE; + + // Populate list of multi-page index entries. + TBlobIndexList blobIdxList; + populateBlobIndices(blobIdxList); + + // Remove the entries for which there is no parent multi-page index. + eraseOrphanDataBlobs(blobIdxList); + + // Purge the blob index list + blobIdxList.clearAndFreeNodes(); + #ifndef ESP_PLATFORM debugCheck(); #endif @@ -74,11 +133,11 @@ bool Storage::isValid() const return mState == StorageState::ACTIVE; } -esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item) +esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart) { for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { size_t itemIndex = 0; - auto err = it->findItem(nsIndex, datatype, key, itemIndex, item); + auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart); if (err == ESP_OK) { page = it; return ESP_OK; @@ -87,6 +146,102 @@ esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, return ESP_ERR_NVS_NOT_FOUND; } +esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize, VerOffset chunkStart) +{ + uint8_t chunkCount = 0; + TUsedPageList usedPages; + size_t remainingSize = dataSize; + size_t offset=0; + esp_err_t err = ESP_OK; + + /* Check how much maximum data can be accommodated**/ + uint32_t max_pages = mPageManager.getPageCount() - 1; + + if(max_pages > (Page::CHUNK_ANY-1)/2) { + max_pages = (Page::CHUNK_ANY-1)/2; + } + + if (dataSize > max_pages * Page::CHUNK_MAX_SIZE) { + return ESP_ERR_NVS_VALUE_TOO_LONG; + } + + do { + Page& page = getCurrentPage(); + size_t tailroom = page.getVarDataTailroom(); + size_t chunkSize =0; + if (!chunkCount && tailroom < dataSize && tailroom < Page::CHUNK_MAX_SIZE/10) { + /** This is the first chunk and tailroom is too small ***/ + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + return err; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + return err; + } else { + continue; + } + } else if (!tailroom) { + err = ESP_ERR_NVS_NOT_ENOUGH_SPACE; + break; + } + + /* Split the blob into two and store the chunk of available size onto the current page */ + assert(tailroom!=0); + chunkSize = (remainingSize > tailroom)? tailroom : remainingSize; + remainingSize -= chunkSize; + + err = page.writeItem(nsIndex, ItemType::BLOB_DATA, key, + static_cast (data) + offset, chunkSize, static_cast (chunkStart) + chunkCount); + chunkCount++; + assert(err != ESP_ERR_NVS_PAGE_FULL); + if (err != ESP_OK) { + break; + } else { + UsedPageNode* node = new UsedPageNode(); + node->mPage = &page; + usedPages.push_back(node); + if (remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) { + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + break; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + break; + } + } + } + offset += chunkSize; + if (!remainingSize) { + /* All pages are stored. Now store the index.*/ + Item item; + item.blobIndex.dataSize = dataSize; + item.blobIndex.chunkCount = chunkCount; + item.blobIndex.chunkStart = chunkStart; + + err = getCurrentPage().writeItem(nsIndex, ItemType::BLOB_IDX, key, item.data, sizeof(item.data)); + assert(err != ESP_ERR_NVS_PAGE_FULL); + break; + } + } while (1); + + if (err != ESP_OK) { + /* Anything failed, then we should erase all the written chunks*/ + int ii=0; + for (auto it = std::begin(usedPages); it != std::end(usedPages); it++) { + it->mPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, ii++); + } + usedPages.clearAndFreeNodes(); + return err; + } + return ESP_OK; +} + esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { if (mState != StorageState::ACTIVE) { @@ -95,40 +250,95 @@ esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key Page* findPage = nullptr; Item item; - auto err = findItem(nsIndex, datatype, key, findPage, item); + + esp_err_t err; + if (datatype == ItemType::BLOB) { + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + } else { + err = findItem(nsIndex, datatype, key, findPage, item); + } + + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { return err; } - Page& page = getCurrentPage(); - err = page.writeItem(nsIndex, datatype, key, data, dataSize); - if (err == ESP_ERR_NVS_PAGE_FULL) { - if (page.state() != Page::PageState::FULL) { - err = page.markFull(); - if (err != ESP_OK) { - return err; + if (datatype == ItemType::BLOB) { + VerOffset prevStart, nextStart; + prevStart = nextStart = VerOffset::VER_0_OFFSET; + if (findPage) { + if (findPage->state() == Page::PageState::UNINITIALIZED || + findPage->state() == Page::PageState::INVALID) { + ESP_ERROR_CHECK(findItem(nsIndex, datatype, key, findPage, item)); } - } - err = mPageManager.requestNewPage(); - if (err != ESP_OK) { - return err; - } + /* Get the version of the previous index with same */ + prevStart = item.blobIndex.chunkStart; + assert(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET); - err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); + /* Toggle the version by changing the offset */ + nextStart + = (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET; + } + /* Write the blob with new version*/ + err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart); if (err == ESP_ERR_NVS_PAGE_FULL) { return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } if (err != ESP_OK) { return err; } - } else if (err != ESP_OK) { - return err; + + if (findPage) { + /* Erase the blob with earlier version*/ + err = eraseMultiPageBlob(nsIndex, key, prevStart); + + if (err == ESP_ERR_FLASH_OP_FAIL) { + return ESP_ERR_NVS_REMOVE_FAILED; + } + if (err != ESP_OK) { + return err; + } + + findPage = nullptr; + } else { + /* Support for earlier versions where BLOBS were stored without index */ + err = findItem(nsIndex, datatype, key, findPage, item); + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } + } + } else { + + Page& page = getCurrentPage(); + err = page.writeItem(nsIndex, datatype, key, data, dataSize); + if (err == ESP_ERR_NVS_PAGE_FULL) { + if (page.state() != Page::PageState::FULL) { + err = page.markFull(); + if (err != ESP_OK) { + return err; + } + } + err = mPageManager.requestNewPage(); + if (err != ESP_OK) { + return err; + } + + err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); + if (err == ESP_ERR_NVS_PAGE_FULL) { + return ESP_ERR_NVS_NOT_ENOUGH_SPACE; + } + if (err != ESP_OK) { + return err; + } + } else if (err != ESP_OK) { + return err; + } } if (findPage) { if (findPage->state() == Page::PageState::UNINITIALIZED || findPage->state() == Page::PageState::INVALID) { - ESP_ERROR_CHECK( findItem(nsIndex, datatype, key, findPage, item) ); + ESP_ERROR_CHECK(findItem(nsIndex, datatype, key, findPage, item)); } err = findPage->eraseItem(nsIndex, datatype, key); if (err == ESP_ERR_FLASH_OP_FAIL) { @@ -187,6 +397,49 @@ esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uin return ESP_OK; } +esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* data, size_t dataSize) +{ + Item item; + Page* findPage = nullptr; + + /* First read the blob index */ + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + if (err != ESP_OK) { + return err; + } + + uint8_t chunkCount = item.blobIndex.chunkCount; + VerOffset chunkStart = item.blobIndex.chunkStart; + size_t readSize = item.blobIndex.dataSize; + size_t offset = 0; + + assert(dataSize == readSize); + + /* Now read corresponding chunks */ + for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + if (err == ESP_ERR_NVS_NOT_FOUND) { + break; + } + return err; + } + err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + return err; + } + assert(static_cast (chunkStart) + chunkNum == item.chunkIndex); + offset += item.varLength.dataSize; + } + if (err == ESP_OK) { + assert(offset == dataSize); + } + if (err == ESP_ERR_NVS_NOT_FOUND) { + eraseMultiPageBlob(nsIndex, key); // cleanup if a chunk is not found + } + return err; +} + esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) { if (mState != StorageState::ACTIVE) { @@ -195,12 +448,64 @@ esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, Item item; Page* findPage = nullptr; + if (datatype == ItemType::BLOB) { + auto err = readMultiPageBlob(nsIndex, key, data, dataSize); + if (err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } // else check if the blob is stored with earlier version format without index + } + auto err = findItem(nsIndex, datatype, key, findPage, item); if (err != ESP_OK) { return err; } - return findPage->readItem(nsIndex, datatype, key, data, dataSize); + +} + +esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart) +{ + if (mState != StorageState::ACTIVE) { + return ESP_ERR_NVS_NOT_INITIALIZED; + } + Item item; + Page* findPage = nullptr; + + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); + if (err != ESP_OK) { + return err; + } + /* Erase the index first and make children blobs orphan*/ + err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); + if (err != ESP_OK) { + return err; + } + + uint8_t chunkCount = item.blobIndex.chunkCount; + + if (chunkStart == VerOffset::VER_ANY) { + chunkStart = item.blobIndex.chunkStart; + } else { + assert(chunkStart == item.blobIndex.chunkStart); + } + + /* Now erase corresponding chunks*/ + for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); + + if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { + return err; + } else if (err == ESP_ERR_NVS_NOT_FOUND) { + continue; // Keep erasing other chunks + } + err = findPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, static_cast (chunkStart) + chunkNum); + if (err != ESP_OK) { + return err; + } + + } + + return ESP_OK; } esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) @@ -209,6 +514,10 @@ esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key return ESP_ERR_NVS_NOT_INITIALIZED; } + if (datatype == ItemType::BLOB) { + return eraseMultiPageBlob(nsIndex, key); + } + Item item; Page* findPage = nullptr; auto err = findItem(nsIndex, datatype, key, findPage, item); @@ -250,7 +559,15 @@ esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const cha Page* findPage = nullptr; auto err = findItem(nsIndex, datatype, key, findPage, item); if (err != ESP_OK) { - return err; + if (datatype != ItemType::BLOB) { + return err; + } + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); + if (err != ESP_OK) { + return err; + } + dataSize = item.blobIndex.dataSize; + return ESP_OK; } dataSize = item.varLength.dataSize; @@ -268,14 +585,14 @@ void Storage::debugDump() void Storage::debugCheck() { std::map keys; - + for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { size_t itemIndex = 0; size_t usedCount = 0; Item item; while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { std::stringstream keyrepr; - keyrepr << static_cast(item.nsIndex) << "_" << static_cast(item.datatype) << "_" << item.key; + keyrepr << static_cast(item.nsIndex) << "_" << static_cast(item.datatype) << "_" << item.key <<"_"<(item.chunkIndex); std::string keystr = keyrepr.str(); if (keys.find(keystr) != std::end(keys)) { printf("Duplicate key: %s\n", keystr.c_str()); @@ -318,7 +635,7 @@ esp_err_t Storage::calcEntriesInNamespace(uint8_t nsIndex, size_t& usedEntries) } usedEntries += item.span; itemIndex += item.span; - if(itemIndex >= it->ENTRY_COUNT) break; + if (itemIndex >= it->ENTRY_COUNT) break; } } return ESP_OK; diff --git a/components/nvs_flash/src/nvs_storage.hpp b/components/nvs_flash/src/nvs_storage.hpp index 18ec8ecd8..769abc270 100644 --- a/components/nvs_flash/src/nvs_storage.hpp +++ b/components/nvs_flash/src/nvs_storage.hpp @@ -42,6 +42,22 @@ class Storage : public intrusive_list_node typedef intrusive_list TNamespaces; + struct UsedPageNode: public intrusive_list_node { + public: Page* mPage; + }; + + typedef intrusive_list TUsedPageList; + + struct BlobIndexNode: public intrusive_list_node { + public: + char key[Item::MAX_KEY_LENGTH + 1]; + uint8_t nsIndex; + uint8_t chunkCount; + VerOffset chunkStart; + }; + + typedef intrusive_list TBlobIndexList; + public: ~Storage(); @@ -85,6 +101,12 @@ public: return mPartitionName; } + esp_err_t writeMultiPageBlob(uint8_t nsIndex, const char* key, const void* data, size_t dataSize, VerOffset chunkStart); + + esp_err_t readMultiPageBlob(uint8_t nsIndex, const char* key, void* data, size_t dataSize); + + esp_err_t eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart = VerOffset::VER_ANY); + void debugDump(); void debugCheck(); @@ -102,7 +124,12 @@ protected: void clearNamespaces(); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item); + void populateBlobIndices(TBlobIndexList&); + + void eraseOrphanDataBlobs(TBlobIndexList&); + + + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); protected: const char *mPartitionName; diff --git a/components/nvs_flash/src/nvs_types.cpp b/components/nvs_flash/src/nvs_types.cpp index d44d8b29f..1f60fc350 100644 --- a/components/nvs_flash/src/nvs_types.cpp +++ b/components/nvs_flash/src/nvs_types.cpp @@ -39,6 +39,7 @@ uint32_t Item::calculateCrc32WithoutValue() const result = crc32_le(result, p + offsetof(Item, nsIndex), offsetof(Item, datatype) - offsetof(Item, nsIndex)); result = crc32_le(result, p + offsetof(Item, key), sizeof(key)); + result = crc32_le(result, p + offsetof(Item, chunkIndex), sizeof(chunkIndex)); return result; } diff --git a/components/nvs_flash/src/nvs_types.hpp b/components/nvs_flash/src/nvs_types.hpp index 5306744b5..035f9780d 100644 --- a/components/nvs_flash/src/nvs_types.hpp +++ b/components/nvs_flash/src/nvs_types.hpp @@ -37,9 +37,17 @@ enum class ItemType : uint8_t { I64 = 0x18, SZ = 0x21, BLOB = 0x41, + BLOB_DATA = 0x42, + BLOB_IDX = 0x48, ANY = 0xff }; +enum class VerOffset: uint8_t { + VER_0_OFFSET = 0x0, + VER_1_OFFSET = 0x80, + VER_ANY = 0xff, +}; + template::value, void*>::type = nullptr> constexpr ItemType itemTypeOf() { @@ -52,6 +60,13 @@ constexpr ItemType itemTypeOf(const T&) return itemTypeOf(); } +inline bool isVariableLengthType(ItemType type) +{ + return (type == ItemType::BLOB || + type == ItemType::SZ || + type == ItemType::BLOB_DATA); +} + class Item { public: @@ -60,15 +75,21 @@ public: uint8_t nsIndex; ItemType datatype; uint8_t span; - uint8_t reserved; + uint8_t chunkIndex; uint32_t crc32; char key[16]; union { struct { uint16_t dataSize; - uint16_t reserved2; + uint16_t reserved; uint32_t dataCrc32; } varLength; + struct { + uint32_t dataSize; + uint8_t chunkCount; // Number of children data blobs. + VerOffset chunkStart; // Offset from which the chunkIndex for children blobs starts + uint16_t reserved; + } blobIndex; uint8_t data[8]; }; }; @@ -77,8 +98,12 @@ public: static const size_t MAX_KEY_LENGTH = sizeof(key) - 1; - Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_) - : nsIndex(nsIndex), datatype(datatype), span(span), reserved(0xff) + // 0xff cannot be used as a valid chunkIndex for blob datatype. + static const uint8_t CHUNK_ANY = 0xff; + + + Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_, uint8_t chunkIdx = CHUNK_ANY) + : nsIndex(nsIndex), datatype(datatype), span(span), chunkIndex(chunkIdx) { std::fill_n(reinterpret_cast(key), sizeof(key) / 4, 0xffffffff); std::fill_n(reinterpret_cast(data), sizeof(data) / 4, 0xffffffff); diff --git a/components/nvs_flash/test/test_nvs.c b/components/nvs_flash/test/test_nvs.c index 87ca51994..32d5b2956 100644 --- a/components/nvs_flash/test/test_nvs.c +++ b/components/nvs_flash/test/test_nvs.c @@ -196,15 +196,15 @@ TEST_CASE("calculate used and free space", "[nvs]") uint32_t blob[12]; TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob))); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - TEST_ASSERT_TRUE(stat1.free_entries + 3 == stat2.free_entries); + TEST_ASSERT_TRUE(stat1.free_entries + 4 == stat2.free_entries); TEST_ASSERT_TRUE(stat1.namespace_count == 3); TEST_ASSERT_TRUE(stat1.total_entries == stat2.total_entries); - TEST_ASSERT_TRUE(stat1.used_entries == 11); + TEST_ASSERT_TRUE(stat1.used_entries == 12); // amount valid pair in namespace 2 size_t h3_count_entries; TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries)); - TEST_ASSERT_TRUE(h3_count_entries == 3); + TEST_ASSERT_TRUE(h3_count_entries == 4); TEST_ASSERT_TRUE(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count)); diff --git a/components/nvs_flash/test_nvs_host/spi_flash_emulation.h b/components/nvs_flash/test_nvs_host/spi_flash_emulation.h index 3ee016163..f17573a02 100644 --- a/components/nvs_flash/test_nvs_host/spi_flash_emulation.h +++ b/components/nvs_flash/test_nvs_host/spi_flash_emulation.h @@ -43,7 +43,8 @@ public: { load(filename); // Atleast one page should be free, hence we create mData of size of 2 sectors. - mData.resize(2 * SPI_FLASH_SEC_SIZE / 4, 0xffffffff); + mData.resize(mData.size() + SPI_FLASH_SEC_SIZE / 4, 0xffffffff); + mUpperSectorBound = mData.size() * 4 / SPI_FLASH_SEC_SIZE; spi_flash_emulator_set(this); } diff --git a/components/nvs_flash/test_nvs_host/test_nvs.cpp b/components/nvs_flash/test_nvs_host/test_nvs.cpp index 22d30d055..4309f6d56 100644 --- a/components/nvs_flash/test_nvs_host/test_nvs.cpp +++ b/components/nvs_flash/test_nvs_host/test_nvs.cpp @@ -17,6 +17,7 @@ #include "spi_flash_emulation.h" #include #include +#include #include #include @@ -47,7 +48,7 @@ TEST_CASE("crc32 behaves as expected", "[nvs]") item1.datatype = ItemType::I32; item1.nsIndex = 1; item1.crc32 = 0; - item1.reserved = 0xff; + item1.chunkIndex = 0xff; fill_n(item1.key, sizeof(item1.key), 0xbb); fill_n(item1.data, sizeof(item1.data), 0xaa); @@ -257,8 +258,8 @@ TEST_CASE("Page validates blob size", "[nvs]") // Check that the second one is actually returned. TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::ENTRY_COUNT * Page::ENTRY_SIZE), ESP_ERR_NVS_VALUE_TOO_LONG); // Should fail as well - TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::BLOB_MAX_SIZE + 1), ESP_ERR_NVS_VALUE_TOO_LONG); - TEST_ESP_OK(page.writeItem(1, ItemType::BLOB, "2", buf, Page::BLOB_MAX_SIZE)); + TEST_ESP_ERR(page.writeItem(1, ItemType::BLOB, "2", buf, Page::CHUNK_MAX_SIZE + 1), ESP_ERR_NVS_VALUE_TOO_LONG); + TEST_ESP_OK(page.writeItem(1, ItemType::BLOB, "2", buf, Page::CHUNK_MAX_SIZE)); } TEST_CASE("Page handles invalid CRC of variable length items", "[nvs][cur]") @@ -376,7 +377,7 @@ TEST_CASE("storage can find items on second page if first is not fully written a Storage storage; CHECK(storage.init(0, 3) == ESP_OK); int bar = 0; - uint8_t bigdata[Page::BLOB_MAX_SIZE] = {0}; + uint8_t bigdata[(Page::CHUNK_MAX_SIZE - Page::ENTRY_SIZE)/2] = {0}; // write one big chunk of data ESP_ERROR_CHECK(storage.writeItem(0, ItemType::BLOB, "1", bigdata, sizeof(bigdata))); // write another big chunk of data @@ -775,16 +776,18 @@ TEST_CASE("nvs api tests, starting with random data in flash", "[nvs][long]") nvs_close(handle_1); } } - extern "C" void nvs_dump(const char *partName); class RandomTest { - static const size_t nKeys = 9; + static const size_t nKeys = 11; int32_t v1 = 0, v2 = 0; uint64_t v3 = 0, v4 = 0; static const size_t strBufLen = 1024; + static const size_t smallBlobLen = Page::CHUNK_MAX_SIZE / 3; + static const size_t largeBlobLen = Page::CHUNK_MAX_SIZE * 3; char v5[strBufLen], v6[strBufLen], v7[strBufLen], v8[strBufLen], v9[strBufLen]; + uint8_t v10[smallBlobLen], v11[largeBlobLen]; bool written[nKeys]; public: @@ -796,10 +799,10 @@ public: template esp_err_t doRandomThings(nvs_handle handle, TGen gen, size_t& count) { - const char* keys[] = {"foo", "bar", "longkey_0123456", "another key", "param1", "param2", "param3", "param4", "param5"}; - const ItemType types[] = {ItemType::I32, ItemType::I32, ItemType::U64, ItemType::U64, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ}; + const char* keys[] = {"foo", "bar", "longkey_0123456", "another key", "param1", "param2", "param3", "param4", "param5", "singlepage", "multipage"}; + const ItemType types[] = {ItemType::I32, ItemType::I32, ItemType::U64, ItemType::U64, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::SZ, ItemType::BLOB, ItemType::BLOB}; - void* values[] = {&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9}; + void* values[] = {&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8, &v9, &v10, &v11}; const size_t nKeys = sizeof(keys) / sizeof(keys[0]); static_assert(nKeys == sizeof(types) / sizeof(types[0]), ""); @@ -858,7 +861,35 @@ public: } break; } - + + case ItemType::BLOB: + { + uint32_t blobBufLen = 0; + if(strncmp(keys[index],"singlepage", sizeof("singlepage")) == 0) { + blobBufLen = smallBlobLen ; + } else { + blobBufLen = largeBlobLen ; + + } + uint8_t buf[blobBufLen]; + memset(buf, 0, blobBufLen); + + size_t len = blobBufLen; + auto err = nvs_get_blob(handle, keys[index], buf, &len); + if (err == ESP_ERR_FLASH_OP_FAIL) { + return err; + } + if (!written[index]) { + REQUIRE(err == ESP_ERR_NVS_NOT_FOUND); + } + else { + REQUIRE(err == ESP_OK); + REQUIRE(memcmp(buf, reinterpret_cast(values[index]), blobBufLen) == 0); + } + break; + } + + default: assert(0); } @@ -931,16 +962,46 @@ public: strncpy(reinterpret_cast(values[index]), buf, strBufLen); break; } - + + case ItemType::BLOB: + { + uint32_t blobBufLen = 0; + if(strncmp(keys[index],"singlepage", sizeof("singlepage")) == 0) { + blobBufLen = smallBlobLen ; + } else { + blobBufLen = largeBlobLen ; + } + uint8_t buf[blobBufLen]; + memset(buf, 0, blobBufLen); + size_t blobLen = gen() % blobBufLen; + std::generate_n(buf, blobLen, [&]() -> uint8_t { + return static_cast(gen() % 256); + }); + + auto err = nvs_set_blob(handle, keys[index], buf, blobLen); + if (err == ESP_ERR_FLASH_OP_FAIL) { + return err; + } + if (err == ESP_ERR_NVS_REMOVE_FAILED) { + written[index] = true; + memcpy(reinterpret_cast(values[index]), buf, blobBufLen); + return ESP_ERR_FLASH_OP_FAIL; + } + REQUIRE(err == ESP_OK); + written[index] = true; + memcpy(reinterpret_cast(values[index]), buf, blobBufLen); + break; + } + default: assert(0); } return ESP_OK; }; - + for (; count != 0; --count) { - size_t index = gen() % nKeys; + size_t index = gen() % (nKeys); switch (gen() % 3) { case 0: // read, 1/3 if (randomRead(index) == ESP_ERR_FLASH_OP_FAIL) { @@ -957,8 +1018,20 @@ public: } return ESP_OK; } -}; + esp_err_t handleExternalWriteAtIndex(uint8_t index, const void* value, const size_t len ) { + if(index == 9) { /* This is only done for small-page blobs for now*/ + if(len > smallBlobLen) { + return ESP_FAIL; + } + memcpy(v10, value, len); + written[index] = true; + return ESP_OK; + } else { + return ESP_FAIL; + } + } +}; TEST_CASE("monkey test", "[nvs][monkey]") { @@ -971,8 +1044,8 @@ TEST_CASE("monkey test", "[nvs][monkey]") emu.randomize(seed); emu.clearStats(); - const uint32_t NVS_FLASH_SECTOR = 6; - const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); @@ -996,8 +1069,8 @@ TEST_CASE("test recovery from sudden poweroff", "[long][nvs][recovery][monkey]") SpiFlashEmulator emu(10); - const uint32_t NVS_FLASH_SECTOR = 6; - const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); size_t totalOps = 0; @@ -1042,7 +1115,6 @@ TEST_CASE("test recovery from sudden poweroff", "[long][nvs][recovery][monkey]") totalOps = emu.getEraseOps() + emu.getWriteBytes() / 4; } } - TEST_CASE("test for memory leaks in open/set", "[leaks]") { SpiFlashEmulator emu(10); @@ -1235,7 +1307,7 @@ TEST_CASE("read/write failure (TW8406)", "[nvs]") TEST_CASE("nvs_flash_init checks for an empty page", "[nvs]") { - const size_t blob_size = Page::BLOB_MAX_SIZE; + const size_t blob_size = Page::CHUNK_MAX_SIZE; uint8_t blob[blob_size] = {0}; SpiFlashEmulator emu(5); TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5) ); @@ -1243,13 +1315,10 @@ TEST_CASE("nvs_flash_init checks for an empty page", "[nvs]") TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); // Fill first page TEST_ESP_OK( nvs_set_blob(handle, "1a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "1b", blob, blob_size) ); // Fill second page TEST_ESP_OK( nvs_set_blob(handle, "2a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "2b", blob, blob_size) ); // Fill third page TEST_ESP_OK( nvs_set_blob(handle, "3a", blob, blob_size) ); - TEST_ESP_OK( nvs_set_blob(handle, "3b", blob, blob_size) ); TEST_ESP_OK( nvs_commit(handle) ); nvs_close(handle); // first two pages are now full, third one is writable, last two are empty @@ -1276,7 +1345,7 @@ TEST_CASE("multiple partitions access check", "[nvs]") TEST_CASE("nvs page selection takes into account free entries also not just erased entries", "[nvs]") { - const size_t blob_size = Page::BLOB_MAX_SIZE; + const size_t blob_size = Page::CHUNK_MAX_SIZE/2; uint8_t blob[blob_size] = {0}; SpiFlashEmulator emu(3); TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3) ); @@ -1415,15 +1484,15 @@ TEST_CASE("calculate used and free space", "[nvs]") uint32_t blob[12]; TEST_ESP_OK(nvs_set_blob(handle_3, "bl1", &blob, sizeof(blob))); TEST_ESP_OK(nvs_get_stats(NULL, &stat1)); - CHECK(stat1.free_entries + 3 == stat2.free_entries); + CHECK(stat1.free_entries + 4 == stat2.free_entries); CHECK(stat1.namespace_count == 3); CHECK(stat1.total_entries == stat2.total_entries); - CHECK(stat1.used_entries == 11); + CHECK(stat1.used_entries == 12); // amount valid pair in namespace 2 size_t h3_count_entries; TEST_ESP_OK(nvs_get_used_entry_count(handle_3, &h3_count_entries)); - CHECK(h3_count_entries == 3); + CHECK(h3_count_entries == 4); CHECK(stat1.used_entries == (h1_count_entries + h2_count_entries + h3_count_entries + stat1.namespace_count)); @@ -1432,7 +1501,7 @@ TEST_CASE("calculate used and free space", "[nvs]") TEST_CASE("Recovery from power-off when the entry being erased is not on active page", "[nvs]") { - const size_t blob_size = Page::BLOB_MAX_SIZE; + const size_t blob_size = Page::CHUNK_MAX_SIZE/2 ; size_t read_size = blob_size; uint8_t blob[blob_size] = {0x11}; SpiFlashEmulator emu(3); @@ -1441,7 +1510,7 @@ TEST_CASE("Recovery from power-off when the entry being erased is not on active TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); emu.clearStats(); - emu.failAfter(2 * Page::BLOB_MAX_SIZE/4 + 36); + emu.failAfter(Page::CHUNK_MAX_SIZE/4 + 75); TEST_ESP_OK( nvs_set_blob(handle, "1a", blob, blob_size) ); TEST_ESP_OK( nvs_set_blob(handle, "1b", blob, blob_size) ); @@ -1499,6 +1568,390 @@ TEST_CASE("Recovery from power-off when page is being freed.", "[nvs]") nvs_close(handle); } +TEST_CASE("Multi-page blobs are supported", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *2; + uint8_t blob[blob_size] = {0}; + SpiFlashEmulator emu(5); + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5)); + nvs_handle handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &handle)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); +} + +TEST_CASE("Failures are handled while storing multi-page blobs", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *7; + uint8_t blob[blob_size] = {0}; + SpiFlashEmulator emu(5); + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5)); + nvs_handle handle; + TEST_ESP_OK(nvs_open("test", NVS_READWRITE, &handle)); + TEST_ESP_ERR(nvs_set_blob(handle, "abc", blob, blob_size), ESP_ERR_NVS_VALUE_TOO_LONG); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE*2)); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); +} + +TEST_CASE("Reading multi-page blobs", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size]; + uint8_t blob_read[blob_size]; + size_t read_size = blob_size; + SpiFlashEmulator emu(5); + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5)); + nvs_handle handle; + memset(blob, 0x11, blob_size); + memset(blob_read, 0xee, blob_size); + TEST_ESP_OK(nvs_open("readTest", NVS_READWRITE, &handle)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, blob_size) == 0); + TEST_ESP_OK(nvs_commit(handle)); + nvs_close(handle); +} + +TEST_CASE("Modification of values for Multi-page blobs are supported", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *2; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xfe};; + uint8_t blob2[blob_size] = {0x11}; + uint8_t blob3[blob_size] = {0x22}; + uint8_t blob4[blob_size] ={ 0x33}; + size_t read_size = blob_size; + SpiFlashEmulator emu(6); + TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 6) ); + nvs_handle handle; + memset(blob, 0x11, blob_size); + memset(blob2, 0x22, blob_size); + memset(blob3, 0x33, blob_size); + memset(blob4, 0x44, blob_size); + memset(blob_read, 0xff, blob_size); + TEST_ESP_OK( nvs_open("test", NVS_READWRITE, &handle) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob2, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob3, blob_size) ); + TEST_ESP_OK( nvs_set_blob(handle, "abc", blob4, blob_size) ); + TEST_ESP_OK( nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob4, blob_read, blob_size) == 0); + TEST_ESP_OK( nvs_commit(handle) ); + nvs_close(handle); +} + +TEST_CASE("Modification from single page blob to multi-page", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xff}; + size_t read_size = blob_size; + SpiFlashEmulator emu(5); + TEST_ESP_OK( nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5) ); + nvs_handle handle; + TEST_ESP_OK(nvs_open("Test", NVS_READWRITE, &handle) ); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE/2)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, blob_size) == 0); + TEST_ESP_OK(nvs_commit(handle) ); + nvs_close(handle); +} + +TEST_CASE("Modification from multi-page to single page", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3; + uint8_t blob[blob_size] = {0}; + uint8_t blob_read[blob_size] = {0xff}; + size_t read_size = blob_size; + SpiFlashEmulator emu(5); + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 5) ); + nvs_handle handle; + TEST_ESP_OK(nvs_open("Test", NVS_READWRITE, &handle) ); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, blob_size)); + TEST_ESP_OK(nvs_set_blob(handle, "abc", blob, Page::CHUNK_MAX_SIZE/2)); + TEST_ESP_OK(nvs_set_blob(handle, "abc2", blob, blob_size)); + TEST_ESP_OK(nvs_get_blob(handle, "abc", blob_read, &read_size)); + CHECK(memcmp(blob, blob_read, Page::CHUNK_MAX_SIZE) == 0); + TEST_ESP_OK(nvs_commit(handle) ); + nvs_close(handle); +} + + +TEST_CASE("Check that orphaned blobs are erased during init", "[nvs]") +{ + const size_t blob_size = Page::CHUNK_MAX_SIZE *3 ; + uint8_t blob[blob_size] = {0x11}; + uint8_t blob2[blob_size] = {0x22}; + uint8_t blob3[blob_size] = {0x33}; + SpiFlashEmulator emu(5); + Storage storage; + + TEST_ESP_OK(storage.init(0, 5)); + + TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, "key", blob, sizeof(blob))); + + + TEST_ESP_OK(storage.init(0, 5)); + /* Check that multi-page item is still available.**/ + TEST_ESP_OK(storage.readItem(1, ItemType::BLOB, "key", blob, sizeof(blob))); + + TEST_ESP_ERR(storage.writeItem(1, ItemType::BLOB, "key2", blob, sizeof(blob)), ESP_ERR_NVS_NOT_ENOUGH_SPACE); + + Page p; + p.load(3); // This is where index will be placed. + p.erase(); + + TEST_ESP_OK(storage.init(0, 5)); + + TEST_ESP_ERR(storage.readItem(1, ItemType::BLOB, "key", blob, sizeof(blob)), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(storage.writeItem(1, ItemType::BLOB, "key3", blob, sizeof(blob))); +} + +TEST_CASE("Check that NVS supports old blob format without blob index", "[nvs]") +{ + SpiFlashEmulator emu("../nvs_partition_generator/part_old_blob_format.bin"); + nvs_handle handle; + + TEST_ESP_OK( nvs_flash_init_custom("test", 0, 2) ); + TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle)); + + char buf[64] = {0}; + size_t buflen = 64; + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + buflen = 64; + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + + Page p; + p.load(0); + + /* Check that item is stored in old format without blob index*/ + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "dummyHex2BinKey")); + + /* Modify the blob so that it is stored in the new format*/ + hexdata[0] = hexdata[1] = hexdata[2] = 0x99; + TEST_ESP_OK(nvs_set_blob(handle, "dummyHex2BinKey", hexdata, sizeof(hexdata))); + + Page p2; + p2.load(0); + + /* Check the type of the blob. Expect type mismatch since the blob is stored in new format*/ + TEST_ESP_ERR(p2.findItem(1, ItemType::BLOB, "dummyHex2BinKey"), ESP_ERR_NVS_TYPE_MISMATCH); + + /* Check that index is present for the modified blob according to new format*/ + TEST_ESP_OK(p2.findItem(1, ItemType::BLOB_IDX, "dummyHex2BinKey")); + + /* Read the blob in new format and check the contents*/ + buflen = 64; + TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); + CHECK(memcmp(buf, base64data, buflen) == 0); + +} + +TEST_CASE("monkey test with old-format blob present", "[nvs][monkey]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + SpiFlashEmulator emu(10); + emu.randomize(seed); + emu.clearStats(); + + const uint32_t NVS_FLASH_SECTOR = 2; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 8; + static const size_t smallBlobLen = Page::CHUNK_MAX_SIZE / 3; + + emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN); + + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); + + nvs_handle handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + RandomTest test; + + for ( uint8_t it = 0; it < 10; it++) { + size_t count = 200; + + /* Erase index and chunks for the blob with "singlepage" key */ + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(num); + p.eraseItem(1, ItemType::BLOB, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + p.eraseItem(1, ItemType::BLOB_IDX, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + p.eraseItem(1, ItemType::BLOB_DATA, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY); + } + + /* Now write "singlepage" blob in old format*/ + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(num); + if (p.state() == Page::PageState::ACTIVE) { + uint8_t buf[smallBlobLen]; + size_t blobLen = gen() % smallBlobLen; + + if(blobLen > p.getVarDataTailroom()) { + blobLen = p.getVarDataTailroom(); + } + + std::generate_n(buf, blobLen, [&]() -> uint8_t { + return static_cast(gen() % 256); + }); + + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", buf, blobLen, Item::CHUNK_ANY)); + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + test.handleExternalWriteAtIndex(9, buf, blobLen); // This assumes "singlepage" is always at index 9 + + break; + } + } + /* Initialize again */ + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, NVS_FLASH_SECTOR, NVS_FLASH_SECTOR_COUNT_MIN)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + /* Perform random things */ + auto res = test.doRandomThings(handle, gen, count); + if (res != ESP_OK) { + nvs_dump(NVS_DEFAULT_PART_NAME); + CHECK(0); + } + + /* Check that only one version is present for "singlepage". Its possible that last iteration did not write + * anything for "singlepage". So either old version or new version should be present.*/ + bool oldVerPresent = false, newVerPresent = false; + + for (uint8_t num = NVS_FLASH_SECTOR; num < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; num++) { + Page p; + p.load(num); + if(!oldVerPresent && p.findItem(1, ItemType::BLOB, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY) == ESP_OK) { + oldVerPresent = true; + } + + if(!newVerPresent && p.findItem(1, ItemType::BLOB_IDX, "singlepage", Item::CHUNK_ANY, VerOffset::VER_ANY) == ESP_OK) { + newVerPresent = true; + } + } + CHECK(oldVerPresent != newVerPresent); + } + + s_perf << "Monkey test: nErase=" << emu.getEraseOps() << " nWrite=" << emu.getWriteOps() << std::endl; +} + +TEST_CASE("Recovery from power-off during modification of blob present in old-format (same page)", "[nvs]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + SpiFlashEmulator emu(3); + emu.clearStats(); + + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3)); + + nvs_handle handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + uint8_t hexdata_old[] = {0x11, 0x12, 0x13, 0xbb, 0xcc, 0xee}; + size_t buflen = sizeof(hexdata); + uint8_t buf[Page::CHUNK_MAX_SIZE]; + + /* Power-off when blob was being written on the same page where its old version in old format + * was present*/ + Page p; + p.load(0); + /* Write blob in old-format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", hexdata_old, sizeof(hexdata_old))); + + /* Write blob in new format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_DATA, "singlepage", hexdata, sizeof(hexdata), 0)); + /* All pages are stored. Now store the index.*/ + Item item; + item.blobIndex.dataSize = sizeof(hexdata); + item.blobIndex.chunkCount = 1; + item.blobIndex.chunkStart = VerOffset::VER_0_OFFSET; + + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_IDX, "singlepage", item.data, sizeof(item.data))); + + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + + /* Initialize again */ + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + TEST_ESP_OK( nvs_get_blob(handle, "singlepage", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + Page p2; + p2.load(0); + TEST_ESP_ERR(p2.findItem(1, ItemType::BLOB, "singlepage"), ESP_ERR_NVS_TYPE_MISMATCH); + +} + +TEST_CASE("Recovery from power-off during modification of blob present in old-format (different page)", "[nvs]") +{ + std::random_device rd; + std::mt19937 gen(rd()); + uint32_t seed = 3; + gen.seed(seed); + + SpiFlashEmulator emu(3); + emu.clearStats(); + + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3)); + + nvs_handle handle; + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + uint8_t hexdata_old[] = {0x11, 0x12, 0x13, 0xbb, 0xcc, 0xee}; + size_t buflen = sizeof(hexdata); + uint8_t buf[Page::CHUNK_MAX_SIZE]; + + + /* Power-off when blob was being written on the different page where its old version in old format + * was present*/ + Page p; + p.load(0); + /* Write blob in old-format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB, "singlepage", hexdata_old, sizeof(hexdata_old))); + + /* Write blob in new format*/ + TEST_ESP_OK(p.writeItem(1, ItemType::BLOB_DATA, "singlepage", hexdata, sizeof(hexdata), 0)); + /* All pages are stored. Now store the index.*/ + Item item; + item.blobIndex.dataSize = sizeof(hexdata); + item.blobIndex.chunkCount = 1; + item.blobIndex.chunkStart = VerOffset::VER_0_OFFSET; + p.markFull(); + Page p2; + p2.load(1); + p2.setSeqNumber(1); + + TEST_ESP_OK(p2.writeItem(1, ItemType::BLOB_IDX, "singlepage", item.data, sizeof(item.data))); + + TEST_ESP_OK(p.findItem(1, ItemType::BLOB, "singlepage")); + + /* Initialize again */ + TEST_ESP_OK(nvs_flash_init_custom(NVS_DEFAULT_PART_NAME, 0, 3)); + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle)); + + TEST_ESP_OK( nvs_get_blob(handle, "singlepage", buf, &buflen)); + CHECK(memcmp(buf, hexdata, buflen) == 0); + + Page p3; + p3.load(0); + TEST_ESP_ERR(p3.findItem(1, ItemType::BLOB, "singlepage"), ESP_ERR_NVS_NOT_FOUND); +} /* Add new tests above */ /* This test has to be the final one */ @@ -1523,7 +1976,7 @@ TEST_CASE("read data from partition generated via partition generation utility", { SpiFlashEmulator emu("../nvs_partition_generator/partition.bin"); nvs_handle handle; - TEST_ESP_OK( nvs_flash_init_custom("test", 0, 2) ); + TEST_ESP_OK( nvs_flash_init_custom("test", 0, 3) ); TEST_ESP_OK( nvs_open_from_partition("test", "dummyNamespace", NVS_READONLY, &handle)); uint8_t u8v; TEST_ESP_OK( nvs_get_u8(handle, "dummyU8Key", &u8v)); @@ -1540,17 +1993,43 @@ TEST_CASE("read data from partition generated via partition generation utility", int32_t i32v; TEST_ESP_OK( nvs_get_i32(handle, "dummyI32Key", &i32v)); CHECK(i32v == -2147483648); + char buf[64] = {0}; size_t buflen = 64; TEST_ESP_OK( nvs_get_str(handle, "dummyStringKey", buf, &buflen)); CHECK(strncmp(buf, "0A:0B:0C:0D:0E:0F", buflen) == 0); - buflen = 64; + uint8_t hexdata[] = {0x01, 0x02, 0x03, 0xab, 0xcd, 0xef}; + buflen = 64; + int j; TEST_ESP_OK( nvs_get_blob(handle, "dummyHex2BinKey", buf, &buflen)); CHECK(memcmp(buf, hexdata, buflen) == 0); + uint8_t base64data[] = {'1', '2', '3', 'a', 'b', 'c'}; TEST_ESP_OK( nvs_get_blob(handle, "dummyBase64Key", buf, &buflen)); CHECK(memcmp(buf, base64data, buflen) == 0); + + buflen = 64; + uint8_t hexfiledata[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}; + TEST_ESP_OK( nvs_get_blob(handle, "hexFileKey", buf, &buflen)); + CHECK(memcmp(buf, hexfiledata, buflen) == 0); + + buflen = 64; + uint8_t strfiledata[64] = "abcdefghijklmnopqrstuvwxyz\0"; + TEST_ESP_OK( nvs_get_str(handle, "stringFileKey", buf, &buflen)); + CHECK(memcmp(buf, strfiledata, buflen) == 0); + + char bin_data[5200]; + size_t bin_len = sizeof(bin_data); + char binfiledata[5200]; + ifstream file; + file.open("../nvs_partition_generator/testdata/sample.bin"); + file.read(binfiledata,5200); + TEST_ESP_OK( nvs_get_blob(handle, "binFileKey", bin_data, &bin_len)); + CHECK(memcmp(bin_data, binfiledata, bin_len) == 0); + + file.close(); + } TEST_CASE("dump all performance data", "[nvs]")