From 81831a27c70c6bfc5941af0fb9006d9c61bd77ea Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 14 Mar 2026 09:25:49 -0700 Subject: [PATCH 01/21] Adds SNode Module And Unit Test Adds SNode module with the ability to create, locate, read, write, erase. Pages are stored as extents in the SNode, meaning they cover a range of pages depending on the data written to the page. Locating operation to find the current extent has time complexity of O(n) and must be performed before reading/writing/erasing. The extents are limited to the multiple extent operation. Read/write/erase operations have time complexity of O(1). --- src/snode.c | 745 ++++++++++++++++++++++++++++++++++++++++++++++ src/snode.h | 75 +++++ test/test_snode.c | 211 +++++++++++++ 3 files changed, 1031 insertions(+) create mode 100644 src/snode.c create mode 100644 src/snode.h create mode 100644 test/test_snode.c diff --git a/src/snode.c b/src/snode.c new file mode 100644 index 0000000..1ea2566 --- /dev/null +++ b/src/snode.c @@ -0,0 +1,745 @@ +#include "snode.h" + +#include "atomic.h" +#include "bitmap.h" +#include "common.h" +#include "crc.h" + +#include + +#define INLINE_DATA_SIZE(f) (f->pageSize - sizeof(SNode)) +#define EXTENTS_PER_PAGE(f) ((f->pageSize / sizeof(SNodeExtent))) +#define CALC_CONTIGUOUS_MAX(f, o) CEIL_DIV(o->bytes_remaining, f->pageSize) + +typedef enum { + SNODE_READ, + SNODE_WRITE, + SNODE_ERASE, +} SNodeOp; + +typedef struct { + SNodeExtent extent; // Allocated extent on write, filled from flash otherwise + SNodeOp op; + uint32_t bytes_remaining; +} SNodeOpInst; + +static storfs_err_t snode_alloc_new_page(storfs_t *fs, storfs_page_t *page) { + storfs_err_t err = bitmap_alloc(fs, page); + if(err != STORFS_OK) { + return err; + } + + memset(fs->working_buf, 0, fs->pageSize); + + return atomic_write(fs, *page); +} + +static storfs_err_t +snode_update(storfs_t *fs, SNode *node, storfs_page_t page) { + // Read snode page to preserve inline data + storfs_err_t err = atomic_read(fs, page); + if(err != STORFS_OK) { + return err; + } + + node->crc = 0; + node->crc = storfs_crc16((const uint8_t *)node, sizeof(SNode)); + + SNode *write_node = (SNode *)fs->working_buf; + memcpy(write_node, node, sizeof(SNode)); + + return atomic_write(fs, page); +} + +storfs_err_t snode_create(storfs_t *fs, + const char *name, + storfs_page_t *page, + uint8_t type) { + if(!fs || !name || !page) { + return STORFS_ERR_NULL_POINTER; + } + + SNode node = { 0 }; + storfs_err_t err = snode_alloc_new_page(fs, page); + if(err != STORFS_OK) { + return err; + } + + strncpy((char *)node.name, name, STORFS_MAX_FILE_NAME); + node.name[STORFS_MAX_FILE_NAME - 1] = '\0'; + + node.type = type; + return snode_update(fs, &node, *page); +} + +storfs_err_t snode_lookup(storfs_t *fs, storfs_page_t page, SNodeInst *inst) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + + SNode *node = &inst->node; + storfs_err_t err = atomic_read(fs, page); + if(err != STORFS_OK) { + return err; + } + memcpy(node, fs->working_buf, sizeof(SNode)); + + uint32_t crc = node->crc; + node->crc = 0; + node->crc = storfs_crc16((uint8_t *)node, sizeof(SNode)); + + if(crc != node->crc) { + return STORFS_ERR_CRC_MISMATCH; + } + + inst->page = page; + inst->write = (SNodeExtentCache){ 0 }; + inst->read = (SNodeExtentCache){ 0 }; + + return STORFS_OK; +} + +static storfs_err_t find_page_in_extents(SNodeExtentCache *cache, + const SNodeExtent *extents, + uint32_t count, + uint32_t *logical_page) { + // If extents[i] == 0, the extent is empty + for(uint32_t i = 0; i < count && extents[i].count > 0; i++) { + if(*logical_page < extents[i].count) { + return STORFS_OK; + } + cache->idx++; + *logical_page -= extents[i].count; + } + + return STORFS_ERR_NOT_FOUND; +} + +static storfs_err_t find_extent_in_indirect_page(storfs_t *fs, + SNodeExtentCache *cache, + storfs_page_t indirect_page, + storfs_page_t *logical_page) { + storfs_err_t err = atomic_read(fs, indirect_page); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *extents = (SNodeExtent *)fs->working_buf; + uint32_t extent_count = EXTENTS_PER_PAGE(fs); + + return find_page_in_extents(cache, extents, extent_count, logical_page); +} + +static storfs_err_t find_double_indirect_location(storfs_t *fs, + SNodeInst *inst, + SNodeExtentCache *cache, + storfs_page_t *logical_page) { + if(!inst->node.indirect.multiple) { + return STORFS_ERR_NOT_FOUND; + } + + storfs_err_t err = atomic_read(fs, inst->node.indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *extents = (SNodeExtent *)fs->working_buf; + storfs_page_t single_indirect_location = 0; + + // Determine which indirect extent page holds the desired logical location + for(uint32_t i = 0; i < EXTENTS_PER_PAGE(fs); i++) { + if(!extents[i].count) { + break; + } + + if(*logical_page < extents[i].count) { + single_indirect_location = extents[i].start; + break; + } + + cache->idx += EXTENTS_PER_PAGE(fs); + *logical_page -= extents[i].count; + } + + if(!single_indirect_location) { + return STORFS_ERR_NOT_FOUND; + } + + err = find_extent_in_indirect_page(fs, + cache, + single_indirect_location, + logical_page); + return err; +} + +static void +find_update_cache(storfs_t *fs, SNodeExtentCache *cache, storfs_loc_t logical) { + cache->offset_bytes = logical.pageLoc * fs->pageSize + logical.byteLoc; + // Increment here as index 0 is the inline data + cache->idx++; +} + +static storfs_err_t find_location(storfs_t *fs, + SNodeInst *inst, + SNodeExtentCache *cache, + storfs_byte_t offset) { + cache->idx = 0; + + if(offset < INLINE_DATA_SIZE(fs)) { + cache->offset_bytes = sizeof(SNode) + offset; + return STORFS_OK; + } + + // Find the logical page offset, how many pages would the data consume + // in a single contiguous block + const uint32_t snode_inline_data_size = INLINE_DATA_SIZE(fs); + + // logical.pageLoc is decremented throughout these operations if non zero, + // this will be the total offset in bytes from the extent start location + storfs_loc_t logical; + uint32_t data_beyond_snode = offset - snode_inline_data_size; + logical.pageLoc = data_beyond_snode / fs->pageSize; + logical.byteLoc = data_beyond_snode % fs->pageSize; + + uint32_t extent_count = ARRAY_SIZE(inst->node.direct); + storfs_err_t err = find_page_in_extents(cache, + inst->node.direct, + extent_count, + &logical.pageLoc); + if(err == STORFS_OK) { + find_update_cache(fs, cache, logical); + return STORFS_OK; + } + + if(!inst->node.indirect.single) { + find_update_cache(fs, cache, logical); + return STORFS_ERR_NOT_FOUND; + } + err = find_extent_in_indirect_page(fs, + cache, + inst->node.indirect.single, + &logical.pageLoc); + if(err == STORFS_OK) { + find_update_cache(fs, cache, logical); + return STORFS_OK; + } else if(err != STORFS_ERR_NOT_FOUND) { + return err; + } + + err = find_double_indirect_location(fs, inst, cache, &logical.pageLoc); + if(err != STORFS_OK && err != STORFS_ERR_NOT_FOUND) { + return err; + } + + find_update_cache(fs, cache, logical); + return err; +} + +storfs_err_t +snode_find_read_location(storfs_t *fs, SNodeInst *inst, storfs_byte_t offset) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + + return find_location(fs, inst, &inst->read, offset); +} + +storfs_err_t snode_find_write_location(storfs_t *fs, SNodeInst *inst) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + + return find_location(fs, inst, &inst->write, inst->node.size); +} + +static storfs_err_t +snode_alloc_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t *page) { + storfs_err_t err = snode_alloc_new_page(fs, page); + if(err != STORFS_OK) { + return err; + } + return snode_update(fs, &inst->node, inst->page); +} + +static inline storfs_page_t calculate_freed(const storfs_t *fs, + const SNodeExtent *extent, + const SNodeOpInst *op) { + storfs_page_t pages_remaining = op->bytes_remaining / fs->pageSize; + storfs_page_t freed = + pages_remaining > extent->count ? extent->count : pages_remaining; + return freed; +} + +static inline void erase_decrement_extent(const storfs_t *fs, + SNodeExtent *extent, + const SNodeOpInst *op) { + // Decrement extent count by the number of pages contiguously freed + extent->count -= calculate_freed(fs, extent, op); + if(!extent->count) { + extent->start = 0; + } +} + +static storfs_err_t +erase_snode_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t page) { + storfs_err_t err = snode_update(fs, &inst->node, inst->page); + if(err != STORFS_OK) { + return err; + } + return bitmap_alloc_page(fs, page, PAGE_FREE); +} + +static storfs_err_t process_extent_pages(storfs_t *fs, + SNodeOpInst *op, + uint32_t indirect_idx, + uint32_t *indirect_page) { + storfs_err_t err = STORFS_OK; + + if(op->op == SNODE_WRITE) { + uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); + err = + bitmap_alloc_contiguous(fs, &op->extent.start, &op->extent.count, max); + if(err != STORFS_OK) { + return err; + } + } + + err = atomic_read(fs, *indirect_page); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *indirect_extent = + &((SNodeExtent *)fs->working_buf)[indirect_idx]; + + switch(op->op) { + case SNODE_WRITE: + indirect_extent->start = op->extent.start; + indirect_extent->count = op->extent.count; + break; + case SNODE_READ: + op->extent.start = indirect_extent->start; + op->extent.count = indirect_extent->count; + break; + case SNODE_ERASE: + op->extent.start = indirect_extent->start; + op->extent.count = indirect_extent->count; + erase_decrement_extent(fs, indirect_extent, op); + break; + } + + if(op->op != SNODE_READ) { + err = atomic_write(fs, *indirect_page); + } + + if(op->op == SNODE_ERASE && !indirect_idx && !indirect_extent->start) { + *indirect_page = 0; + } + + return err; +} + +static storfs_err_t process_direct_extents(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache) { + SNode *node = &inst->node; + SNodeExtent *extent = &op->extent; + SNodeExtent *direct_extent = &node->direct[cache.idx]; + + storfs_err_t err = STORFS_OK; + extent->start = direct_extent->start; + extent->count = direct_extent->count; + + if(op->op == SNODE_WRITE) { + uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); + err = bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); + if(err != STORFS_OK) { + return err; + } + + direct_extent->start = extent->start; + direct_extent->count = extent->count; + } else if(op->op == SNODE_ERASE) { + erase_decrement_extent(fs, direct_extent, op); + } + + if(op->op != SNODE_READ) { + err = snode_update(fs, node, inst->page); + } + + return err; +} + +static storfs_err_t process_indirect_extents(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache) { + SNode *node = &inst->node; + uint32_t single_indirect_extent_idx = cache.idx - DIRECT_EXTENT_SIZE; + storfs_err_t err = STORFS_OK; + + if(op->op == SNODE_WRITE && !node->indirect.single) { + err = snode_alloc_indirect_page(fs, inst, &node->indirect.single); + if(err != STORFS_OK) { + return err; + } + } + + if(!node->indirect.single) { + return STORFS_ERR_NOT_FOUND; + } + + storfs_page_t init_single_indirect = node->indirect.single; + + err = process_extent_pages(fs, + op, + single_indirect_extent_idx, + &node->indirect.single); + if(err != STORFS_OK) { + return err; + } + + if(op->op == SNODE_ERASE && !node->indirect.single && init_single_indirect) { + err = erase_snode_indirect_page(fs, inst, init_single_indirect); + } + + return err; +} + +static storfs_err_t process_multiple_extents(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache) { + storfs_err_t err = STORFS_OK; + SNode *node = &inst->node; + const uint32_t epp = EXTENTS_PER_PAGE(fs); + const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; + + if(op->op == SNODE_WRITE && !node->indirect.multiple) { + err = snode_alloc_indirect_page(fs, inst, &node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + } + + if(!node->indirect.multiple) { + return STORFS_ERR_NOT_FOUND; + } + + uint32_t single_indirect_page_idx = (cache.idx - si_size) / epp; + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *multiple_extents = (SNodeExtent *)fs->working_buf; + storfs_page_t single_indirect_page = + multiple_extents[single_indirect_page_idx].start; + if(op->op == SNODE_WRITE && !single_indirect_page) { + err = snode_alloc_new_page(fs, &single_indirect_page); + if(err != STORFS_OK) { + return err; + } + + // Must re-read indirect multiple as snode_alloc_new_page clobers buffer + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + multiple_extents = (SNodeExtent *)fs->working_buf; + multiple_extents[single_indirect_page_idx].start = single_indirect_page; + + err = atomic_write(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + } + + uint32_t single_indirect_extent_idx = (cache.idx - si_size) % epp; + storfs_page_t init_single_indirect = single_indirect_page; + + err = process_extent_pages(fs, + op, + single_indirect_extent_idx, + &single_indirect_page); + if(err != STORFS_OK) { + return err; + } + + if(op->op != SNODE_READ) { + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + multiple_extents = (SNodeExtent *)fs->working_buf; + SNodeExtent *multiple_extent = &multiple_extents[single_indirect_page_idx]; + if(op->op == SNODE_WRITE) { + multiple_extent->count += op->extent.count; + } else { + multiple_extent->count -= calculate_freed(fs, &op->extent, op); + if(!multiple_extent->count) { + multiple_extent->start = 0; + } + } + + err = atomic_write(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + // Safe to free single-indirect page now that parent is on flash + if(op->op == SNODE_ERASE && !single_indirect_page && init_single_indirect) { + err = bitmap_alloc_page(fs, init_single_indirect, PAGE_FREE); + if(err != STORFS_OK) { + return err; + } + } + + bool empty_first_extent = + !single_indirect_page_idx && !multiple_extent->start; + if(op->op == SNODE_ERASE && empty_first_extent) { + storfs_page_t init_multiple_indirect = node->indirect.multiple; + node->indirect.multiple = 0; + err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); + } + } + + return err; +} + +static storfs_err_t +get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { + SNodeExtentCache extent_cache; + + const uint32_t epp = EXTENTS_PER_PAGE(fs); + const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; + const uint32_t mi_size = si_size + epp * epp; + + switch(op->op) { + case SNODE_WRITE: + case SNODE_ERASE: + extent_cache = inst->write; + break; + case SNODE_READ: + default: + extent_cache = inst->read; + break; + } + + // Inline page + if(!extent_cache.idx) { + op->extent.start = inst->page; + op->extent.count = 1; + return STORFS_OK; + } + // Subtract 1 as 1 is inline page + extent_cache.idx -= 1; + + if(extent_cache.idx < DIRECT_EXTENT_SIZE) { + return process_direct_extents(fs, inst, op, extent_cache); + } else if(extent_cache.idx < si_size) { + return process_indirect_extents(fs, inst, op, extent_cache); + } else if(extent_cache.idx < mi_size) { + return process_multiple_extents(fs, inst, op, extent_cache); + } + + return STORFS_ERR_NO_FREE_BLOCKS; +} + +static storfs_err_t snode_read_or_write_data(storfs_t *fs, + SNodeOpInst *op, + uint8_t *data, + uint32_t size, + SNodeExtentCache *cache) { + storfs_loc_t location; + location.pageLoc = op->extent.start + cache->offset_bytes / fs->pageSize; + location.byteLoc = cache->offset_bytes % fs->pageSize; + + storfs_err_t err = atomic_read(fs, location.pageLoc); + if(err != STORFS_OK) { + return err; + } + + storfs_page_t pages_accessed = 0; + storfs_page_t pages_accessed_total = + op->extent.count - (location.pageLoc - op->extent.start); + uint32_t bytes_to_process = 0; + + // Loop through extents performing necessary action + while(pages_accessed < pages_accessed_total) { + uint32_t page_size_left = fs->pageSize - location.byteLoc; + bytes_to_process = MIN(op->bytes_remaining, page_size_left); + uint32_t data_offset = size - op->bytes_remaining; + + switch(op->op) { + case SNODE_WRITE: + memcpy(&fs->working_buf[location.byteLoc], + &data[data_offset], + bytes_to_process); + + err = atomic_write(fs, location.pageLoc); + break; + case SNODE_READ: + memcpy(&data[data_offset], + &fs->working_buf[location.byteLoc], + bytes_to_process); + if(op->bytes_remaining > bytes_to_process && + pages_accessed < pages_accessed_total - 1) { + err = atomic_read(fs, location.pageLoc + 1); + } + break; + } + if(err != STORFS_OK) { + break; + } + + location.byteLoc = 0; + location.pageLoc++; + pages_accessed++; + op->bytes_remaining -= bytes_to_process; + } + + // Update offset within contiguous block + if(err != STORFS_OK || !op->bytes_remaining) { + if(pages_accessed) { + cache->offset_bytes = (pages_accessed - 1) * fs->pageSize; + } + cache->offset_bytes += bytes_to_process; + } else { + cache->idx++; + cache->offset_bytes = 0; + } + + return err; +} + +static storfs_err_t erase_data(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache *cache) { + storfs_err_t err = STORFS_OK; + storfs_page_t freed = calculate_freed(fs, &op->extent, op); + uint32_t page_start = op->extent.start + op->extent.count - freed; + uint32_t pages_erased = freed; + uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; + + if(pages_erased) { + err = bitmap_free_contiguous(fs, page_start, &pages_erased, pages_erased); + } + + uint32_t bytes_erased = pages_erased * fs->pageSize; + op->bytes_remaining -= bytes_erased; + + // Is there a partial erase needed for this extent? + if(err == STORFS_OK && erase_bytes_offset && + pages_erased < op->extent.count) { + // Erase page previous to the contiguous start location + page_start--; + err = atomic_read(fs, page_start); + if(err != STORFS_OK) { + goto finish; + } + + // If page is SNode inline data the start of data increases by sizeof(SNode) + erase_bytes_offset += inst->page == page_start ? sizeof(SNode) : 0; + + uint32_t zero_size = fs->pageSize - erase_bytes_offset; + memset(&fs->working_buf[erase_bytes_offset], 0, zero_size); + err = atomic_write(fs, page_start); + if(err != STORFS_OK) { + goto finish; + } + bytes_erased += op->bytes_remaining; + op->bytes_remaining = 0; + } + +finish: + if(err != STORFS_OK || !op->bytes_remaining) { + cache->offset_bytes = op->extent.count * fs->pageSize - bytes_erased; + } else { + cache->idx = cache->idx ? cache->idx - 1 : 0; + cache->offset_bytes = 0; + } + + return err; +} + +static storfs_err_t snode_perform_op(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + uint8_t *data, + uint32_t size) { + if(!fs || !inst || (!data && op->op != SNODE_ERASE)) { + return STORFS_ERR_NULL_POINTER; + } + + storfs_err_t err = STORFS_OK; + SNodeExtentCache *cache = &inst->write; + + if(op->op == SNODE_READ) { + cache = &inst->read; + } + + // Do not erase past end of file + if(op->op == SNODE_ERASE && size > inst->node.size) { + return STORFS_ERR_INVALID_PARAM; + } + + while(err == STORFS_OK && op->bytes_remaining) { + err = get_modify_extents(fs, inst, op); + if(err != STORFS_OK) { + break; + } + + if(op->op != SNODE_ERASE) { + err = snode_read_or_write_data(fs, op, data, size, cache); + } else { + err = erase_data(fs, inst, op, cache); + } + } + + uint32_t processed_bytes = size - op->bytes_remaining; + if(!processed_bytes) { + return err; + } + + if(op->op == SNODE_WRITE) { + inst->node.size += processed_bytes; + } else if(op->op == SNODE_ERASE) { + inst->node.size -= processed_bytes; + } + + if(op->op != SNODE_READ) { + storfs_err_t update_err = snode_update(fs, &inst->node, inst->page); + if(err == STORFS_OK) { + err = update_err; + } + } + + return err; +} + +storfs_err_t snode_write_data(storfs_t *fs, + SNodeInst *inst, + const uint8_t *data, + uint32_t size) { + SNodeOpInst op = { .op = SNODE_WRITE, .bytes_remaining = size }; + + return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); +} + +storfs_err_t +snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { + SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = size }; + + return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); +} + +storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t size) { + SNodeOpInst op = { .op = SNODE_ERASE, .bytes_remaining = size }; + + return snode_perform_op(fs, inst, &op, NULL, size); +} diff --git a/src/snode.h b/src/snode.h new file mode 100644 index 0000000..1f8e203 --- /dev/null +++ b/src/snode.h @@ -0,0 +1,75 @@ +#ifndef __STORFS_SNODE_H__ +#define __STORFS_SNODE_H__ + +#include "storfs.h" + +#include +#include + +#define SNODE_TOTAL_SIZE 128 + +#define SNODE_INFO_SIZE 64 +#define SNODE_RESERVED_SIZE \ + (SNODE_TOTAL_SIZE - STORFS_MAX_FILE_NAME - SNODE_INFO_SIZE) + +#define DIRECT_EXTENT_SIZE 4 + +// Type flags +#define SNODE_TYPE_MASK 0x000F +#define SNODE_TYPE_FILE 0x0001 +#define SNODE_TYPE_DIR 0x0002 + +#define SNODE_CHECK_TYPE_FREE(snode) (snode->type == 0) +#define SNODE_CHECK_TYPE_FILE(snode) (snode->type & SNODE_TYPE_FILE) +#define SNODE_CHECK_TYPE_DIR(snode) (snode->type & SNODE_TYPE_DIR) + +typedef struct { + storfs_page_t start; + storfs_page_t count; +} SNodeExtent; + +typedef struct { + uint64_t modified_time; + uint64_t size; + storfs_page_t extent_idx; + SNodeExtent direct[DIRECT_EXTENT_SIZE]; + struct { + storfs_page_t single; + storfs_page_t multiple; + } indirect; + uint16_t crc; + uint8_t type; + uint8_t flags; + uint8_t reserved[SNODE_RESERVED_SIZE]; + uint8_t name[STORFS_MAX_FILE_NAME]; +} SNode; + +typedef struct { + uint32_t offset_bytes; + uint32_t idx; +} SNodeExtentCache; + +typedef struct { + SNode node; + storfs_page_t page; + SNodeExtentCache read; + SNodeExtentCache write; +} SNodeInst; + +_Static_assert(sizeof(SNode) == SNODE_TOTAL_SIZE, + "Snode structure is not equivalent to expected size"); + +storfs_err_t +snode_create(storfs_t *fs, const char *name, storfs_page_t *page, uint8_t type); +storfs_err_t snode_lookup(storfs_t *fs, storfs_page_t page, SNodeInst *inst); +storfs_err_t +snode_find_read_location(storfs_t *fs, SNodeInst *inst, storfs_byte_t offset); +storfs_err_t snode_find_write_location(storfs_t *fs, SNodeInst *inst); +storfs_err_t snode_write_data(storfs_t *fs, + SNodeInst *inst, + const uint8_t *data, + uint32_t size); +storfs_err_t +snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size); +storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t size); +#endif diff --git a/test/test_snode.c b/test/test_snode.c new file mode 100644 index 0000000..b6f823d --- /dev/null +++ b/test/test_snode.c @@ -0,0 +1,211 @@ +#include "atomic.h" +#include "bitmap.h" +#include "common.h" +#include "fake_flash.h" +#include "helper_randomizer.h" +#include "mock_crc.h" +#include "snode.h" +#include "unity.h" + +#include +#include +#include +#include +#include + +#define SIZE_SNODE_COMPARE fake_storfs_get_page_size() +#define NUM_SNODE_COMPARE (SIZE_SNODE_COMPARE / sizeof(SNode)) +#define FAKE_CRC16 (0x123A) +#define FAKE_NAME "STORFS_FILE_TEST_" +#define MEMBER_SIZE(type, member) (sizeof(((type *)0)->member)) +#define DIRECT_ARRAY_SIZE (MEMBER_SIZE(SNode, direct) / sizeof(SNodeExtent)) + +#define DATA_PAGES_PER_INDIRECT_PAGE(f) ((f->pageSize / sizeof(SNodeExtent))) +#define SINGLE_INDIRECT_DATA_CALC(f) \ + (f->pageSize * DATA_PAGES_PER_INDIRECT_PAGE(f)) + +#define PAGE_DATA_SIZE(f) (f->pageSize - sizeof(SNode)) +#define DIRECT_DATA_SIZE(f) \ + (PAGE_DATA_SIZE(f) + (f->pageSize * DIRECT_ARRAY_SIZE)) +#define SINGLE_INDIRECT_DATA_SIZE(f) \ + (DIRECT_DATA_SIZE(f) + SINGLE_INDIRECT_DATA_CALC(f)) +#define MULTIPLE_INDIRECT_DATA_SIZE(f) \ + (SINGLE_INDIRECT_DATA_SIZE(f) + \ + (SINGLE_INDIRECT_DATA_CALC(f) * DATA_PAGES_PER_INDIRECT_PAGE(f))) + +static storfs_t *fs; + +void setUp(void) { + fs = fake_storfs_init(); + TEST_ASSERT_EQUAL(bitmap_create(fs), STORFS_OK); +} + +void tearDown(void) {} + +void test_snode_create(void) { + SNode snode = { 0 }; + char name[STORFS_MAX_FILE_NAME] = { 0 }; + storfs_page_t page; + + // Create nodes until filled up filesystem + for(size_t i = 17; i < fs->pageCount; i++) { + page = i; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + snprintf(name, STORFS_MAX_FILE_NAME, FAKE_NAME "%lu", i); + + storfs_page_t read_page; + TEST_ASSERT_EQUAL(snode_create(fs, name, &read_page, SNODE_TYPE_FILE), + STORFS_OK); + TEST_ASSERT_EQUAL(read_page, page); + bitmap_alloc_page(fs, read_page, PAGE_FREE); + } + + // Test creating a node out of bounds + // TEST_ASSERT_EQUAL(snode_create(fs, name, &page), + // STORFS_ERR_NO_FREE_BLOCKS); + + // Test read failure + fake_storfs_fail_op(READ, true, 1); + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_READ_FAILED); + fake_storfs_fail_op(READ, false, 0); + + // Test write failure + fake_storfs_fail_op(WRITE, true, 1); + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_WRITE_FAILED); + fake_storfs_fail_op(WRITE, false, 0); + fake_storfs_fail_op(WRITE, true, 2); + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_WRITE_FAILED); + fake_storfs_fail_op(WRITE, false, 0); + fake_storfs_fail_op(WRITE, true, 3); + storfs_crc16_ExpectAnyArgsAndReturn(FAKE_CRC16); + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_WRITE_FAILED); + fake_storfs_fail_op(WRITE, false, 0); + + // Test improper input + // TEST_ASSERT_EQUAL(snode_create(NULL, name, &page), + // STORFS_ERR_NULL_POINTER); TEST_ASSERT_EQUAL(snode_create(fs, NULL, &page), + // STORFS_ERR_NULL_POINTER); TEST_ASSERT_EQUAL(snode_create(fs, name, NULL), + // STORFS_ERR_NULL_POINTER); +} + +void test_snode_write(void) { + const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; + storfs_page_t page = 17; + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + SNodeInst inst = { 0 }; + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + // Write all the data from the file, read it and compare + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), + STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.node.size, 0); + free(write_buf); + free(read_buf); +} + +// void test_snode_write_read(void) { +// const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); +// uint8_t *write_buf = random_array(buf_size); +// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page), STORFS_OK); +// +// // Write all the data from the file, read it and compare +// TEST_ASSERT_EQUAL(snode_write_data(fs, page, write_buf, buf_size), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_read_data(fs, page, 0, read_buf, +// buf_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); +// +// // Chunk read it to check offset paramter +// uint32_t chunk_size = fs->pageSize * 3; +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t read_remain = buf_size - i; +// uint32_t read_size = MIN(read_remain, chunk_size); +// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, read_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); +// } +// +// // Test reading past boundaries +// TEST_ASSERT_EQUAL(snode_read_data(fs, page, buf_size, read_buf, 1), +// STORFS_ERR_INVALID_PARAM); +// +// random_array_free(write_buf); +// free(read_buf); +// } +// +void test_snode_write_read_alternate(void) { + const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); + storfs_page_t snode_1_page = 17; + storfs_page_t snode_2_page = snode_1_page++; + char snode_1_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; + char snode_2_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL( + snode_create(fs, snode_1_name, &snode_1_page, SNODE_TYPE_FILE), + STORFS_OK); + TEST_ASSERT_EQUAL( + snode_create(fs, snode_2_name, &snode_2_page, SNODE_TYPE_FILE), + STORFS_OK); + + // Write all the data from the file, read it and compare + SNodeInst inst_1 = { 0 }; + SNodeInst inst_2 = { 0 }; + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_1), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_1, 0), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_2), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_2, 0), STORFS_OK); + + // Chunk read it to check offset paramter + uint32_t chunk_size = 693; + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t read_remain = buf_size - i; + uint32_t read_size = MIN(read_remain, chunk_size); + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], read_size), + STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, read_size), + STORFS_OK); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], read_size), + STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, read_size), + STORFS_OK); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); + } + TEST_ASSERT_EQUAL(inst_1.node.size, buf_size); + TEST_ASSERT_EQUAL(inst_2.node.size, buf_size); + + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_1, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst_1.node.size, 0); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst_2.node.size, 0); + + random_array_free(write_buf); + free(read_buf); +} From 4df77250c8256f0a5f6c980d8a714bfe3b5f5a79 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 29 Mar 2026 20:37:26 -0700 Subject: [PATCH 02/21] Add some function comments --- src/snode.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/snode.c b/src/snode.c index 1ea2566..7c388a8 100644 --- a/src/snode.c +++ b/src/snode.c @@ -51,6 +51,20 @@ snode_update(storfs_t *fs, SNode *node, storfs_page_t page) { return atomic_write(fs, page); } +/*! + @brief Create an snode + + @details This function will allocate a page for a new snode + + @param fs pointer to the filesystem instance + @param name snode name + @param page page which has been allocated to the snode + @param size type of snode: + SNODE_TYPE_FILE + SNODE_TYPE_DIR + + @return storfs_err_t + */ storfs_err_t snode_create(storfs_t *fs, const char *name, storfs_page_t *page, @@ -72,6 +86,24 @@ storfs_err_t snode_create(storfs_t *fs, return snode_update(fs, &node, *page); } +/*! + @brief Lookup an snode based on it's input page + + @details Find + Find + Find + + @param fs pointer to the filesystem instance + @param inst pointer to snode instance + @param data data to read from the snode + @param size size of data to read from the snode + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_READ_FAILED if reading from the filesystem fails + STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails + STORFS_ERR_WRITE_FAILED if writing from the filesystem fails + */ storfs_err_t snode_lookup(storfs_t *fs, storfs_page_t page, SNodeInst *inst) { if(!fs || !inst) { return STORFS_ERR_NULL_POINTER; @@ -722,6 +754,24 @@ static storfs_err_t snode_perform_op(storfs_t *fs, return err; } +/*! + @brief Write data to an snode instance + + @details Must call @link snode_find_write_location @endlink before writing to + the snode. This function will append data to the end of the snode. + The cache write location will be updated with each write. + + @param fs pointer to the filesystem instance + @param inst pointer to snode instance + @param data data to write to the snode + @param size size of data to write to the snode + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_READ_FAILED if reading from the filesystem fails + STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails + STORFS_ERR_WRITE_FAILED if writing from the filesystem fails + */ storfs_err_t snode_write_data(storfs_t *fs, SNodeInst *inst, const uint8_t *data, @@ -731,6 +781,24 @@ storfs_err_t snode_write_data(storfs_t *fs, return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); } +/*! + @brief Read data from an snode instance + + @details Must call @link snode_find_read_location @endlink before reading from + an snode. This function will begin reading from the offset indicated + in @link snode_find_read_location @endlink. + + @param fs pointer to the filesystem instance + @param inst pointer to snode instance + @param data data to read from the snode + @param size size of data to read from the snode + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_READ_FAILED if reading from the filesystem fails + STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails + STORFS_ERR_WRITE_FAILED if writing from the filesystem fails + */ storfs_err_t snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = size }; @@ -738,6 +806,24 @@ snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); } +/*! + @brief Erase data from an snode instance + + @details Must call @link snode_find_write_location @endlink before erasing + data from an snode. This function will erase data from the end + of the snode. The cache write location will be updated with each + erase. + + @param fs pointer to the filesystem instance + @param inst pointer to snode instance + @param size size of data to erase from the snode + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_READ_FAILED if reading from the filesystem fails + STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails + STORFS_ERR_WRITE_FAILED if writing from the filesystem fails + */ storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t size) { SNodeOpInst op = { .op = SNODE_ERASE, .bytes_remaining = size }; From 7a96fae290e3a87304dc8d3404eaaa20b59e9d31 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 29 Mar 2026 21:46:39 -0700 Subject: [PATCH 03/21] Add documentation and more function comments --- docs/SNODE.md | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/snode.c | 49 +++++++++++++------- 2 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 docs/SNODE.md diff --git a/docs/SNODE.md b/docs/SNODE.md new file mode 100644 index 0000000..8a04784 --- /dev/null +++ b/docs/SNODE.md @@ -0,0 +1,121 @@ +# STORfs SNode + +In STORfs, +SNodes are inspired by the inode (index node) from Unix-style file systems. +The SNode is responsible for storing information about the file or directory of concern. +This includes: + +- Last modified time +- Parent SNode +- Size of the data held in the SNode +- CRC of the SNode structure +- Flags +- File Name + +SNodes are saved to pages in the filesystem and are exactly 128 bytes. +The rest of the page it is saved to is reserved for inline file or directory data. +The following calculates the amount of inline data following the SNode: + +```math +s = p_s - 128 +``` + +Where $s$ is the size of the data and $p_s$ is the filesystem's page size. +All other data for the associated SNode is stored elsewhere on the file system. +This is done through a concept of page extents, +which point to the other locations on the file system. + +## Extents + +SNode extents represent a contiguous area of storage reserved for a file. +Extent information is stored in the following format: + +```c +typedef struct { + storfs_page_t start; + storfs_page_t count; +} SNodeExtent; +``` + +Where start represents the beginning page location of the extent, +and count represents the number of contiguous pages after. +A visual example of this in the filesystem with `start = 100` and `count = 5` would look like: + +``` +start = 100 + | + | ++------------------------------------------------------+ +| Page 100 | Page 101 | Page 102 | Page 103 | Page 104 | ++------------------------------------------------------+ + | | + | | + +-------------------------------------------+ + | + | + count = 5 +``` + +The extent starts on page **100** and extends for **5** pages to page **104**. + +Within each SNode there are three layers of extents storage. +These are described in the subsequent sections. + +### Direct + +These extents exist as an array of `DIRECT_EXTENT_SIZE` elements directly in the SNode data structure. +When finding a file's location, +the direct extents are checked first. +This is the location in which the fastest look up times are available. +Keeping files within the direct extents can greatly speed up initialy accessing an SNode (this process will be explained in SNode [operations](#operations)). + +### Single Indirect + +The single indirect parameter in the SNode entry points to a page consisting fully of extent entries. +This page of extents can be visually represented as follows: + +``` ++----------+ +| Extent 1 | ++----------+ +| Extent 2 | ++----------+ +| Extent 3 | ++----------+ +| Extent 4 | ++----------+ +| Extent 5 | ++----------+ +| Extent 6 | ++----------+ +| Extent 7 | ++----------+ +| ... | +| Extent N | ++----------+ +``` + +Indirect pages have the following number of extents: + +```math +N = p_s / s_e +``` + +Where $N$ is the number of extents, +$p_s$ is the page size, +and $s_e$ is the size of an extent entry, `sizeof(SNodeExtent)`. + +### Multiple Extents + +The final layer to the extents are the multiple extents. +Each individual + + +## Operations + +Four operations are available for an SNode: + +- locate +- read +- write +- erase diff --git a/src/snode.c b/src/snode.c index 7c388a8..3e1d450 100644 --- a/src/snode.c +++ b/src/snode.c @@ -54,7 +54,10 @@ snode_update(storfs_t *fs, SNode *node, storfs_page_t page) { /*! @brief Create an snode - @details This function will allocate a page for a new snode + @details This function will allocate a page for a new snode and save it. + The snode will be available again through the @link + snode_lookup @endlink function. When the snode is saved, a crc + is calculated just on the data structure. @param fs pointer to the filesystem instance @param name snode name @@ -87,22 +90,19 @@ storfs_err_t snode_create(storfs_t *fs, } /*! - @brief Lookup an snode based on it's input page + @brief Lookup an snode based on a page location - @details Find - Find - Find + @details Finds information about an snode. Will validate the crc matches what + is expected in order to validate the contents of the snode. @param fs pointer to the filesystem instance + @param page page to obtain snode information @param inst pointer to snode instance - @param data data to read from the snode - @param size size of data to read from the snode @return STORFS_OK on success STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_CRC_MISMATCH if crc calculation fails STORFS_ERR_READ_FAILED if reading from the filesystem fails - STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails - STORFS_ERR_WRITE_FAILED if writing from the filesystem fails */ storfs_err_t snode_lookup(storfs_t *fs, storfs_page_t page, SNodeInst *inst) { if(!fs || !inst) { @@ -135,7 +135,7 @@ static storfs_err_t find_page_in_extents(SNodeExtentCache *cache, const SNodeExtent *extents, uint32_t count, uint32_t *logical_page) { - // If extents[i] == 0, the extent is empty + // If extents[i] == {0}, the extent is empty for(uint32_t i = 0; i < count && extents[i].count > 0; i++) { if(*logical_page < extents[i].count) { return STORFS_OK; @@ -207,7 +207,7 @@ static storfs_err_t find_double_indirect_location(storfs_t *fs, static void find_update_cache(storfs_t *fs, SNodeExtentCache *cache, storfs_loc_t logical) { cache->offset_bytes = logical.pageLoc * fs->pageSize + logical.byteLoc; - // Increment here as index 0 is the inline data + // Increment here to account for index 0 being the snode inline data cache->idx++; } @@ -222,17 +222,19 @@ static storfs_err_t find_location(storfs_t *fs, return STORFS_OK; } - // Find the logical page offset, how many pages would the data consume + // Find the logical page offset, how many pages would all the data consume // in a single contiguous block const uint32_t snode_inline_data_size = INLINE_DATA_SIZE(fs); - // logical.pageLoc is decremented throughout these operations if non zero, - // this will be the total offset in bytes from the extent start location + // The logical page location is decremented throughout these operations if + // it is non-zero, this will be the total offset in bytes from the extent's + // starting location storfs_loc_t logical; uint32_t data_beyond_snode = offset - snode_inline_data_size; logical.pageLoc = data_beyond_snode / fs->pageSize; logical.byteLoc = data_beyond_snode % fs->pageSize; + // Determine if location is in direct extents uint32_t extent_count = ARRAY_SIZE(inst->node.direct); storfs_err_t err = find_page_in_extents(cache, inst->node.direct, @@ -267,6 +269,23 @@ static storfs_err_t find_location(storfs_t *fs, return err; } +/*! + @brief Find the read location extent index based on an offset byte location + + @details Will find the location to begin reading an snode from. This must be + invoked before an snode is initially read. Will update the read cache + when STORFS_OK or STORFS_ERR_NOT_FOUND is returned. + + @param fs pointer to the filesystem instance + @param page page to obtain snode information + @param inst pointer to snode instance + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_NOT_FOUND could not find the location offset + STORFS_ERR_CRC_MISMATCH if crc calculation fails + STORFS_ERR_READ_FAILED if reading from the filesystem fails + */ storfs_err_t snode_find_read_location(storfs_t *fs, SNodeInst *inst, storfs_byte_t offset) { if(!fs || !inst) { @@ -715,7 +734,7 @@ static storfs_err_t snode_perform_op(storfs_t *fs, cache = &inst->read; } - // Do not erase past end of file + // Do not erase past end of an snode if(op->op == SNODE_ERASE && size > inst->node.size) { return STORFS_ERR_INVALID_PARAM; } From e6a859d415f93b3e36265a348a4709289fb8a05f Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 20 Jun 2026 20:24:58 -0700 Subject: [PATCH 04/21] Begin docs and update typedef structs name --- docs/SNODE.md | 49 ++++++++++++++++++++++++++++++++++++++++--------- src/snode.c | 31 +++++++++++++++++++------------ 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/SNODE.md b/docs/SNODE.md index 8a04784..3704028 100644 --- a/docs/SNODE.md +++ b/docs/SNODE.md @@ -76,19 +76,19 @@ This page of extents can be visually represented as follows: ``` +----------+ -| Extent 1 | +| Extent 1 | ---> start = 100, count = 5 +----------+ -| Extent 2 | +| Extent 2 | ---> start = 140, count = 12 +----------+ -| Extent 3 | +| Extent 3 | ---> start = 409, count = 2 +----------+ -| Extent 4 | +| Extent 4 | ---> start = 1030, count = 136 +----------+ -| Extent 5 | +| Extent 5 | ---> start = 1117, count = 23 +----------+ -| Extent 6 | +| Extent 6 | ---> start = 2020, count = 1 +----------+ -| Extent 7 | +| Extent 7 | ---> start = 2023, count = 30 +----------+ | ... | | Extent N | @@ -107,8 +107,39 @@ and $s_e$ is the size of an extent entry, `sizeof(SNodeExtent)`. ### Multiple Extents -The final layer to the extents are the multiple extents. -Each individual +The final layer of extents are multiple extents. +Each entry in the multiple extent pages has the following contents: + +```c +typedef struct { + storfs_page_t single_location; + storfs_page_t total; +} SNodeMultiple; +``` + +`single_location` contains the location of a single indirect extent page +and `total` indicates the total number of pages allocated in the indirect page. +The following provides an example of multiple extents: + +``` ++------------+ +| Multiple 1 | ---> +----------+ ++------------+ | Extent 1 | ---> start = 100, count = 5 + +----------+ + | Extent 2 | ---> start = 140, count = 12 + +----------+ + | Extent 3 | ---> start = 409, count = 2 + +----------+ + | Extent 4 | ---> start = 1030, count = 136 + +----------+ + | Extent 5 | ---> start = 1117, count = 23 + +----------+ + | Extent 6 | ---> start = 2020, count = 1 + +----------+ + | Extent 7 | ---> start = 2023, count = 30 + +----------+ +``` + ## Operations diff --git a/src/snode.c b/src/snode.c index 3e1d450..9fe58af 100644 --- a/src/snode.c +++ b/src/snode.c @@ -23,6 +23,11 @@ typedef struct { uint32_t bytes_remaining; } SNodeOpInst; +typedef struct { + storfs_page_t single_location; + storfs_page_t total; +} SNodeMultiple; + static storfs_err_t snode_alloc_new_page(storfs_t *fs, storfs_page_t *page) { storfs_err_t err = bitmap_alloc(fs, page); if(err != STORFS_OK) { @@ -384,6 +389,7 @@ static storfs_err_t process_extent_pages(storfs_t *fs, } if(op->op == SNODE_ERASE && !indirect_idx && !indirect_extent->start) { + // If this is the first indirect index and it is empty *indirect_page = 0; } @@ -484,9 +490,9 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - SNodeExtent *multiple_extents = (SNodeExtent *)fs->working_buf; - storfs_page_t single_indirect_page = - multiple_extents[single_indirect_page_idx].start; + SNodeMultiple *multiple_extents = (SNodeMultiple *)fs->working_buf; + storfs_page_t single_indirect_page = + multiple_extents[single_indirect_page_idx].single_location; if(op->op == SNODE_WRITE && !single_indirect_page) { err = snode_alloc_new_page(fs, &single_indirect_page); if(err != STORFS_OK) { @@ -499,8 +505,9 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - multiple_extents = (SNodeExtent *)fs->working_buf; - multiple_extents[single_indirect_page_idx].start = single_indirect_page; + multiple_extents = (SNodeMultiple *)fs->working_buf; + multiple_extents[single_indirect_page_idx].single_location = + single_indirect_page; err = atomic_write(fs, node->indirect.multiple); if(err != STORFS_OK) { @@ -525,14 +532,14 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - multiple_extents = (SNodeExtent *)fs->working_buf; - SNodeExtent *multiple_extent = &multiple_extents[single_indirect_page_idx]; + multiple_extents = (SNodeMultiple *)fs->working_buf; + SNodeMultiple *multiple = &multiple_extents[single_indirect_page_idx]; if(op->op == SNODE_WRITE) { - multiple_extent->count += op->extent.count; + multiple->total += op->extent.count; } else { - multiple_extent->count -= calculate_freed(fs, &op->extent, op); - if(!multiple_extent->count) { - multiple_extent->start = 0; + multiple->total -= calculate_freed(fs, &op->extent, op); + if(!multiple->total) { + multiple->single_location = 0; } } @@ -550,7 +557,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, } bool empty_first_extent = - !single_indirect_page_idx && !multiple_extent->start; + !single_indirect_page_idx && !multiple->single_location; if(op->op == SNODE_ERASE && empty_first_extent) { storfs_page_t init_multiple_indirect = node->indirect.multiple; node->indirect.multiple = 0; From e51e4052f1d84bd545419ecd1c118106d869826b Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 21 Jun 2026 00:54:02 -0700 Subject: [PATCH 05/21] Update docs and tests, fix some bugs --- docs/SNODE.md | 147 ++++++++++++++++++++++++++++++++++++++++------ src/snode.c | 51 ++++++++++++---- test/test_snode.c | 119 ++++++++++++++++++++++++++++++------- 3 files changed, 270 insertions(+), 47 deletions(-) diff --git a/docs/SNODE.md b/docs/SNODE.md index 3704028..e0dc77e 100644 --- a/docs/SNODE.md +++ b/docs/SNODE.md @@ -22,12 +22,12 @@ s = p_s - 128 Where $s$ is the size of the data and $p_s$ is the filesystem's page size. All other data for the associated SNode is stored elsewhere on the file system. -This is done through a concept of page extents, -which point to the other locations on the file system. +This is done through a concept called page extents, +which point to the other data locations on the file system. ## Extents -SNode extents represent a contiguous area of storage reserved for a file. +SNode extents represent a contiguous area of storage reserved for SNode data. Extent information is stored in the following format: ```c @@ -38,8 +38,8 @@ typedef struct { ``` Where start represents the beginning page location of the extent, -and count represents the number of contiguous pages after. -A visual example of this in the filesystem with `start = 100` and `count = 5` would look like: +and count represents the total number of contiguous pages in the extent. +A visual example of this with `start = 100` and `count = 5` would look like: ``` start = 100 @@ -58,21 +58,21 @@ start = 100 The extent starts on page **100** and extends for **5** pages to page **104**. -Within each SNode there are three layers of extents storage. +Within each SNode there are three layers of extent storage. These are described in the subsequent sections. ### Direct These extents exist as an array of `DIRECT_EXTENT_SIZE` elements directly in the SNode data structure. -When finding a file's location, +When finding an SNode's data location, the direct extents are checked first. -This is the location in which the fastest look up times are available. -Keeping files within the direct extents can greatly speed up initialy accessing an SNode (this process will be explained in SNode [operations](#operations)). +This is the location in which the fastest lookup times are available: **O(1)**. +Keeping files within the direct extents can greatly speed up initially accessing an SNode (this process will be explained in SNode [operations](#operations)). ### Single Indirect The single indirect parameter in the SNode entry points to a page consisting fully of extent entries. -This page of extents can be visually represented as follows: +A single indirect extent page can be visually represented as follows: ``` +----------+ @@ -118,8 +118,18 @@ typedef struct { ``` `single_location` contains the location of a single indirect extent page -and `total` indicates the total number of pages allocated in the indirect page. -The following provides an example of multiple extents: +and `total` indicates the absolute number of pages allocated in that indirect page. +`total` can be calculated as: + +```math +t = \sum_{i=1}^{N} c_i +``` + +Where $t$ is the total number of pages allocated in the single indirect extent page. +$N$ is the number of extents in the single indirect extent page. +$c_i$ is the count of the $i$th extent in the single indirect extent page. + +A multiple extent page can be visually represented as follows: ``` +------------+ @@ -140,13 +150,116 @@ The following provides an example of multiple extents: +----------+ ``` +In the above example, +`total` of `Multiple 1` would be `209`. + +The number of multiple entries in a page can be calculated as: + +```math +N = p_s / s_e +``` + +Where $N$ is the number of multiple extent entries, +$p_s$ is the page size, +and $s_e$ is the size of a multiple entry, `sizeof(SNodeMultiple)`. + +The total number of extent entries, +`SNodeExtent`, +which can be held in the multiple extent page can be calculated as: + + +```math +N = N_m * N_s +``` + +Where $N$ is the total number of extent entries, +$N_m$ is the number of entries in the multiple extent page, +and $N_s$ is the number of entries in a single extent page. ## Operations -Four operations are available for an SNode: +Five operations are available on an SNode after it has been created: + +| Operation | Complexity | +|-----------|------------| +| Lookup | O(1) | +| Locate | O(n) | +| Read | O(1) | +| Write | O(1) | +| Erase | O(1) | + +Where *n* is the number of extent entries traversed to find the target position. + +### Lookup + +Function declaration: + +`snode_lookup` + +If the SNode has been created, +it must be looked up before it can be used. +The lookup functionality finds the SNode by page number, +which must be found by another means before calling this API. + +### Locate + +Function declarations: + +- `snode_find_read_location` +- `snode_find_write_location` + +The locate operation must be done before performing a read or write/erase. + +If an SNode is opened to be read, +the find read location function must be invoked. +This function takes an offset parameter to allow reading from a desired location in the SNode. + +If an SNode is opened to be written to or data is to be erased, +the find write location must be invoked. +This function finds the end of the SNode's data, +due to the following reasons: + +- All write operations append to an SNode's data +- All erase operations erase from the end of an SNode + +All locate functionality has an **O(n)** time complexity, +where *n* is the number of extent entries traversed. +This has the lowest performance of the operations performed on an SNode, +but it only needs to be run once on an SNode. + +### Read + +Function declaration: + +`snode_read_data` + +Data is read starting from the locate position. +As data is read, +a data pointer is used to keep track of the location in the SNode's data. +This means that the SNode's data can be read in chunks. +Data is read from the SNode and stored in a buffer passed in from the caller. +All read functionality has an **O(1)** time complexity. + +### Write + +Function declaration: + +`snode_write_data` + +Data is always written to the end of an SNode. +As data is written, +an SNode data pointer is used to keep track of where the end of the SNode is. +All write functionality has an **O(1)** time complexity. + +### Erase + +Function declaration: + +`snode_erase_data` -- locate -- read -- write -- erase +Data is always erased from the end of an SNode. +As data is erased, +an SNode data pointer is used to keep track of where the end of the SNode data is. +Write and erase operations can be used in conjunction with one another. +All erase functionality has an **O(1)** time complexity. diff --git a/src/snode.c b/src/snode.c index 9fe58af..0a7f5da 100644 --- a/src/snode.c +++ b/src/snode.c @@ -8,6 +8,7 @@ #include #define INLINE_DATA_SIZE(f) (f->pageSize - sizeof(SNode)) +#define NUM_MULTIPLE_EXTENTS(f) ((f->pageSize / sizeof(SNodeMultiple))) #define EXTENTS_PER_PAGE(f) ((f->pageSize / sizeof(SNodeExtent))) #define CALC_CONTIGUOUS_MAX(f, o) CEIL_DIV(o->bytes_remaining, f->pageSize) @@ -180,22 +181,22 @@ static storfs_err_t find_double_indirect_location(storfs_t *fs, return err; } - SNodeExtent *extents = (SNodeExtent *)fs->working_buf; - storfs_page_t single_indirect_location = 0; + SNodeMultiple *multiple = (SNodeMultiple *)fs->working_buf; + storfs_page_t single_indirect_location = 0; // Determine which indirect extent page holds the desired logical location - for(uint32_t i = 0; i < EXTENTS_PER_PAGE(fs); i++) { - if(!extents[i].count) { + for(uint32_t i = 0; i < NUM_MULTIPLE_EXTENTS(fs); i++) { + if(!multiple[i].total) { break; } - if(*logical_page < extents[i].count) { - single_indirect_location = extents[i].start; + if(*logical_page < multiple[i].total) { + single_indirect_location = multiple[i].single_location; break; } cache->idx += EXTENTS_PER_PAGE(fs); - *logical_page -= extents[i].count; + *logical_page -= multiple[i].total; } if(!single_indirect_location) { @@ -206,6 +207,11 @@ static storfs_err_t find_double_indirect_location(storfs_t *fs, cache, single_indirect_location, logical_page); + // The location is at the end of the data + if(err == STORFS_ERR_NOT_FOUND) { + err = STORFS_ERR_NO_SPACE; + } + return err; } @@ -266,7 +272,7 @@ static storfs_err_t find_location(storfs_t *fs, } err = find_double_indirect_location(fs, inst, cache, &logical.pageLoc); - if(err != STORFS_OK && err != STORFS_ERR_NOT_FOUND) { + if(err != STORFS_OK && err != STORFS_ERR_NO_SPACE) { return err; } @@ -279,15 +285,18 @@ static storfs_err_t find_location(storfs_t *fs, @details Will find the location to begin reading an snode from. This must be invoked before an snode is initially read. Will update the read cache - when STORFS_OK or STORFS_ERR_NOT_FOUND is returned. + when STORFS_OK, STORFS_ERR_NOT_FOUND or STORFS_ERR_NO_SPACE is + returned. @param fs pointer to the filesystem instance @param page page to obtain snode information @param inst pointer to snode instance + @param offset @return STORFS_OK on success STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments STORFS_ERR_NOT_FOUND could not find the location offset + STORFS_ERR_NO_SPACE there is no more data to read from the file STORFS_ERR_CRC_MISMATCH if crc calculation fails STORFS_ERR_READ_FAILED if reading from the filesystem fails */ @@ -300,6 +309,25 @@ snode_find_read_location(storfs_t *fs, SNodeInst *inst, storfs_byte_t offset) { return find_location(fs, inst, &inst->read, offset); } +/*! + @brief Find the write location extent index based + + @details Will find the location to begin writing to or erasing from an snode. + This must be before an snode is initially written to or erased from. + Will update the write cache when STORFS_OK, STORFS_ERR_NOT_FOUND or + STORFS_ERR_NO_SPACE is returned. + + @param fs pointer to the filesystem instance + @param page page to obtain snode information + @param inst pointer to snode instance + + @return STORFS_OK on success + STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments + STORFS_ERR_NO_SPACE there is no more data to write to the file + STORFS_ERR_NOT_FOUND could not find the location offset + STORFS_ERR_CRC_MISMATCH if crc calculation fails + STORFS_ERR_READ_FAILED if reading from the filesystem fails + */ storfs_err_t snode_find_write_location(storfs_t *fs, SNodeInst *inst) { if(!fs || !inst) { return STORFS_ERR_NULL_POINTER; @@ -526,7 +554,10 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - if(op->op != SNODE_READ) { + // Only update multiple if it is on a byte boundary + bool is_boundary = cache.offset_bytes % fs->pageSize == 0; + + if(op->op != SNODE_READ && is_boundary) { err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; diff --git a/test/test_snode.c b/test/test_snode.c index b6f823d..8063b25 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -48,7 +48,8 @@ void test_snode_create(void) { storfs_page_t page; // Create nodes until filled up filesystem - for(size_t i = 17; i < fs->pageCount; i++) { + storfs_page_t data_begin = fs->bitmap.hint; + for(size_t i = data_begin; i < fs->pageCount; i++) { page = i; storfs_crc16_IgnoreAndReturn(FAKE_CRC16); @@ -59,12 +60,17 @@ void test_snode_create(void) { TEST_ASSERT_EQUAL(snode_create(fs, name, &read_page, SNODE_TYPE_FILE), STORFS_OK); TEST_ASSERT_EQUAL(read_page, page); - bitmap_alloc_page(fs, read_page, PAGE_FREE); } // Test creating a node out of bounds - // TEST_ASSERT_EQUAL(snode_create(fs, name, &page), - // STORFS_ERR_NO_FREE_BLOCKS); + page++; + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_NO_FREE_BLOCKS); + + // Free all pages + for(size_t i = data_begin; i < fs->pageCount + 1000; i++) { + bitmap_alloc_page(fs, i, PAGE_FREE); + } // Test read failure fake_storfs_fail_op(READ, true, 1); @@ -88,10 +94,12 @@ void test_snode_create(void) { fake_storfs_fail_op(WRITE, false, 0); // Test improper input - // TEST_ASSERT_EQUAL(snode_create(NULL, name, &page), - // STORFS_ERR_NULL_POINTER); TEST_ASSERT_EQUAL(snode_create(fs, NULL, &page), - // STORFS_ERR_NULL_POINTER); TEST_ASSERT_EQUAL(snode_create(fs, name, NULL), - // STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_create(NULL, name, &page, SNODE_TYPE_FILE), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_create(fs, NULL, &page, SNODE_TYPE_FILE), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_create(fs, name, NULL, SNODE_TYPE_FILE), + STORFS_ERR_NULL_POINTER); } void test_snode_write(void) { @@ -141,11 +149,11 @@ void test_snode_write(void) { // // Chunk read it to check offset paramter // uint32_t chunk_size = fs->pageSize * 3; // for(uint32_t i = 0; i < buf_size; i += chunk_size) { -// uint32_t read_remain = buf_size - i; -// uint32_t read_size = MIN(read_remain, chunk_size); -// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, read_size), +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, data_size), // STORFS_OK); -// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); // } // // // Test reading past boundaries @@ -156,6 +164,75 @@ void test_snode_write(void) { // free(read_buf); // } // + +void test_snode_find_location(void) { + const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), + STORFS_OK); + + // Test obtaining the location now, will be in indirect extents which has yet + // to be created + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NOT_FOUND); + + // Erase and then fill the SNode all the way up + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.node.size, 0); + + // Fill up an snode to the maximum size of data + uint32_t chunk_size = 1024; + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, &write_buf[i], data_size), + STORFS_OK); + + // Test obtaining data from the last written location + data_remain -= data_size; + if(data_remain) { + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, data_size), + STORFS_OK); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + } + } + + // Test obtaining the location now, will be at the very end of the SNode + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NO_SPACE); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size), + STORFS_ERR_NO_SPACE); + + // Test getting the data almost at the end of the file + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), + STORFS_OK); + + random_array_free(write_buf); + free(read_buf); + + // Test improper parameters + TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), + STORFS_ERR_NULL_POINTER); +} + void test_snode_write_read_alternate(void) { const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); uint8_t *write_buf = random_array(buf_size); @@ -185,18 +262,18 @@ void test_snode_write_read_alternate(void) { // Chunk read it to check offset paramter uint32_t chunk_size = 693; for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t read_remain = buf_size - i; - uint32_t read_size = MIN(read_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], read_size), + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], data_size), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, read_size), + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, data_size), STORFS_OK); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], read_size), + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], data_size), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, read_size), + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, data_size), STORFS_OK); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, read_size), 0); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } TEST_ASSERT_EQUAL(inst_1.node.size, buf_size); TEST_ASSERT_EQUAL(inst_2.node.size, buf_size); @@ -206,6 +283,8 @@ void test_snode_write_read_alternate(void) { TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst_2.node.size, 0); + // TODO test read here + random_array_free(write_buf); free(read_buf); } From fec5378f21f966d31c6bfc8bb23de3bc82ed8f80 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 21 Jun 2026 18:36:09 -0700 Subject: [PATCH 06/21] Fix write extent bug, not properly updating extents when not on a zero cross boundary --- src/snode.c | 44 ++++++++++++++++++++++++++++++-------------- test/test_snode.c | 47 +++++++++++++++++++++++++++++++---------------- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/snode.c b/src/snode.c index 0a7f5da..9446932 100644 --- a/src/snode.c +++ b/src/snode.c @@ -22,6 +22,7 @@ typedef struct { SNodeExtent extent; // Allocated extent on write, filled from flash otherwise SNodeOp op; uint32_t bytes_remaining; + bool is_boundary; } SNodeOpInst; typedef struct { @@ -168,10 +169,11 @@ static storfs_err_t find_extent_in_indirect_page(storfs_t *fs, return find_page_in_extents(cache, extents, extent_count, logical_page); } -static storfs_err_t find_double_indirect_location(storfs_t *fs, - SNodeInst *inst, - SNodeExtentCache *cache, - storfs_page_t *logical_page) { +static storfs_err_t +find_multiple_indirect_location(storfs_t *fs, + SNodeInst *inst, + SNodeExtentCache *cache, + storfs_page_t *logical_page) { if(!inst->node.indirect.multiple) { return STORFS_ERR_NOT_FOUND; } @@ -200,6 +202,11 @@ static storfs_err_t find_double_indirect_location(storfs_t *fs, } if(!single_indirect_location) { + // The location is at the end of the data + if(*logical_page == 0) { + return STORFS_ERR_NO_SPACE; + } + return STORFS_ERR_NOT_FOUND; } @@ -207,10 +214,6 @@ static storfs_err_t find_double_indirect_location(storfs_t *fs, cache, single_indirect_location, logical_page); - // The location is at the end of the data - if(err == STORFS_ERR_NOT_FOUND) { - err = STORFS_ERR_NO_SPACE; - } return err; } @@ -271,7 +274,7 @@ static storfs_err_t find_location(storfs_t *fs, return err; } - err = find_double_indirect_location(fs, inst, cache, &logical.pageLoc); + err = find_multiple_indirect_location(fs, inst, cache, &logical.pageLoc); if(err != STORFS_OK && err != STORFS_ERR_NO_SPACE) { return err; } @@ -379,7 +382,7 @@ static storfs_err_t process_extent_pages(storfs_t *fs, uint32_t *indirect_page) { storfs_err_t err = STORFS_OK; - if(op->op == SNODE_WRITE) { + if(op->op == SNODE_WRITE && op->is_boundary) { uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); err = bitmap_alloc_contiguous(fs, &op->extent.start, &op->extent.count, max); @@ -396,6 +399,12 @@ static storfs_err_t process_extent_pages(storfs_t *fs, SNodeExtent *indirect_extent = &((SNodeExtent *)fs->working_buf)[indirect_idx]; + if(!op->is_boundary) { + op->extent.start = indirect_extent->start; + op->extent.count = indirect_extent->count; + return STORFS_OK; + } + switch(op->op) { case SNODE_WRITE: indirect_extent->start = op->extent.start; @@ -436,6 +445,10 @@ static storfs_err_t process_direct_extents(storfs_t *fs, extent->start = direct_extent->start; extent->count = direct_extent->count; + if(!op->is_boundary) { + return STORFS_OK; + } + if(op->op == SNODE_WRITE) { uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); err = bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); @@ -554,10 +567,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - // Only update multiple if it is on a byte boundary - bool is_boundary = cache.offset_bytes % fs->pageSize == 0; - - if(op->op != SNODE_READ && is_boundary) { + if(op->op != SNODE_READ && op->is_boundary) { err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; @@ -627,6 +637,10 @@ get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { // Subtract 1 as 1 is inline page extent_cache.idx -= 1; + // Only update the extent if it is on a byte boundary, else the current + // block must be processed + op->is_boundary = extent_cache.offset_bytes % fs->pageSize == 0; + if(extent_cache.idx < DIRECT_EXTENT_SIZE) { return process_direct_extents(fs, inst, op, extent_cache); } else if(extent_cache.idx < si_size) { @@ -828,6 +842,8 @@ static storfs_err_t snode_perform_op(storfs_t *fs, STORFS_ERR_READ_FAILED if reading from the filesystem fails STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails STORFS_ERR_WRITE_FAILED if writing from the filesystem fails + STORFS_ERR_NO_FREE_BLOCKS if data is attempted to be writen beyond + the bounds of the SNode */ storfs_err_t snode_write_data(storfs_t *fs, SNodeInst *inst, diff --git a/test/test_snode.c b/test/test_snode.c index 8063b25..1816398 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -102,6 +102,19 @@ void test_snode_create(void) { STORFS_ERR_NULL_POINTER); } +static void fill_snode(SNodeInst *inst, + const uint8_t *buf, + storfs_size_t buf_size, + storfs_size_t chunk_size) { + // Fill up an snode + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + TEST_ASSERT_EQUAL(snode_write_data(fs, inst, &buf[i], data_size), + STORFS_OK); + } +} + void test_snode_write(void) { const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); uint8_t *write_buf = random_array(buf_size); @@ -124,6 +137,13 @@ void test_snode_write(void) { TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst.node.size, 0); + + fill_snode(&inst, write_buf, buf_size, fs->pageSize); + + // Test writing beyond the bounds of the SNode + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), + STORFS_ERR_NO_FREE_BLOCKS); + free(write_buf); free(read_buf); } @@ -191,24 +211,22 @@ void test_snode_find_location(void) { TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst.node.size, 0); - // Fill up an snode to the maximum size of data - uint32_t chunk_size = 1024; - uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + storfs_size_t chunk_size = fs->pageSize; + fill_snode(&inst, write_buf, buf_size, chunk_size); + + // Read all of the data to make sure it matches + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); for(uint32_t i = 0; i < buf_size; i += chunk_size) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, &write_buf[i], data_size), + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, data_size), STORFS_OK); - - // Test obtaining data from the last written location - data_remain -= data_size; - if(data_remain) { - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, data_size), - STORFS_OK); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - } + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } + free(read_buf); + + random_array_free(write_buf); // Test obtaining the location now, will be at the very end of the SNode TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NO_SPACE); @@ -219,9 +237,6 @@ void test_snode_find_location(void) { TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), STORFS_OK); - random_array_free(write_buf); - free(read_buf); - // Test improper parameters TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), STORFS_ERR_NULL_POINTER); From aa5a343d80cc5eaf5826aa2a83cdb0f0e4dacad8 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 28 Jun 2026 21:02:18 -0700 Subject: [PATCH 07/21] More bug fixes, more tests --- include/storfs.h | 1 + src/snode.c | 167 ++++++++++++------- src/snode.h | 6 +- test/helper_randomizer.c | 1 - test/test_snode.c | 351 ++++++++++++++++++++++++++++++++------- 5 files changed, 397 insertions(+), 129 deletions(-) diff --git a/include/storfs.h b/include/storfs.h index 0bccaad..e3ccea6 100644 --- a/include/storfs.h +++ b/include/storfs.h @@ -124,6 +124,7 @@ typedef enum { STORFS_ERR_NO_SPACE = 10, STORFS_ERR_NO_FREE_BLOCKS = 11, STORFS_ERR_TOO_MANY_OPEN_FILES = 12, + STORFS_ERR_END_OF_FILE = 13, // I/O errors STORFS_ERR_IO = 20, diff --git a/src/snode.c b/src/snode.c index 9446932..4b2a120 100644 --- a/src/snode.c +++ b/src/snode.c @@ -30,6 +30,11 @@ typedef struct { storfs_page_t total; } SNodeMultiple; +typedef struct { + storfs_size_t single; + storfs_size_t multiple; +} SNodeIdxCount; + static storfs_err_t snode_alloc_new_page(storfs_t *fs, storfs_page_t *page) { storfs_err_t err = bitmap_alloc(fs, page); if(err != STORFS_OK) { @@ -58,6 +63,16 @@ snode_update(storfs_t *fs, SNode *node, storfs_page_t page) { return atomic_write(fs, page); } +static inline SNodeIdxCount calc_snode_idx(const storfs_t *fs) { + const uint32_t epp = EXTENTS_PER_PAGE(fs); + const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; + const uint32_t mi_size = si_size + epp * epp; + return (SNodeIdxCount){ + .single = si_size, + .multiple = mi_size, + }; +} + /*! @brief Create an snode @@ -187,12 +202,10 @@ find_multiple_indirect_location(storfs_t *fs, storfs_page_t single_indirect_location = 0; // Determine which indirect extent page holds the desired logical location - for(uint32_t i = 0; i < NUM_MULTIPLE_EXTENTS(fs); i++) { - if(!multiple[i].total) { - break; - } - - if(*logical_page < multiple[i].total) { + uint32_t multiple_count = NUM_MULTIPLE_EXTENTS(fs); + for(uint32_t i = 0; i < multiple_count; i++) { + bool at_boundary = (i + 1 >= multiple_count) || !multiple[i + 1].total; + if(*logical_page < multiple[i].total || at_boundary) { single_indirect_location = multiple[i].single_location; break; } @@ -275,7 +288,8 @@ static storfs_err_t find_location(storfs_t *fs, } err = find_multiple_indirect_location(fs, inst, cache, &logical.pageLoc); - if(err != STORFS_OK && err != STORFS_ERR_NO_SPACE) { + if(err != STORFS_OK && err != STORFS_ERR_NOT_FOUND && + err != STORFS_ERR_NO_SPACE) { return err; } @@ -589,20 +603,25 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - // Safe to free single-indirect page now that parent is on flash - if(op->op == SNODE_ERASE && !single_indirect_page && init_single_indirect) { - err = bitmap_alloc_page(fs, init_single_indirect, PAGE_FREE); - if(err != STORFS_OK) { - return err; + if(op->op == SNODE_ERASE) { + + // Save state if multiple is empty + bool multiple_empty = !multiple->single_location; + + // Safe to free single-indirect page now that parent is on flash + if(!single_indirect_page && init_single_indirect) { + err = bitmap_alloc_page(fs, init_single_indirect, PAGE_FREE); + if(err != STORFS_OK) { + return err; + } } - } - bool empty_first_extent = - !single_indirect_page_idx && !multiple->single_location; - if(op->op == SNODE_ERASE && empty_first_extent) { - storfs_page_t init_multiple_indirect = node->indirect.multiple; - node->indirect.multiple = 0; - err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); + // If the first multiple extent is empty, free it + if(!single_indirect_page_idx && multiple_empty) { + storfs_page_t init_multiple_indirect = node->indirect.multiple; + node->indirect.multiple = 0; + err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); + } } } @@ -612,10 +631,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, static storfs_err_t get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { SNodeExtentCache extent_cache; - - const uint32_t epp = EXTENTS_PER_PAGE(fs); - const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; - const uint32_t mi_size = si_size + epp * epp; + SNodeIdxCount idx = calc_snode_idx(fs); switch(op->op) { case SNODE_WRITE: @@ -639,13 +655,13 @@ get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { // Only update the extent if it is on a byte boundary, else the current // block must be processed - op->is_boundary = extent_cache.offset_bytes % fs->pageSize == 0; + op->is_boundary = !extent_cache.offset_bytes; if(extent_cache.idx < DIRECT_EXTENT_SIZE) { return process_direct_extents(fs, inst, op, extent_cache); - } else if(extent_cache.idx < si_size) { + } else if(extent_cache.idx < idx.single) { return process_indirect_extents(fs, inst, op, extent_cache); - } else if(extent_cache.idx < mi_size) { + } else if(extent_cache.idx < idx.multiple) { return process_multiple_extents(fs, inst, op, extent_cache); } @@ -695,27 +711,35 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, } break; } - if(err != STORFS_OK) { - break; - } location.byteLoc = 0; location.pageLoc++; - pages_accessed++; op->bytes_remaining -= bytes_to_process; - } - // Update offset within contiguous block - if(err != STORFS_OK || !op->bytes_remaining) { - if(pages_accessed) { - cache->offset_bytes = (pages_accessed - 1) * fs->pageSize; + // Update offset within contiguous block + if(err != STORFS_OK || !op->bytes_remaining) { + if(pages_accessed) { + cache->offset_bytes = (pages_accessed - 1) * fs->pageSize; + } + cache->offset_bytes += bytes_to_process; + break; } - cache->offset_bytes += bytes_to_process; - } else { + + pages_accessed++; + } + + if(op->bytes_remaining || cache->offset_bytes % fs->pageSize == 0) { cache->idx++; cache->offset_bytes = 0; } + SNodeIdxCount idx = calc_snode_idx(fs); + if(cache->idx > idx.multiple) { + op->bytes_remaining = 0; + + err = STORFS_ERR_END_OF_FILE; + } + return err; } @@ -746,7 +770,8 @@ static storfs_err_t erase_data(storfs_t *fs, goto finish; } - // If page is SNode inline data the start of data increases by sizeof(SNode) + // If page is SNode inline data the start of data increases by + // sizeof(SNode) erase_bytes_offset += inst->page == page_start ? sizeof(SNode) : 0; uint32_t zero_size = fs->pageSize - erase_bytes_offset; @@ -774,23 +799,17 @@ static storfs_err_t snode_perform_op(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, uint8_t *data, - uint32_t size) { - if(!fs || !inst || (!data && op->op != SNODE_ERASE)) { + uint32_t *size) { + if(!data && op->op != SNODE_ERASE) { return STORFS_ERR_NULL_POINTER; } storfs_err_t err = STORFS_OK; SNodeExtentCache *cache = &inst->write; - if(op->op == SNODE_READ) { cache = &inst->read; } - // Do not erase past end of an snode - if(op->op == SNODE_ERASE && size > inst->node.size) { - return STORFS_ERR_INVALID_PARAM; - } - while(err == STORFS_OK && op->bytes_remaining) { err = get_modify_extents(fs, inst, op); if(err != STORFS_OK) { @@ -798,13 +817,16 @@ static storfs_err_t snode_perform_op(storfs_t *fs, } if(op->op != SNODE_ERASE) { - err = snode_read_or_write_data(fs, op, data, size, cache); + err = snode_read_or_write_data(fs, op, data, *size, cache); } else { err = erase_data(fs, inst, op, cache); } } - uint32_t processed_bytes = size - op->bytes_remaining; + // Set the number of bytes processed + *size = *size - op->bytes_remaining; + uint32_t processed_bytes = *size; + if(!processed_bytes) { return err; } @@ -835,7 +857,8 @@ static storfs_err_t snode_perform_op(storfs_t *fs, @param fs pointer to the filesystem instance @param inst pointer to snode instance @param data data to write to the snode - @param size size of data to write to the snode + @param size size of data to write to the snode, the total size is written + here @return STORFS_OK on success STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments @@ -848,23 +871,27 @@ static storfs_err_t snode_perform_op(storfs_t *fs, storfs_err_t snode_write_data(storfs_t *fs, SNodeInst *inst, const uint8_t *data, - uint32_t size) { - SNodeOpInst op = { .op = SNODE_WRITE, .bytes_remaining = size }; + uint32_t *size) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + SNodeOpInst op = { .op = SNODE_WRITE, .bytes_remaining = *size }; return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); } /*! @brief Read data from an snode instance - @details Must call @link snode_find_read_location @endlink before reading from - an snode. This function will begin reading from the offset indicated - in @link snode_find_read_location @endlink. + @details Must call @link snode_find_read_location @endlink before reading + from an snode. This function will begin reading from the offset indicated in + @link snode_find_read_location @endlink. @param fs pointer to the filesystem instance @param inst pointer to snode instance @param data data to read from the snode - @param size size of data to read from the snode + @param size size of data to read from the snode, the total size read is + written here @return STORFS_OK on success STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments @@ -873,9 +900,12 @@ storfs_err_t snode_write_data(storfs_t *fs, STORFS_ERR_WRITE_FAILED if writing from the filesystem fails */ storfs_err_t -snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { - SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = size }; +snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t *size) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = *size }; return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); } @@ -887,9 +917,10 @@ snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { of the snode. The cache write location will be updated with each erase. - @param fs pointer to the filesystem instance + @param fs pointer to the filesystem i18786nstance @param inst pointer to snode instance - @param size size of data to erase from the snode + @param size size of data to erase from the snode, the number of bytes erased + are written here @return STORFS_OK on success STORFS_ERR_NULL_POINTER if NULL pointers passed into arguments @@ -897,8 +928,22 @@ snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size) { STORFS_ERR_ERASE_FAILED if erasing from the filesystem fails STORFS_ERR_WRITE_FAILED if writing from the filesystem fails */ -storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t size) { - SNodeOpInst op = { .op = SNODE_ERASE, .bytes_remaining = size }; +storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t *size) { + if(!fs || !inst) { + return STORFS_ERR_NULL_POINTER; + } + + SNodeIdxCount idx = calc_snode_idx(fs); + + // Do not erase past end of an snode + if(*size > inst->node.size) { + return STORFS_ERR_INVALID_PARAM; + } + + if(inst->write.idx == idx.multiple) { + inst->write.idx--; + } + SNodeOpInst op = { .op = SNODE_ERASE, .bytes_remaining = *size }; return snode_perform_op(fs, inst, &op, NULL, size); } diff --git a/src/snode.h b/src/snode.h index 1f8e203..35cbea3 100644 --- a/src/snode.h +++ b/src/snode.h @@ -68,8 +68,8 @@ storfs_err_t snode_find_write_location(storfs_t *fs, SNodeInst *inst); storfs_err_t snode_write_data(storfs_t *fs, SNodeInst *inst, const uint8_t *data, - uint32_t size); + uint32_t *size); storfs_err_t -snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t size); -storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t size); +snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t *size); +storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t *size); #endif diff --git a/test/helper_randomizer.c b/test/helper_randomizer.c index cc5e4a2..ec8a1b1 100644 --- a/test/helper_randomizer.c +++ b/test/helper_randomizer.c @@ -16,5 +16,4 @@ uint8_t *random_array(size_t size) { void random_array_free(uint8_t *buf) { free(buf); - buf = NULL; } diff --git a/test/test_snode.c b/test/test_snode.c index 1816398..60fab35 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -102,25 +102,60 @@ void test_snode_create(void) { STORFS_ERR_NULL_POINTER); } -static void fill_snode(SNodeInst *inst, - const uint8_t *buf, - storfs_size_t buf_size, - storfs_size_t chunk_size) { - // Fill up an snode - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_write_data(fs, inst, &buf[i], data_size), - STORFS_OK); +static inline storfs_err_t get_err_compare(uint32_t data_remain) { + storfs_err_t compare = STORFS_ERR_END_OF_FILE; + if(data_remain > fs->pageSize) { + compare = STORFS_OK; } + + return compare; +} + +static inline uint32_t calc_max_idx(void) { +#define EXTENTS_PER_PAGE(f) (((f)->pageSize / sizeof(SNodeExtent))) + const uint32_t epp = EXTENTS_PER_PAGE(fs); + const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; + return si_size + epp * epp; +} + +static uint8_t * +fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { + uint32_t max = calc_max_idx(); + + *buf_size = chunk_size * max; + uint32_t leftover = fs->pageSize - sizeof(SNode); + *buf_size += leftover; + uint8_t *buf = random_array(*buf_size); + + TEST_ASSERT_EQUAL(snode_write_data(fs, inst, buf, &leftover), STORFS_OK); + TEST_ASSERT_EQUAL(inst->write.idx, 1); + + uint32_t i; + for(i = leftover; i < *buf_size; i += chunk_size) { + uint32_t data_remain = *buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + storfs_err_t err = snode_write_data(fs, inst, &buf[i], &data_size); + uint32_t write_idx = i / chunk_size + 2; + TEST_ASSERT_EQUAL(inst->write.idx, write_idx); + storfs_err_t compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, compare); + } + + if(i < *buf_size) { + uint32_t data_size = *buf_size - i; + TEST_ASSERT_EQUAL(snode_write_data(fs, inst, &buf[i], &data_size), + STORFS_ERR_END_OF_FILE); + } + + return buf; } void test_snode_write(void) { - const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); - uint8_t *write_buf = random_array(buf_size); - uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; - storfs_page_t page = 17; + uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; + storfs_page_t page = 17; storfs_crc16_IgnoreAndReturn(FAKE_CRC16); SNodeInst inst = { 0 }; @@ -131,20 +166,27 @@ void test_snode_write(void) { TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), - STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, buf_size), STORFS_OK); + uint32_t bytes = buf_size; + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, buf_size); + TEST_ASSERT_EQUAL(inst.node.size, buf_size); + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &bytes), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, buf_size); TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); TEST_ASSERT_EQUAL(inst.node.size, 0); + random_array_free(write_buf); - fill_snode(&inst, write_buf, buf_size, fs->pageSize); + write_buf = fill_snode(&inst, &buf_size, fs->pageSize); // Test writing beyond the bounds of the SNode - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), STORFS_ERR_NO_FREE_BLOCKS); - free(write_buf); + random_array_free(write_buf); free(read_buf); } @@ -160,9 +202,9 @@ void test_snode_write(void) { // TEST_ASSERT_EQUAL(snode_create(fs, name, &page), STORFS_OK); // // // Write all the data from the file, read it and compare -// TEST_ASSERT_EQUAL(snode_write_data(fs, page, write_buf, buf_size), +// TEST_ASSERT_EQUAL(snode_write_data(fs, page, write_buf, &buf_size), // STORFS_OK); TEST_ASSERT_EQUAL(snode_read_data(fs, page, 0, read_buf, -// buf_size), +// &buf_size), // STORFS_OK); // TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); // @@ -171,13 +213,13 @@ void test_snode_write(void) { // for(uint32_t i = 0; i < buf_size; i += chunk_size) { // uint32_t data_remain = buf_size - i; // uint32_t data_size = MIN(data_remain, chunk_size); -// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, data_size), +// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, &data_size), // STORFS_OK); // TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); // } // // // Test reading past boundaries -// TEST_ASSERT_EQUAL(snode_read_data(fs, page, buf_size, read_buf, 1), +// TEST_ASSERT_EQUAL(snode_read_data(fs, page, buf_size, read_buf&, 1), // STORFS_ERR_INVALID_PARAM); // // random_array_free(write_buf); @@ -185,12 +227,87 @@ void test_snode_write(void) { // } // -void test_snode_find_location(void) { - const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); - uint8_t *write_buf = random_array(buf_size); - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; +// void test_snode_find_location(void) { +// uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); +// uint8_t *write_buf = random_array(buf_size); +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// uint32_t bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), +// STORFS_OK); TEST_ASSERT_EQUAL(bytes, buf_size); +// +// // Test obtaining the location now, will be in indirect extents which has +// yet +// // to be created +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), +// STORFS_ERR_NOT_FOUND); +// +// // Erase and then fill the SNode all the way up +// bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(bytes, buf_size); +// TEST_ASSERT_EQUAL(inst.node.size, 0); +// +// random_array_free(write_buf); +// storfs_size_t chunk_size = fs->pageSize; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// +// // Now compare after filling +// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); +// bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &buf_size), +// STORFS_OK); TEST_ASSERT_EQUAL(bytes, buf_size); +// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); +// free(read_buf); +// printf("Fucker: %d\n", inst.write.idx); +// +// // Test obtaining the location now, will be at the very end of the SNode +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), +// STORFS_ERR_NO_SPACE); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, +// buf_size), +// STORFS_ERR_NO_SPACE); +// +// printf("Fucker: %d\n", inst.write.idx); +// +// // Test getting the data almost at the end of the file +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), +// STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &buf_size), +// STORFS_ERR_END_OF_FILE); +// printf("Fucker: %d\n", inst.write.idx); +// random_array_free(write_buf); +// printf("Fucker: %d\n", inst.write.idx); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.node.size, 0); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// +// // Test improper parameters +// TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), +// STORFS_ERR_NULL_POINTER); +// } +// +// +void test_read_after_full_write(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; storfs_crc16_IgnoreAndReturn(FAKE_CRC16); @@ -200,19 +317,43 @@ void test_snode_find_location(void) { TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, buf_size), - STORFS_OK); + storfs_size_t chunk_size = fs->pageSize; + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); - // Test obtaining the location now, will be in indirect extents which has yet - // to be created - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NOT_FOUND); + // Read all of the data to make sure it matches + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); + storfs_err_t compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, compare); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + } - // Erase and then fill the SNode all the way up - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.node.size, 0); + random_array_free(write_buf); + free(read_buf); +} + +void test_read_after_full_write_update_cache(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); storfs_size_t chunk_size = fs->pageSize; - fill_snode(&inst, write_buf, buf_size, chunk_size); + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); // Read all of the data to make sure it matches uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); @@ -220,32 +361,114 @@ void test_snode_find_location(void) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, data_size), - STORFS_OK); + storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); + storfs_err_t compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, compare); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } + + random_array_free(write_buf); free(read_buf); +} + +void test_fill_stagger(void) { + uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + storfs_size_t chunk_size = fs->pageSize; + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + uint32_t idx_compare = i / chunk_size + 1; + + storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); + storfs_err_t err_compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, err_compare); + TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); + // Test that finding the location is also correct + snode_find_write_location(fs, &inst); + TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); + } + + // Read all of the data to make sure it matches + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); + storfs_err_t compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, compare); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + } random_array_free(write_buf); + free(read_buf); +} - // Test obtaining the location now, will be at the very end of the SNode - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NO_SPACE); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size), - STORFS_ERR_NO_SPACE); +void test_fill_boundary(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; - // Test getting the data almost at the end of the file - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + // Test writing on boundary + storfs_size_t chunk_size = fs->pageSize; + uint32_t buf_size = chunk_size * calc_max_idx(); + uint32_t leftover = fs->pageSize - sizeof(SNode); + buf_size += leftover; + uint8_t *write_buf = random_array(buf_size); + + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &leftover), STORFS_OK); - // Test improper parameters - TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), - STORFS_ERR_NULL_POINTER); + for(uint32_t i = leftover; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = chunk_size; + + // printf("Leftover: %d\n", data_remain); + storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); + storfs_err_t err_compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, err_compare); + + const uint32_t idx_compare = i / chunk_size + 2; + TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); + + // Test obtaining write location works as expected as well + snode_find_write_location(fs, &inst); + TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); + } + + // Read all of the data to make sure it matches + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); + storfs_err_t compare = get_err_compare(data_remain); + TEST_ASSERT_EQUAL(err, compare); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + } + + random_array_free(write_buf); + free(read_buf); } void test_snode_write_read_alternate(void) { @@ -279,23 +502,23 @@ void test_snode_write_read_alternate(void) { for(uint32_t i = 0; i < buf_size; i += chunk_size) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], data_size), + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], &data_size), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, data_size), + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, &data_size), STORFS_OK); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], data_size), + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], &data_size), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, data_size), + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, &data_size), STORFS_OK); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } TEST_ASSERT_EQUAL(inst_1.node.size, buf_size); TEST_ASSERT_EQUAL(inst_2.node.size, buf_size); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_1, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_1, &buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst_1.node.size, 0); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, &buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst_2.node.size, 0); // TODO test read here From e9bea61d0e8db1a040565aafc6868bc115c4d976 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Wed, 8 Jul 2026 20:38:43 -0700 Subject: [PATCH 08/21] Updated unit tests and other bug fixes --- src/snode.c | 29 ++-- test/helper_randomizer.c | 6 + test/helper_randomizer.h | 1 + test/test_snode.c | 292 +++++++++++++++++++++++++++------------ 4 files changed, 223 insertions(+), 105 deletions(-) diff --git a/src/snode.c b/src/snode.c index 4b2a120..4c668da 100644 --- a/src/snode.c +++ b/src/snode.c @@ -215,11 +215,6 @@ find_multiple_indirect_location(storfs_t *fs, } if(!single_indirect_location) { - // The location is at the end of the data - if(*logical_page == 0) { - return STORFS_ERR_NO_SPACE; - } - return STORFS_ERR_NOT_FOUND; } @@ -228,6 +223,11 @@ find_multiple_indirect_location(storfs_t *fs, single_indirect_location, logical_page); + // The location is at the end of the data known data if not found + if(err == STORFS_ERR_NOT_FOUND) { + err = STORFS_ERR_NO_SPACE; + } + return err; } @@ -712,19 +712,20 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, break; } - location.byteLoc = 0; - location.pageLoc++; op->bytes_remaining -= bytes_to_process; - // Update offset within contiguous block + // Update offset within contiguous block. Must be computed before + // location is advanced to the next page below, since it needs the + // page/byte location the data was just written/read at. if(err != STORFS_OK || !op->bytes_remaining) { - if(pages_accessed) { - cache->offset_bytes = (pages_accessed - 1) * fs->pageSize; - } - cache->offset_bytes += bytes_to_process; + cache->offset_bytes = + (location.pageLoc - op->extent.start) * fs->pageSize + + location.byteLoc + bytes_to_process; break; } + location.byteLoc = 0; + location.pageLoc++; pages_accessed++; } @@ -940,8 +941,8 @@ storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t *size) { return STORFS_ERR_INVALID_PARAM; } - if(inst->write.idx == idx.multiple) { - inst->write.idx--; + if(inst->write.idx > idx.multiple) { + inst->write.idx = idx.multiple; } SNodeOpInst op = { .op = SNODE_ERASE, .bytes_remaining = *size }; diff --git a/test/helper_randomizer.c b/test/helper_randomizer.c index ec8a1b1..73aad47 100644 --- a/test/helper_randomizer.c +++ b/test/helper_randomizer.c @@ -17,3 +17,9 @@ uint8_t *random_array(size_t size) { void random_array_free(uint8_t *buf) { free(buf); } + +uint64_t random_integer(uint64_t max) { + time_t t = clock(); + srand(t * 10000); + return rand() % max; +} diff --git a/test/helper_randomizer.h b/test/helper_randomizer.h index c4db8fc..0ba00d5 100644 --- a/test/helper_randomizer.h +++ b/test/helper_randomizer.h @@ -6,5 +6,6 @@ uint8_t *random_array(size_t size); void random_array_free(uint8_t *buf); +uint64_t random_integer(uint64_t max); #endif diff --git a/test/test_snode.c b/test/test_snode.c index 60fab35..3800085 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -102,9 +102,10 @@ void test_snode_create(void) { STORFS_ERR_NULL_POINTER); } -static inline storfs_err_t get_err_compare(uint32_t data_remain) { +static inline storfs_err_t get_err_compare(uint32_t data_remain, + storfs_size_t chunk_size) { storfs_err_t compare = STORFS_ERR_END_OF_FILE; - if(data_remain > fs->pageSize) { + if(data_remain > chunk_size) { compare = STORFS_OK; } @@ -118,6 +119,40 @@ static inline uint32_t calc_max_idx(void) { return si_size + epp * epp; } +/*! + @brief Simulate IDX Advancement + + @details Mirrors how get_modify_extents/snode_read_or_write_data pack bytes + into extents, a fresh extent is sized to exactly cover whatever is + left of the current write call (rounded up to whole pages), so + write.idx only advances once that extent's page-aligned capacity is + fully consumed. Chunk sizes that aren't a multiple of the page size + leave a partial page of slack that carries over into the next write + call. + + @param data_size data to advance + @param extent_remaining the remainder of the extent before moving to the next + index + + @return The change in index + */ +static uint32_t simulate_idx_advance(uint32_t data_size, + uint32_t *extent_remaining) { + uint32_t idx_delta = 0; + while(data_size > 0) { + if(!*extent_remaining) { + *extent_remaining = CEIL_DIV(data_size, fs->pageSize) * fs->pageSize; + } + uint32_t consumed = MIN(data_size, *extent_remaining); + *extent_remaining -= consumed; + data_size -= consumed; + if(!*extent_remaining) { + idx_delta++; + } + } + return idx_delta; +} + static uint8_t * fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { uint32_t max = calc_max_idx(); @@ -130,14 +165,17 @@ fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { TEST_ASSERT_EQUAL(snode_write_data(fs, inst, buf, &leftover), STORFS_OK); TEST_ASSERT_EQUAL(inst->write.idx, 1); + uint32_t extent_remaining = 0; + uint32_t write_idx = 1; uint32_t i; for(i = leftover; i < *buf_size; i += chunk_size) { uint32_t data_remain = *buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); storfs_err_t err = snode_write_data(fs, inst, &buf[i], &data_size); - uint32_t write_idx = i / chunk_size + 2; + write_idx += + simulate_idx_advance(MIN(data_remain, chunk_size), &extent_remaining); TEST_ASSERT_EQUAL(inst->write.idx, write_idx); - storfs_err_t compare = get_err_compare(data_remain); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, compare); } @@ -227,83 +265,97 @@ void test_snode_write(void) { // } // -// void test_snode_find_location(void) { -// uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); -// uint8_t *write_buf = random_array(buf_size); -// storfs_page_t page = 17; -// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; -// SNodeInst inst = { 0 }; -// -// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); -// -// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), -// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); -// -// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); -// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); -// -// uint32_t bytes = buf_size; -// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), -// STORFS_OK); TEST_ASSERT_EQUAL(bytes, buf_size); -// -// // Test obtaining the location now, will be in indirect extents which has -// yet -// // to be created -// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), -// STORFS_ERR_NOT_FOUND); -// -// // Erase and then fill the SNode all the way up -// bytes = buf_size; -// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); -// TEST_ASSERT_EQUAL(bytes, buf_size); -// TEST_ASSERT_EQUAL(inst.node.size, 0); -// -// random_array_free(write_buf); -// storfs_size_t chunk_size = fs->pageSize; -// write_buf = fill_snode(&inst, &buf_size, chunk_size); -// -// // Now compare after filling -// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); -// bytes = buf_size; -// TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &buf_size), -// STORFS_OK); TEST_ASSERT_EQUAL(bytes, buf_size); -// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); -// free(read_buf); -// printf("Fucker: %d\n", inst.write.idx); -// -// // Test obtaining the location now, will be at the very end of the SNode -// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), -// STORFS_ERR_NO_SPACE); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, -// buf_size), -// STORFS_ERR_NO_SPACE); -// -// printf("Fucker: %d\n", inst.write.idx); -// -// // Test getting the data almost at the end of the file -// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), -// STORFS_OK); -// -// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &buf_size), -// STORFS_ERR_END_OF_FILE); -// printf("Fucker: %d\n", inst.write.idx); -// random_array_free(write_buf); -// printf("Fucker: %d\n", inst.write.idx); -// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); -// TEST_ASSERT_EQUAL(inst.node.size, 0); -// TEST_ASSERT_EQUAL(inst.write.idx, 0); -// -// // Test improper parameters -// TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), -// STORFS_ERR_NULL_POINTER); -// TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), -// STORFS_ERR_NULL_POINTER); -// TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), -// STORFS_ERR_NULL_POINTER); -// TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), -// STORFS_ERR_NULL_POINTER); -// } -// -// +void test_snode_find_location(void) { + uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); + uint8_t *write_buf = random_array(buf_size); + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + uint32_t bytes = buf_size; + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, buf_size); + + // Test obtaining the location now, will be in indirect extents which has yet + // to be created + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NOT_FOUND); + + // Erase and then fill the SNode all the way up + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, buf_size); + TEST_ASSERT_EQUAL(inst.node.size, 0); + + random_array_free(write_buf); + storfs_size_t chunk_size = fs->pageSize; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + + // Now compare after filling + uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &buf_size), + STORFS_ERR_END_OF_FILE); + TEST_ASSERT_EQUAL(bytes, buf_size); + TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); + free(read_buf); + + // Test obtaining the location now, will be at the very end of the SNode + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NO_SPACE); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size), + STORFS_ERR_NO_SPACE); + + // Test getting the data almost at the end of the file + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), + STORFS_OK); + + bytes = buf_size; + TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), + STORFS_ERR_NO_FREE_BLOCKS); + random_array_free(write_buf); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.node.size, 0); + TEST_ASSERT_EQUAL(inst.write.idx, 0); + + // Test improper parameters + TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), + STORFS_ERR_NULL_POINTER); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), + STORFS_ERR_NULL_POINTER); +} + +static void +simple_read_helper(SNodeInst *inst, uint8_t *write_buf, uint32_t write_size) { + // Read all data in one go + uint8_t *read_buf = (uint8_t *)calloc(write_size, sizeof(uint8_t)); + uint32_t bytes = write_size; + TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); + TEST_ASSERT_EQUAL(snode_read_data(fs, inst, read_buf, &bytes), + STORFS_ERR_END_OF_FILE); + for(uint32_t i = 0; i < write_size; i++) { + if(read_buf[i] != write_buf[i]) { + printf("MEMCMP unequal at: %d. Data values - write: %d - read: %d\n", + i, + write_buf[i], + read_buf[i]); + break; + } + } + TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, write_size), 0); + free(read_buf); +} + void test_read_after_full_write(void) { storfs_page_t page = 17; char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; @@ -328,7 +380,7 @@ void test_read_after_full_write(void) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, compare); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } @@ -362,13 +414,71 @@ void test_read_after_full_write_update_cache(void) { uint32_t data_size = MIN(data_remain, chunk_size); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, compare); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } + free(read_buf); + + simple_read_helper(&inst, write_buf, buf_size); random_array_free(write_buf); - free(read_buf); +} + +void test_read_after_random_page_full_write(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + storfs_size_t chunk_size = fs->pageSize * 5; + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + simple_read_helper(&inst, write_buf, buf_size); + random_array_free(write_buf); + snode_erase_data(fs, &inst, &buf_size); + + chunk_size = fs->pageSize * 12; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + simple_read_helper(&inst, write_buf, buf_size); + random_array_free(write_buf); + snode_erase_data(fs, &inst, &buf_size); + + chunk_size = (storfs_size_t)((float)fs->pageSize * 5.25); + write_buf = fill_snode(&inst, &buf_size, chunk_size); + simple_read_helper(&inst, write_buf, buf_size); + random_array_free(write_buf); + snode_erase_data(fs, &inst, &buf_size); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + write_buf = NULL; + uint32_t allocated = 0; + storfs_err_t err; + do { + chunk_size = random_integer(8192); + uint32_t written = chunk_size; + uint32_t offset = allocated; + allocated += chunk_size; + write_buf = realloc(write_buf, allocated); + err = snode_write_data(fs, &inst, &write_buf[offset], &written); + printf("Allocated: %d\n", allocated); + if(err == STORFS_OK) { + TEST_ASSERT_EQUAL(written, chunk_size); + } else { + TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); + } + } while(err == STORFS_OK); + simple_read_helper(&inst, write_buf, allocated); + random_array_free(write_buf); + snode_erase_data(fs, &inst, &buf_size); } void test_fill_stagger(void) { @@ -393,7 +503,7 @@ void test_fill_stagger(void) { uint32_t idx_compare = i / chunk_size + 1; storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); - storfs_err_t err_compare = get_err_compare(data_remain); + storfs_err_t err_compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, err_compare); TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); // Test that finding the location is also correct @@ -407,7 +517,7 @@ void test_fill_stagger(void) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, compare); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } @@ -443,9 +553,8 @@ void test_fill_boundary(void) { uint32_t data_remain = buf_size - i; uint32_t data_size = chunk_size; - // printf("Leftover: %d\n", data_remain); storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); - storfs_err_t err_compare = get_err_compare(data_remain); + storfs_err_t err_compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, err_compare); const uint32_t idx_compare = i / chunk_size + 2; @@ -462,7 +571,7 @@ void test_fill_boundary(void) { uint32_t data_remain = buf_size - i; uint32_t data_size = MIN(data_remain, chunk_size); storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); TEST_ASSERT_EQUAL(err, compare); TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); } @@ -492,6 +601,9 @@ void test_snode_write_read_alternate(void) { // Write all the data from the file, read it and compare SNodeInst inst_1 = { 0 }; SNodeInst inst_2 = { 0 }; + TEST_ASSERT_EQUAL(snode_lookup(fs, snode_1_page, &inst_1), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, snode_2_page, &inst_2), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_1), STORFS_OK); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_1, 0), STORFS_OK); TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_2), STORFS_OK); @@ -521,8 +633,6 @@ void test_snode_write_read_alternate(void) { TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, &buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst_2.node.size, 0); - // TODO test read here - random_array_free(write_buf); free(read_buf); } From 9c57e2683c193ad270e8c5c75bb28bcade739181 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Wed, 8 Jul 2026 20:50:44 -0700 Subject: [PATCH 09/21] More bug fixes, more tests --- src/snode.c | 2 -- test/test_snode.c | 29 ++++++++++++++--------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/snode.c b/src/snode.c index 4c668da..0c18777 100644 --- a/src/snode.c +++ b/src/snode.c @@ -736,8 +736,6 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, SNodeIdxCount idx = calc_snode_idx(fs); if(cache->idx > idx.multiple) { - op->bytes_remaining = 0; - err = STORFS_ERR_END_OF_FILE; } diff --git a/test/test_snode.c b/test/test_snode.c index 3800085..cb61b60 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -343,15 +343,6 @@ simple_read_helper(SNodeInst *inst, uint8_t *write_buf, uint32_t write_size) { TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); TEST_ASSERT_EQUAL(snode_read_data(fs, inst, read_buf, &bytes), STORFS_ERR_END_OF_FILE); - for(uint32_t i = 0; i < write_size; i++) { - if(read_buf[i] != write_buf[i]) { - printf("MEMCMP unequal at: %d. Data values - write: %d - read: %d\n", - i, - write_buf[i], - read_buf[i]); - break; - } - } TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, write_size), 0); free(read_buf); } @@ -444,24 +435,30 @@ void test_read_after_random_page_full_write(void) { write_buf = fill_snode(&inst, &buf_size, chunk_size); simple_read_helper(&inst, write_buf, buf_size); random_array_free(write_buf); - snode_erase_data(fs, &inst, &buf_size); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.write.idx, 0); chunk_size = fs->pageSize * 12; write_buf = fill_snode(&inst, &buf_size, chunk_size); simple_read_helper(&inst, write_buf, buf_size); random_array_free(write_buf); - snode_erase_data(fs, &inst, &buf_size); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.write.idx, 0); + // Offset chunk not exactly on a byte boundary chunk_size = (storfs_size_t)((float)fs->pageSize * 5.25); write_buf = fill_snode(&inst, &buf_size, chunk_size); simple_read_helper(&inst, write_buf, buf_size); random_array_free(write_buf); - snode_erase_data(fs, &inst, &buf_size); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.write.idx, 0); + // Write random data to the file until it filles up TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); write_buf = NULL; uint32_t allocated = 0; storfs_err_t err; + uint32_t file_size = 0; do { chunk_size = random_integer(8192); uint32_t written = chunk_size; @@ -469,16 +466,18 @@ void test_read_after_random_page_full_write(void) { allocated += chunk_size; write_buf = realloc(write_buf, allocated); err = snode_write_data(fs, &inst, &write_buf[offset], &written); - printf("Allocated: %d\n", allocated); if(err == STORFS_OK) { TEST_ASSERT_EQUAL(written, chunk_size); } else { TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); + // The total file size is equal to the last data written + file_size = allocated - (chunk_size - written); } } while(err == STORFS_OK); - simple_read_helper(&inst, write_buf, allocated); + simple_read_helper(&inst, write_buf, file_size); random_array_free(write_buf); - snode_erase_data(fs, &inst, &buf_size); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &file_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.write.idx, 0); } void test_fill_stagger(void) { From 4eaab461b99eea04bf299376dcce52b46b665aa0 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Wed, 8 Jul 2026 21:00:01 -0700 Subject: [PATCH 10/21] Add back in read test --- test/test_snode.c | 75 ++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/test/test_snode.c b/test/test_snode.c index cb61b60..c184821 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -228,42 +228,45 @@ void test_snode_write(void) { free(read_buf); } -// void test_snode_write_read(void) { -// const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); -// uint8_t *write_buf = random_array(buf_size); -// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); -// storfs_page_t page = 17; -// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; -// -// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); -// -// TEST_ASSERT_EQUAL(snode_create(fs, name, &page), STORFS_OK); -// -// // Write all the data from the file, read it and compare -// TEST_ASSERT_EQUAL(snode_write_data(fs, page, write_buf, &buf_size), -// STORFS_OK); TEST_ASSERT_EQUAL(snode_read_data(fs, page, 0, read_buf, -// &buf_size), -// STORFS_OK); -// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); -// -// // Chunk read it to check offset paramter -// uint32_t chunk_size = fs->pageSize * 3; -// for(uint32_t i = 0; i < buf_size; i += chunk_size) { -// uint32_t data_remain = buf_size - i; -// uint32_t data_size = MIN(data_remain, chunk_size); -// TEST_ASSERT_EQUAL(snode_read_data(fs, page, i, read_buf, &data_size), -// STORFS_OK); -// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); -// } -// -// // Test reading past boundaries -// TEST_ASSERT_EQUAL(snode_read_data(fs, page, buf_size, read_buf&, 1), -// STORFS_ERR_INVALID_PARAM); -// -// random_array_free(write_buf); -// free(read_buf); -// } -// +void test_snode_write_read(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + + storfs_size_t chunk_size = fs->pageSize; + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + + // Chunk read it to check offset paramter + chunk_size = fs->pageSize * 3; + uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); + for(uint32_t i = 0; i < buf_size; i += chunk_size) { + uint32_t data_remain = buf_size - i; + uint32_t data_size = MIN(data_remain, chunk_size); + uint32_t data_read = data_size; + storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_read); + TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_read), 0); + storfs_err_t compare = get_err_compare(data_remain, chunk_size); + TEST_ASSERT_EQUAL(err, compare); + } + + // Test reading past boundaries + uint32_t data_read = chunk_size; + TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &data_read), + STORFS_ERR_NO_FREE_BLOCKS); + + random_array_free(write_buf); + free(read_buf); +} void test_snode_find_location(void) { uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); From 154529aed07920d5504971211acbb14b54f08d81 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 11 Jul 2026 23:07:22 -0700 Subject: [PATCH 11/21] Bug fix more test --- src/snode.c | 3 ++- test/test_snode.c | 32 +++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/snode.c b/src/snode.c index 0c18777..6ba88a2 100644 --- a/src/snode.c +++ b/src/snode.c @@ -729,7 +729,8 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, pages_accessed++; } - if(op->bytes_remaining || cache->offset_bytes % fs->pageSize == 0) { + if(op->bytes_remaining || + cache->offset_bytes == op->extent.count * fs->pageSize) { cache->idx++; cache->offset_bytes = 0; } diff --git a/test/test_snode.c b/test/test_snode.c index c184821..3f760fe 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -456,19 +456,22 @@ void test_read_after_random_page_full_write(void) { TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); TEST_ASSERT_EQUAL(inst.write.idx, 0); - // Write random data to the file until it filles up + // Write random chunk size to the file until it filles up + static const uint16_t random_chunk_max = 8192; TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); write_buf = NULL; uint32_t allocated = 0; storfs_err_t err; uint32_t file_size = 0; do { - chunk_size = random_integer(8192); - uint32_t written = chunk_size; - uint32_t offset = allocated; + chunk_size = random_integer(random_chunk_max); + uint32_t written = chunk_size; + storfs_size_t offset = allocated; + allocated += chunk_size; write_buf = realloc(write_buf, allocated); - err = snode_write_data(fs, &inst, &write_buf[offset], &written); + + err = snode_write_data(fs, &inst, &write_buf[offset], &written); if(err == STORFS_OK) { TEST_ASSERT_EQUAL(written, chunk_size); } else { @@ -478,6 +481,25 @@ void test_read_after_random_page_full_write(void) { } } while(err == STORFS_OK); simple_read_helper(&inst, write_buf, file_size); + + // Read back random chunk size + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); + storfs_size_t offset = 0; + do { + chunk_size = random_integer(random_chunk_max); + uint8_t *read_buf = (uint8_t *)malloc(chunk_size); + storfs_size_t read = chunk_size; + err = snode_read_data(fs, &inst, read_buf, &read); + if(err == STORFS_OK) { + TEST_ASSERT_EQUAL(read, chunk_size); + } else { + TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); + } + TEST_ASSERT_EQUAL(memcmp(&write_buf[offset], read_buf, read), 0); + offset += read; + free(read_buf); + } while(err == STORFS_OK); + random_array_free(write_buf); TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &file_size), STORFS_OK); TEST_ASSERT_EQUAL(inst.write.idx, 0); From 9355e86bbc1d420029c298e9c26983fddf0df23c Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 11 Jul 2026 23:31:52 -0700 Subject: [PATCH 12/21] More test more cleanup --- src/snode.c | 8 ++--- test/test_snode.c | 82 ++++++++++++++++++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/src/snode.c b/src/snode.c index 6ba88a2..8bcbb3b 100644 --- a/src/snode.c +++ b/src/snode.c @@ -800,10 +800,6 @@ static storfs_err_t snode_perform_op(storfs_t *fs, SNodeOpInst *op, uint8_t *data, uint32_t *size) { - if(!data && op->op != SNODE_ERASE) { - return STORFS_ERR_NULL_POINTER; - } - storfs_err_t err = STORFS_OK; SNodeExtentCache *cache = &inst->write; if(op->op == SNODE_READ) { @@ -872,7 +868,7 @@ storfs_err_t snode_write_data(storfs_t *fs, SNodeInst *inst, const uint8_t *data, uint32_t *size) { - if(!fs || !inst) { + if(!fs || !inst || !data || !size) { return STORFS_ERR_NULL_POINTER; } @@ -901,7 +897,7 @@ storfs_err_t snode_write_data(storfs_t *fs, */ storfs_err_t snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t *size) { - if(!fs || !inst) { + if(!fs || !inst || !data || !size) { return STORFS_ERR_NULL_POINTER; } diff --git a/test/test_snode.c b/test/test_snode.c index 3f760fe..a12f554 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -20,6 +20,8 @@ #define MEMBER_SIZE(type, member) (sizeof(((type *)0)->member)) #define DIRECT_ARRAY_SIZE (MEMBER_SIZE(SNode, direct) / sizeof(SNodeExtent)) +#define RANDOM_DATA_MAX_CHUNK (8192) + #define DATA_PAGES_PER_INDIRECT_PAGE(f) ((f->pageSize / sizeof(SNodeExtent))) #define SINGLE_INDIRECT_DATA_CALC(f) \ (f->pageSize * DATA_PAGES_PER_INDIRECT_PAGE(f)) @@ -338,6 +340,65 @@ void test_snode_find_location(void) { STORFS_ERR_NULL_POINTER); } +static void full_read_helper(SNodeInst *inst, + uint8_t *write_buf, + storfs_size_t file_size, + storfs_size_t read_offset, + bool random, + storfs_size_t chunk_read) { + TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); + storfs_size_t offset = 0; + storfs_err_t err; + storfs_size_t chunk_size; + do { + if(random) { + chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); + } else { + chunk_size = chunk_read; + } + + uint8_t *read_buf = (uint8_t *)malloc(chunk_size); + storfs_size_t read = chunk_size; + err = snode_read_data(fs, inst, read_buf, &read); + if(err == STORFS_OK) { + TEST_ASSERT_EQUAL(read, chunk_size); + } else { + TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); + } + TEST_ASSERT_EQUAL(memcmp(&write_buf[offset], read_buf, read), 0); + offset += read; + free(read_buf); + } while(err == STORFS_OK); +} + +void test_snode_misaligned_read(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + + storfs_size_t chunk_size = fs->pageSize * 6; + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + + uint32_t read_offset = fs->pageSize * 2; + uint32_t read_size = fs->pageSize * 3; + TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, read_offset), + STORFS_OK); + full_read_helper(&inst, write_buf, buf_size, read_offset, false, read_size); + + random_array_free(write_buf); + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); + TEST_ASSERT_EQUAL(inst.write.idx, 0); +} + static void simple_read_helper(SNodeInst *inst, uint8_t *write_buf, uint32_t write_size) { // Read all data in one go @@ -457,14 +518,13 @@ void test_read_after_random_page_full_write(void) { TEST_ASSERT_EQUAL(inst.write.idx, 0); // Write random chunk size to the file until it filles up - static const uint16_t random_chunk_max = 8192; TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); write_buf = NULL; uint32_t allocated = 0; storfs_err_t err; uint32_t file_size = 0; do { - chunk_size = random_integer(random_chunk_max); + chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); uint32_t written = chunk_size; storfs_size_t offset = allocated; @@ -483,23 +543,7 @@ void test_read_after_random_page_full_write(void) { simple_read_helper(&inst, write_buf, file_size); // Read back random chunk size - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - storfs_size_t offset = 0; - do { - chunk_size = random_integer(random_chunk_max); - uint8_t *read_buf = (uint8_t *)malloc(chunk_size); - storfs_size_t read = chunk_size; - err = snode_read_data(fs, &inst, read_buf, &read); - if(err == STORFS_OK) { - TEST_ASSERT_EQUAL(read, chunk_size); - } else { - TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); - } - TEST_ASSERT_EQUAL(memcmp(&write_buf[offset], read_buf, read), 0); - offset += read; - free(read_buf); - } while(err == STORFS_OK); - + full_read_helper(&inst, write_buf, file_size, 0, true, 0); random_array_free(write_buf); TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &file_size), STORFS_OK); TEST_ASSERT_EQUAL(inst.write.idx, 0); From 764d130f275eb1056b279f8a95b252df651d8ac5 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sun, 2 Aug 2026 22:31:55 -0700 Subject: [PATCH 13/21] Get erase stuff going! --- src/snode.c | 104 ++++++++++++++++++++++--------------- src/snode.h | 5 +- test/test_snode.c | 130 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 163 insertions(+), 76 deletions(-) diff --git a/src/snode.c b/src/snode.c index 8bcbb3b..ec7ef26 100644 --- a/src/snode.c +++ b/src/snode.c @@ -64,12 +64,13 @@ snode_update(storfs_t *fs, SNode *node, storfs_page_t page) { } static inline SNodeIdxCount calc_snode_idx(const storfs_t *fs) { - const uint32_t epp = EXTENTS_PER_PAGE(fs); - const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; - const uint32_t mi_size = si_size + epp * epp; + const uint32_t extents_per_page = EXTENTS_PER_PAGE(fs); + const uint32_t single_indirect_size = DIRECT_EXTENT_SIZE + extents_per_page; + const uint32_t multiple_indirect_size = + single_indirect_size + extents_per_page * extents_per_page; return (SNodeIdxCount){ - .single = si_size, - .multiple = mi_size, + .single = single_indirect_size, + .multiple = multiple_indirect_size, }; } @@ -242,7 +243,8 @@ static storfs_err_t find_location(storfs_t *fs, SNodeInst *inst, SNodeExtentCache *cache, storfs_byte_t offset) { - cache->idx = 0; + cache->idx = 0; + cache->processed_bytes = offset; if(offset < INLINE_DATA_SIZE(fs)) { cache->offset_bytes = sizeof(SNode) + offset; @@ -364,18 +366,22 @@ snode_alloc_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t *page) { static inline storfs_page_t calculate_freed(const storfs_t *fs, const SNodeExtent *extent, - const SNodeOpInst *op) { - storfs_page_t pages_remaining = op->bytes_remaining / fs->pageSize; - storfs_page_t freed = - pages_remaining > extent->count ? extent->count : pages_remaining; - return freed; + const SNodeOpInst *op, + SNodeExtentCache cache) { + storfs_size_t extent_bytes = + extent->count * fs->pageSize + cache.offset_bytes; + storfs_page_t bytes_remaining = op->bytes_remaining; + storfs_page_t pages_remaining = bytes_remaining / fs->pageSize; + + return bytes_remaining > extent_bytes ? extent->count : pages_remaining; } static inline void erase_decrement_extent(const storfs_t *fs, SNodeExtent *extent, - const SNodeOpInst *op) { + const SNodeOpInst *op, + SNodeExtentCache cache) { // Decrement extent count by the number of pages contiguously freed - extent->count -= calculate_freed(fs, extent, op); + extent->count -= calculate_freed(fs, extent, op, cache); if(!extent->count) { extent->start = 0; } @@ -390,10 +396,11 @@ erase_snode_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t page) { return bitmap_alloc_page(fs, page, PAGE_FREE); } -static storfs_err_t process_extent_pages(storfs_t *fs, - SNodeOpInst *op, - uint32_t indirect_idx, - uint32_t *indirect_page) { +static storfs_err_t process_extent_pages(storfs_t *fs, + SNodeOpInst *op, + uint32_t indirect_idx, + uint32_t *indirect_page, + SNodeExtentCache cache) { storfs_err_t err = STORFS_OK; if(op->op == SNODE_WRITE && op->is_boundary) { @@ -413,14 +420,14 @@ static storfs_err_t process_extent_pages(storfs_t *fs, SNodeExtent *indirect_extent = &((SNodeExtent *)fs->working_buf)[indirect_idx]; - if(!op->is_boundary) { - op->extent.start = indirect_extent->start; - op->extent.count = indirect_extent->count; - return STORFS_OK; - } - switch(op->op) { case SNODE_WRITE: + if(!op->is_boundary) { + op->extent.start = indirect_extent->start; + op->extent.count = indirect_extent->count; + // We don't want to update the idirect page here + return STORFS_OK; + } indirect_extent->start = op->extent.start; indirect_extent->count = op->extent.count; break; @@ -431,7 +438,7 @@ static storfs_err_t process_extent_pages(storfs_t *fs, case SNODE_ERASE: op->extent.start = indirect_extent->start; op->extent.count = indirect_extent->count; - erase_decrement_extent(fs, indirect_extent, op); + erase_decrement_extent(fs, indirect_extent, op, cache); break; } @@ -473,7 +480,7 @@ static storfs_err_t process_direct_extents(storfs_t *fs, direct_extent->start = extent->start; direct_extent->count = extent->count; } else if(op->op == SNODE_ERASE) { - erase_decrement_extent(fs, direct_extent, op); + erase_decrement_extent(fs, direct_extent, op, cache); } if(op->op != SNODE_READ) { @@ -507,7 +514,8 @@ static storfs_err_t process_indirect_extents(storfs_t *fs, err = process_extent_pages(fs, op, single_indirect_extent_idx, - &node->indirect.single); + &node->indirect.single, + cache); if(err != STORFS_OK) { return err; } @@ -576,12 +584,13 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, err = process_extent_pages(fs, op, single_indirect_extent_idx, - &single_indirect_page); + &single_indirect_page, + cache); if(err != STORFS_OK) { return err; } - if(op->op != SNODE_READ && op->is_boundary) { + if((op->op == SNODE_WRITE && op->is_boundary) || op->op == SNODE_ERASE) { err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; @@ -592,7 +601,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, if(op->op == SNODE_WRITE) { multiple->total += op->extent.count; } else { - multiple->total -= calculate_freed(fs, &op->extent, op); + multiple->total -= calculate_freed(fs, &op->extent, op, cache); if(!multiple->total) { multiple->single_location = 0; } @@ -713,6 +722,7 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, } op->bytes_remaining -= bytes_to_process; + cache->processed_bytes += bytes_to_process; // Update offset within contiguous block. Must be computed before // location is advanced to the next page below, since it needs the @@ -748,7 +758,7 @@ static storfs_err_t erase_data(storfs_t *fs, SNodeOpInst *op, SNodeExtentCache *cache) { storfs_err_t err = STORFS_OK; - storfs_page_t freed = calculate_freed(fs, &op->extent, op); + storfs_page_t freed = calculate_freed(fs, &op->extent, op, *cache); uint32_t page_start = op->extent.start + op->extent.count - freed; uint32_t pages_erased = freed; uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; @@ -763,19 +773,16 @@ static storfs_err_t erase_data(storfs_t *fs, // Is there a partial erase needed for this extent? if(err == STORFS_OK && erase_bytes_offset && pages_erased < op->extent.count) { - // Erase page previous to the contiguous start location - page_start--; err = atomic_read(fs, page_start); if(err != STORFS_OK) { goto finish; } - // If page is SNode inline data the start of data increases by - // sizeof(SNode) - erase_bytes_offset += inst->page == page_start ? sizeof(SNode) : 0; - - uint32_t zero_size = fs->pageSize - erase_bytes_offset; - memset(&fs->working_buf[erase_bytes_offset], 0, zero_size); + uint32_t page_capacity = (inst->page == page_start) + ? fs->pageSize - sizeof(SNode) + : fs->pageSize; + uint32_t page_offset = page_capacity - erase_bytes_offset; + memset(&fs->working_buf[page_offset], 0, op->bytes_remaining); err = atomic_write(fs, page_start); if(err != STORFS_OK) { goto finish; @@ -783,11 +790,13 @@ static storfs_err_t erase_data(storfs_t *fs, bytes_erased += op->bytes_remaining; op->bytes_remaining = 0; } + cache->processed_bytes -= bytes_erased; finish: if(err != STORFS_OK || !op->bytes_remaining) { cache->offset_bytes = op->extent.count * fs->pageSize - bytes_erased; - } else { + } else if(op->bytes_remaining || + cache->offset_bytes == op->extent.count * fs->pageSize) { cache->idx = cache->idx ? cache->idx - 1 : 0; cache->offset_bytes = 0; } @@ -901,8 +910,21 @@ snode_read_data(storfs_t *fs, SNodeInst *inst, uint8_t *data, uint32_t *size) { return STORFS_ERR_NULL_POINTER; } - SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = *size }; - return snode_perform_op(fs, inst, &op, (uint8_t *)data, size); + // Ensures that reads cannot go beyond end of SNode + storfs_err_t storage_err = STORFS_OK; + storfs_size_t remaining = inst->node.size - inst->read.processed_bytes; + if(*size > remaining) { + storage_err = STORFS_ERR_END_OF_FILE; + *size = remaining; + } + + SNodeOpInst op = { .op = SNODE_READ, .bytes_remaining = *size }; + storfs_err_t op_err = snode_perform_op(fs, inst, &op, (uint8_t *)data, size); + if(op_err != STORFS_OK) { + return op_err; + } + + return storage_err; } /*! diff --git a/src/snode.h b/src/snode.h index 35cbea3..bbd4b74 100644 --- a/src/snode.h +++ b/src/snode.h @@ -45,8 +45,9 @@ typedef struct { } SNode; typedef struct { - uint32_t offset_bytes; - uint32_t idx; + storfs_size_t processed_bytes; + uint32_t offset_bytes; + uint32_t idx; } SNodeExtentCache; typedef struct { diff --git a/test/test_snode.c b/test/test_snode.c index a12f554..3ebadec 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -155,6 +155,21 @@ static uint32_t simulate_idx_advance(uint32_t data_size, return idx_delta; } +/*! + @brief Fill an SNode + + @details Fills a complete SNode to the last double extent with random data. + Will fill extents with a specified chunk size. The chunk size does + not have to be page aligned. Will return a pointer to the data + written and the total buffer size of the data. The written data must + be freed after use. + + @param inst SNode instance to write to + @param buf_size the buffer size of the data written + @param chunk_size the size of chunks to write to the SNode. + + @return Pointer to the data written to the SNode. + */ static uint8_t * fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { uint32_t max = calc_max_idx(); @@ -190,6 +205,38 @@ fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { return buf; } +static void full_read_helper(SNodeInst *inst, + uint8_t *write_buf, + storfs_size_t file_size, + storfs_size_t read_offset, + bool random, + storfs_size_t chunk_read) { + TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); + storfs_size_t offset = 0; + storfs_err_t err; + storfs_size_t chunk_size; + do { + if(random) { + chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); + } else { + chunk_size = chunk_read; + } + + uint8_t *read_buf = (uint8_t *)malloc(chunk_size); + storfs_size_t read = chunk_size; + err = snode_read_data(fs, inst, read_buf, &read); + if(err == STORFS_OK) { + TEST_ASSERT_EQUAL(read, chunk_size); + } else { + TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); + } + bool equal = memcmp(&write_buf[offset], read_buf, read) == 0; + TEST_ASSERT_TRUE(equal); + offset += read; + free(read_buf); + } while(err == STORFS_OK); +} + void test_snode_write(void) { uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); uint8_t *write_buf = random_array(buf_size); @@ -230,7 +277,7 @@ void test_snode_write(void) { free(read_buf); } -void test_snode_write_read(void) { +void test_snode_read(void) { storfs_page_t page = 17; char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; SNodeInst inst = { 0 }; @@ -264,12 +311,60 @@ void test_snode_write_read(void) { // Test reading past boundaries uint32_t data_read = chunk_size; TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &data_read), - STORFS_ERR_NO_FREE_BLOCKS); + STORFS_ERR_END_OF_FILE); random_array_free(write_buf); free(read_buf); } +void test_snode_erase(void) { + storfs_page_t page = 17; + char name[STORFS_MAX_FILE_NAME] = FAKE_NAME "123"; + SNodeInst inst = { 0 }; + + storfs_crc16_IgnoreAndReturn(FAKE_CRC16); + + TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); + TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); + + TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); + + storfs_size_t chunk_size = fs->pageSize * 3; + uint32_t buf_size; + uint8_t *write_buf; + write_buf = fill_snode(&inst, &buf_size, chunk_size); + + // Perform partial erase not page aligned, read back data, verify it matches + const uint32_t erase_sizes[] = { + fs->pageSize * 3 / 4, + fs->pageSize * 10 / 3, + buf_size / 2, + }; + + for(uint16_t erase = 0; erase < ARRAY_SIZE(erase_sizes); erase++) { + uint32_t erase_size = erase_sizes[erase]; + uint32_t bytes = erase_size; + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, erase_size); + buf_size -= erase_size; + TEST_ASSERT_EQUAL(inst.node.size, buf_size); + full_read_helper(&inst, write_buf, buf_size, 0, true, 0); + } + + // Perform a read at a random offset to ensure reading to end works as + // expected + uint32_t offset = buf_size / 2; + full_read_helper(&inst, write_buf, buf_size, offset, true, 0); + + // Erase the rest + uint32_t bytes = buf_size; + TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); + TEST_ASSERT_EQUAL(bytes, buf_size); + + // Test erasing random sizes + // write_buf = fill_snode(&inst, &buf_size, chunk_size); +} + void test_snode_find_location(void) { uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); uint8_t *write_buf = random_array(buf_size); @@ -340,37 +435,6 @@ void test_snode_find_location(void) { STORFS_ERR_NULL_POINTER); } -static void full_read_helper(SNodeInst *inst, - uint8_t *write_buf, - storfs_size_t file_size, - storfs_size_t read_offset, - bool random, - storfs_size_t chunk_read) { - TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); - storfs_size_t offset = 0; - storfs_err_t err; - storfs_size_t chunk_size; - do { - if(random) { - chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); - } else { - chunk_size = chunk_read; - } - - uint8_t *read_buf = (uint8_t *)malloc(chunk_size); - storfs_size_t read = chunk_size; - err = snode_read_data(fs, inst, read_buf, &read); - if(err == STORFS_OK) { - TEST_ASSERT_EQUAL(read, chunk_size); - } else { - TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); - } - TEST_ASSERT_EQUAL(memcmp(&write_buf[offset], read_buf, read), 0); - offset += read; - free(read_buf); - } while(err == STORFS_OK); -} - void test_snode_misaligned_read(void) { storfs_page_t page = 17; char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; From fa8e5734928761de1b828fa188a02b7a65d68835 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Tue, 11 Aug 2026 19:32:01 -0700 Subject: [PATCH 14/21] Some more updates for SNODE, need to determine why random erases break SNode --- src/snode.c | 31 +++++++++++++++++-------------- test/test_snode.c | 22 +++++++++++++++++++--- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/snode.c b/src/snode.c index ec7ef26..2e72140 100644 --- a/src/snode.c +++ b/src/snode.c @@ -466,11 +466,11 @@ static storfs_err_t process_direct_extents(storfs_t *fs, extent->start = direct_extent->start; extent->count = direct_extent->count; - if(!op->is_boundary) { - return STORFS_OK; - } - if(op->op == SNODE_WRITE) { + if(!op->is_boundary) { + return STORFS_OK; + } + uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); err = bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); if(err != STORFS_OK) { @@ -765,6 +765,9 @@ static storfs_err_t erase_data(storfs_t *fs, if(pages_erased) { err = bitmap_free_contiguous(fs, page_start, &pages_erased, pages_erased); + if(err != STORFS_OK) { + return err; + } } uint32_t bytes_erased = pages_erased * fs->pageSize; @@ -778,11 +781,11 @@ static storfs_err_t erase_data(storfs_t *fs, goto finish; } - uint32_t page_capacity = (inst->page == page_start) - ? fs->pageSize - sizeof(SNode) - : fs->pageSize; - uint32_t page_offset = page_capacity - erase_bytes_offset; - memset(&fs->working_buf[page_offset], 0, op->bytes_remaining); + uint32_t page_capacity = + !cache->idx ? fs->pageSize - sizeof(SNode) : fs->pageSize; + uint32_t page_offset = page_capacity - erase_bytes_offset; + uint32_t trim_bytes = MIN(op->bytes_remaining, page_capacity - page_offset); + memset(&fs->working_buf[page_offset], 0, trim_bytes); err = atomic_write(fs, page_start); if(err != STORFS_OK) { goto finish; @@ -793,12 +796,12 @@ static storfs_err_t erase_data(storfs_t *fs, cache->processed_bytes -= bytes_erased; finish: - if(err != STORFS_OK || !op->bytes_remaining) { - cache->offset_bytes = op->extent.count * fs->pageSize - bytes_erased; - } else if(op->bytes_remaining || - cache->offset_bytes == op->extent.count * fs->pageSize) { + bool erased_full_extent = bytes_erased == op->extent.count * fs->pageSize; + if(op->bytes_remaining || erased_full_extent) { cache->idx = cache->idx ? cache->idx - 1 : 0; cache->offset_bytes = 0; + } else if(err != STORFS_OK || !op->bytes_remaining) { + cache->offset_bytes = op->extent.count * fs->pageSize - bytes_erased; } return err; @@ -955,7 +958,7 @@ storfs_err_t snode_erase_data(storfs_t *fs, SNodeInst *inst, uint32_t *size) { // Do not erase past end of an snode if(*size > inst->node.size) { - return STORFS_ERR_INVALID_PARAM; + *size = inst->node.size; } if(inst->write.idx > idx.multiple) { diff --git a/test/test_snode.c b/test/test_snode.c index 3ebadec..6344d08 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -179,6 +179,7 @@ fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { *buf_size += leftover; uint8_t *buf = random_array(*buf_size); + TEST_ASSERT_EQUAL(snode_find_write_location(fs, inst), STORFS_OK); TEST_ASSERT_EQUAL(snode_write_data(fs, inst, buf, &leftover), STORFS_OK); TEST_ASSERT_EQUAL(inst->write.idx, 1); @@ -327,8 +328,6 @@ void test_snode_erase(void) { TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - storfs_size_t chunk_size = fs->pageSize * 3; uint32_t buf_size; uint8_t *write_buf; @@ -339,6 +338,7 @@ void test_snode_erase(void) { fs->pageSize * 3 / 4, fs->pageSize * 10 / 3, buf_size / 2, + 612, }; for(uint16_t erase = 0; erase < ARRAY_SIZE(erase_sizes); erase++) { @@ -362,7 +362,23 @@ void test_snode_erase(void) { TEST_ASSERT_EQUAL(bytes, buf_size); // Test erasing random sizes - // write_buf = fill_snode(&inst, &buf_size, chunk_size); + write_buf = fill_snode(&inst, &buf_size, chunk_size); + storfs_err_t err; + uint32_t i = 0; + do { + chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); + + storfs_size_t erased = chunk_size; + err = snode_erase_data(fs, &inst, &erased); + printf("Erased: %d %d\n", erased, chunk_size); + printf("Error: %d\n", i++); + TEST_ASSERT_EQUAL(err, STORFS_OK); + buf_size -= erased; + TEST_ASSERT_EQUAL(inst.node.size, buf_size); + if(i % 256) { + // full_read_helper(&inst, write_buf, buf_size, 0, true, 0); + } + } while(inst.node.size && err == STORFS_OK); } void test_snode_find_location(void) { From d4972d4806a341b493245ca8065f3ae32bb6297e Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Tue, 11 Aug 2026 19:42:12 -0700 Subject: [PATCH 15/21] Proceed with callback structure for processing extents --- src/snode.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/snode.c b/src/snode.c index 2e72140..bbc0daf 100644 --- a/src/snode.c +++ b/src/snode.c @@ -35,6 +35,17 @@ typedef struct { storfs_size_t multiple; } SNodeIdxCount; +typedef storfs_err_t (*ProcessExtentCb)(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache); + +typedef struct { + ProcessExtentCb direct; + ProcessExtentCb single; + ProcessExtentCb multiple; +} SNodeHandleExtentCbs; + static storfs_err_t snode_alloc_new_page(storfs_t *fs, storfs_page_t *page) { storfs_err_t err = bitmap_alloc(fs, page); if(err != STORFS_OK) { @@ -637,8 +648,10 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } -static storfs_err_t -get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { +static storfs_err_t get_modify_extents(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeHandleExtentCbs cbs) { SNodeExtentCache extent_cache; SNodeIdxCount idx = calc_snode_idx(fs); @@ -667,11 +680,11 @@ get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { op->is_boundary = !extent_cache.offset_bytes; if(extent_cache.idx < DIRECT_EXTENT_SIZE) { - return process_direct_extents(fs, inst, op, extent_cache); + return cbs.direct(fs, inst, op, extent_cache); } else if(extent_cache.idx < idx.single) { - return process_indirect_extents(fs, inst, op, extent_cache); + return cbs.single(fs, inst, op, extent_cache); } else if(extent_cache.idx < idx.multiple) { - return process_multiple_extents(fs, inst, op, extent_cache); + return cbs.multiple(fs, inst, op, extent_cache); } return STORFS_ERR_NO_FREE_BLOCKS; @@ -773,7 +786,7 @@ static storfs_err_t erase_data(storfs_t *fs, uint32_t bytes_erased = pages_erased * fs->pageSize; op->bytes_remaining -= bytes_erased; - // Is there a partial erase needed for this extent? + // Is there a partial page erase needed for this extent? if(err == STORFS_OK && erase_bytes_offset && pages_erased < op->extent.count) { err = atomic_read(fs, page_start); @@ -818,8 +831,13 @@ static storfs_err_t snode_perform_op(storfs_t *fs, cache = &inst->read; } + const SNodeHandleExtentCbs cbs = { + process_direct_extents, + process_indirect_extents, + process_multiple_extents, + }; while(err == STORFS_OK && op->bytes_remaining) { - err = get_modify_extents(fs, inst, op); + err = get_modify_extents(fs, inst, op, cbs); if(err != STORFS_OK) { break; } From c9475fae9fb34f5477f0b5ac961b28cf938e2f15 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Tue, 11 Aug 2026 22:02:14 -0700 Subject: [PATCH 16/21] Get tests almost passing --- src/snode.c | 238 ++++++++++++++++++++++++++++++++-------------- test/test_snode.c | 4 +- 2 files changed, 169 insertions(+), 73 deletions(-) diff --git a/src/snode.c b/src/snode.c index bbc0daf..8cfc7f4 100644 --- a/src/snode.c +++ b/src/snode.c @@ -38,7 +38,8 @@ typedef struct { typedef storfs_err_t (*ProcessExtentCb)(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, - SNodeExtentCache cache); + SNodeExtentCache cache, + void *arg); typedef struct { ProcessExtentCb direct; @@ -441,34 +442,24 @@ static storfs_err_t process_extent_pages(storfs_t *fs, } indirect_extent->start = op->extent.start; indirect_extent->count = op->extent.count; + err = atomic_write(fs, *indirect_page); break; case SNODE_READ: - op->extent.start = indirect_extent->start; - op->extent.count = indirect_extent->count; - break; case SNODE_ERASE: op->extent.start = indirect_extent->start; op->extent.count = indirect_extent->count; - erase_decrement_extent(fs, indirect_extent, op, cache); break; } - if(op->op != SNODE_READ) { - err = atomic_write(fs, *indirect_page); - } - - if(op->op == SNODE_ERASE && !indirect_idx && !indirect_extent->start) { - // If this is the first indirect index and it is empty - *indirect_page = 0; - } - return err; } static storfs_err_t process_direct_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, - SNodeExtentCache cache) { + SNodeExtentCache cache, + void *arg) { + (void)arg; SNode *node = &inst->node; SNodeExtent *extent = &op->extent; SNodeExtent *direct_extent = &node->direct[cache.idx]; @@ -490,11 +481,7 @@ static storfs_err_t process_direct_extents(storfs_t *fs, direct_extent->start = extent->start; direct_extent->count = extent->count; - } else if(op->op == SNODE_ERASE) { - erase_decrement_extent(fs, direct_extent, op, cache); - } - if(op->op != SNODE_READ) { err = snode_update(fs, node, inst->page); } @@ -504,7 +491,9 @@ static storfs_err_t process_direct_extents(storfs_t *fs, static storfs_err_t process_indirect_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, - SNodeExtentCache cache) { + SNodeExtentCache cache, + void *arg) { + (void)arg; SNode *node = &inst->node; uint32_t single_indirect_extent_idx = cache.idx - DIRECT_EXTENT_SIZE; storfs_err_t err = STORFS_OK; @@ -520,28 +509,19 @@ static storfs_err_t process_indirect_extents(storfs_t *fs, return STORFS_ERR_NOT_FOUND; } - storfs_page_t init_single_indirect = node->indirect.single; - - err = process_extent_pages(fs, - op, - single_indirect_extent_idx, - &node->indirect.single, - cache); - if(err != STORFS_OK) { - return err; - } - - if(op->op == SNODE_ERASE && !node->indirect.single && init_single_indirect) { - err = erase_snode_indirect_page(fs, inst, init_single_indirect); - } - - return err; + return process_extent_pages(fs, + op, + single_indirect_extent_idx, + &node->indirect.single, + cache); } static storfs_err_t process_multiple_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, - SNodeExtentCache cache) { + SNodeExtentCache cache, + void *arg) { + (void)arg; storfs_err_t err = STORFS_OK; SNode *node = &inst->node; const uint32_t epp = EXTENTS_PER_PAGE(fs); @@ -601,7 +581,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return err; } - if((op->op == SNODE_WRITE && op->is_boundary) || op->op == SNODE_ERASE) { + if(op->op == SNODE_WRITE && op->is_boundary) { err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; @@ -611,38 +591,11 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, SNodeMultiple *multiple = &multiple_extents[single_indirect_page_idx]; if(op->op == SNODE_WRITE) { multiple->total += op->extent.count; - } else { - multiple->total -= calculate_freed(fs, &op->extent, op, cache); - if(!multiple->total) { - multiple->single_location = 0; - } } - err = atomic_write(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; } - - if(op->op == SNODE_ERASE) { - - // Save state if multiple is empty - bool multiple_empty = !multiple->single_location; - - // Safe to free single-indirect page now that parent is on flash - if(!single_indirect_page && init_single_indirect) { - err = bitmap_alloc_page(fs, init_single_indirect, PAGE_FREE); - if(err != STORFS_OK) { - return err; - } - } - - // If the first multiple extent is empty, free it - if(!single_indirect_page_idx && multiple_empty) { - storfs_page_t init_multiple_indirect = node->indirect.multiple; - node->indirect.multiple = 0; - err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); - } - } } return err; @@ -651,7 +604,8 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, static storfs_err_t get_modify_extents(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, - SNodeHandleExtentCbs cbs) { + SNodeHandleExtentCbs cbs, + void *arg) { SNodeExtentCache extent_cache; SNodeIdxCount idx = calc_snode_idx(fs); @@ -680,11 +634,11 @@ static storfs_err_t get_modify_extents(storfs_t *fs, op->is_boundary = !extent_cache.offset_bytes; if(extent_cache.idx < DIRECT_EXTENT_SIZE) { - return cbs.direct(fs, inst, op, extent_cache); + return cbs.direct(fs, inst, op, extent_cache, arg); } else if(extent_cache.idx < idx.single) { - return cbs.single(fs, inst, op, extent_cache); + return cbs.single(fs, inst, op, extent_cache, arg); } else if(extent_cache.idx < idx.multiple) { - return cbs.multiple(fs, inst, op, extent_cache); + return cbs.multiple(fs, inst, op, extent_cache, arg); } return STORFS_ERR_NO_FREE_BLOCKS; @@ -766,6 +720,138 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, return err; } +static inline void decrement_extent(storfs_t *fs, + SNodeExtent *extent, + storfs_size_t bytes_erased) { + extent->count -= bytes_erased / fs->pageSize; + if(!extent->count) { + extent->start = 0; + } +} + +static storfs_err_t erase_direct_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + SNode *node = &inst->node; + SNodeExtent *direct_extent = &node->direct[cache.idx]; + storfs_size_t bytes_erased = *(storfs_size_t *)arg; + + decrement_extent(fs, direct_extent, bytes_erased); + return snode_update(fs, node, inst->page); +} + +static storfs_err_t erase_indirect(storfs_t *fs, + SNodeExtent *extent, + storfs_size_t idx, + storfs_size_t bytes_erased, + storfs_page_t *indirect_page) { + storfs_err_t err = atomic_read(fs, *indirect_page); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *tmp_extent = &((SNodeExtent *)fs->working_buf)[idx]; + decrement_extent(fs, tmp_extent, bytes_erased); + *extent = *tmp_extent; + + err = atomic_write(fs, *indirect_page); + if(err != STORFS_OK) { + return err; + } + + if(!idx && !extent->start) { + // If this is the first indirect index and it is empty + *indirect_page = 0; + } + + return err; +} + +static storfs_err_t erase_indirect_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + SNode *node = &inst->node; + uint32_t indirect_idx = cache.idx - DIRECT_EXTENT_SIZE; + storfs_size_t bytes_erased = *(storfs_size_t *)arg; + SNodeExtent extent; + + storfs_page_t original_single = node->indirect.single; + storfs_err_t err = erase_indirect(fs, + &extent, + indirect_idx, + bytes_erased, + &node->indirect.single); + if(err != STORFS_OK) { + return err; + } + + if(!node->indirect.single && !extent.start) { + err = erase_snode_indirect_page(fs, inst, original_single); + } + + return err; +} + +static storfs_err_t erase_multiple_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + const uint32_t epp = EXTENTS_PER_PAGE(fs); + SNode *node = &inst->node; + SNodeIdxCount idx = calc_snode_idx(fs); + storfs_size_t indirect_idx = (cache.idx - idx.single) % epp; + storfs_size_t multiple_idx = (cache.idx - idx.single) / epp; + storfs_size_t bytes_erased = *(storfs_size_t *)arg; + + storfs_err_t err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + SNodeMultiple *multiple = &((SNodeMultiple *)fs->working_buf)[multiple_idx]; + storfs_page_t single_location = multiple->single_location; + multiple->total -= bytes_erased / fs->pageSize; + if(!multiple->total) { + multiple->single_location = 0; + } + err = atomic_write(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + // Save state if multiple is empty + bool multiple_empty = !multiple->single_location; + storfs_page_t original_single = single_location; + SNodeExtent extent; + + err = + erase_indirect(fs, &extent, indirect_idx, bytes_erased, &single_location); + if(err != STORFS_OK) { + return err; + } + + // Safe to free single-indirect page now that parent is on flash + if(!extent.start && !indirect_idx) { + err = bitmap_alloc_page(fs, original_single, PAGE_FREE); + if(err != STORFS_OK) { + return err; + } + } + + // If the first multiple extent is empty, free it + if(!multiple_idx && multiple_empty) { + storfs_page_t init_multiple_indirect = node->indirect.multiple; + node->indirect.multiple = 0; + err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); + } + + return err; +} + static storfs_err_t erase_data(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, @@ -786,6 +872,18 @@ static storfs_err_t erase_data(storfs_t *fs, uint32_t bytes_erased = pages_erased * fs->pageSize; op->bytes_remaining -= bytes_erased; + if(bytes_erased) { + const SNodeHandleExtentCbs cbs = { + erase_direct_extent, + erase_indirect_extent, + erase_multiple_extent, + }; + err = get_modify_extents(fs, inst, op, cbs, &bytes_erased); + if(err != STORFS_OK) { + return err; + } + } + // Is there a partial page erase needed for this extent? if(err == STORFS_OK && erase_bytes_offset && pages_erased < op->extent.count) { @@ -837,7 +935,7 @@ static storfs_err_t snode_perform_op(storfs_t *fs, process_multiple_extents, }; while(err == STORFS_OK && op->bytes_remaining) { - err = get_modify_extents(fs, inst, op, cbs); + err = get_modify_extents(fs, inst, op, cbs, NULL); if(err != STORFS_OK) { break; } diff --git a/test/test_snode.c b/test/test_snode.c index 6344d08..4eaefef 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -370,13 +370,11 @@ void test_snode_erase(void) { storfs_size_t erased = chunk_size; err = snode_erase_data(fs, &inst, &erased); - printf("Erased: %d %d\n", erased, chunk_size); - printf("Error: %d\n", i++); TEST_ASSERT_EQUAL(err, STORFS_OK); buf_size -= erased; TEST_ASSERT_EQUAL(inst.node.size, buf_size); if(i % 256) { - // full_read_helper(&inst, write_buf, buf_size, 0, true, 0); + full_read_helper(&inst, write_buf, buf_size, 0, true, 0); } } while(inst.node.size && err == STORFS_OK); } From 7ff81e009a7b5341de608170c48cd8695ad00d5c Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Fri, 14 Aug 2026 23:02:19 -0700 Subject: [PATCH 17/21] Update when snode gets erased... --- src/snode.c | 53 +++++++++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/src/snode.c b/src/snode.c index 8cfc7f4..b0d0d97 100644 --- a/src/snode.c +++ b/src/snode.c @@ -5,6 +5,7 @@ #include "common.h" #include "crc.h" +#include #include #define INLINE_DATA_SIZE(f) (f->pageSize - sizeof(SNode)) @@ -388,17 +389,6 @@ static inline storfs_page_t calculate_freed(const storfs_t *fs, return bytes_remaining > extent_bytes ? extent->count : pages_remaining; } -static inline void erase_decrement_extent(const storfs_t *fs, - SNodeExtent *extent, - const SNodeOpInst *op, - SNodeExtentCache cache) { - // Decrement extent count by the number of pages contiguously freed - extent->count -= calculate_freed(fs, extent, op, cache); - if(!extent->count) { - extent->start = 0; - } -} - static storfs_err_t erase_snode_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t page) { storfs_err_t err = snode_update(fs, &inst->node, inst->page); @@ -814,36 +804,40 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, } SNodeMultiple *multiple = &((SNodeMultiple *)fs->working_buf)[multiple_idx]; storfs_page_t single_location = multiple->single_location; - multiple->total -= bytes_erased / fs->pageSize; - if(!multiple->total) { - multiple->single_location = 0; - } - err = atomic_write(fs, node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - - // Save state if multiple is empty - bool multiple_empty = !multiple->single_location; - storfs_page_t original_single = single_location; - SNodeExtent extent; + SNodeExtent extent; - err = - erase_indirect(fs, &extent, indirect_idx, bytes_erased, &single_location); + err = erase_indirect(fs, + &extent, + indirect_idx, + bytes_erased, + &multiple->single_location); if(err != STORFS_OK) { return err; } // Safe to free single-indirect page now that parent is on flash if(!extent.start && !indirect_idx) { - err = bitmap_alloc_page(fs, original_single, PAGE_FREE); + err = bitmap_alloc_page(fs, single_location, PAGE_FREE); if(err != STORFS_OK) { return err; } } + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + multiple = &((SNodeMultiple *)fs->working_buf)[multiple_idx]; + multiple->total -= bytes_erased / fs->pageSize; + if(!multiple->total) { + multiple->single_location = 0; + } + err = atomic_write(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } // If the first multiple extent is empty, free it - if(!multiple_idx && multiple_empty) { + if(!multiple_idx && !multiple->single_location) { storfs_page_t init_multiple_indirect = node->indirect.multiple; node->indirect.multiple = 0; err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); @@ -870,8 +864,6 @@ static storfs_err_t erase_data(storfs_t *fs, } uint32_t bytes_erased = pages_erased * fs->pageSize; - op->bytes_remaining -= bytes_erased; - if(bytes_erased) { const SNodeHandleExtentCbs cbs = { erase_direct_extent, @@ -883,6 +875,7 @@ static storfs_err_t erase_data(storfs_t *fs, return err; } } + op->bytes_remaining -= bytes_erased; // Is there a partial page erase needed for this extent? if(err == STORFS_OK && erase_bytes_offset && From 7a5fb3609d72d561787aaac2f87078695572d2f6 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Fri, 14 Aug 2026 23:47:41 -0700 Subject: [PATCH 18/21] Encapsulate some calculation functions for indices --- src/snode.c | 195 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 82 deletions(-) diff --git a/src/snode.c b/src/snode.c index b0d0d97..5e5e90c 100644 --- a/src/snode.c +++ b/src/snode.c @@ -87,6 +87,22 @@ static inline SNodeIdxCount calc_snode_idx(const storfs_t *fs) { }; } +static inline storfs_size_t calc_single_idx(storfs_size_t idx) { + return idx - DIRECT_EXTENT_SIZE; +} + +typedef struct { + storfs_size_t indirect; + storfs_size_t multiple; +} SNodeMultipleIdx; +static inline SNodeMultipleIdx calc_multiple_idx(const storfs_t *fs, + storfs_size_t idx) { + SNodeIdxCount count = calc_snode_idx(fs); + const uint32_t epp = EXTENTS_PER_PAGE(fs); + return (SNodeMultipleIdx){ (idx - count.single) % epp, + (idx - count.single) / epp }; +} + /*! @brief Create an snode @@ -427,7 +443,7 @@ static storfs_err_t process_extent_pages(storfs_t *fs, if(!op->is_boundary) { op->extent.start = indirect_extent->start; op->extent.count = indirect_extent->count; - // We don't want to update the idirect page here + // We don't want to update the indirect page here return STORFS_OK; } indirect_extent->start = op->extent.start; @@ -458,23 +474,6 @@ static storfs_err_t process_direct_extents(storfs_t *fs, extent->start = direct_extent->start; extent->count = direct_extent->count; - if(op->op == SNODE_WRITE) { - if(!op->is_boundary) { - return STORFS_OK; - } - - uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); - err = bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); - if(err != STORFS_OK) { - return err; - } - - direct_extent->start = extent->start; - direct_extent->count = extent->count; - - err = snode_update(fs, node, inst->page); - } - return err; } @@ -484,9 +483,9 @@ static storfs_err_t process_indirect_extents(storfs_t *fs, SNodeExtentCache cache, void *arg) { (void)arg; - SNode *node = &inst->node; - uint32_t single_indirect_extent_idx = cache.idx - DIRECT_EXTENT_SIZE; - storfs_err_t err = STORFS_OK; + SNode *node = &inst->node; + uint32_t idx = calc_single_idx(cache.idx); + storfs_err_t err = STORFS_OK; if(op->op == SNODE_WRITE && !node->indirect.single) { err = snode_alloc_indirect_page(fs, inst, &node->indirect.single); @@ -499,11 +498,7 @@ static storfs_err_t process_indirect_extents(storfs_t *fs, return STORFS_ERR_NOT_FOUND; } - return process_extent_pages(fs, - op, - single_indirect_extent_idx, - &node->indirect.single, - cache); + return process_extent_pages(fs, op, idx, &node->indirect.single, cache); } static storfs_err_t process_multiple_extents(storfs_t *fs, @@ -512,10 +507,9 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, SNodeExtentCache cache, void *arg) { (void)arg; - storfs_err_t err = STORFS_OK; - SNode *node = &inst->node; - const uint32_t epp = EXTENTS_PER_PAGE(fs); - const uint32_t si_size = DIRECT_EXTENT_SIZE + epp; + storfs_err_t err = STORFS_OK; + SNode *node = &inst->node; + SNodeMultipleIdx idx = calc_multiple_idx(fs, cache.idx); if(op->op == SNODE_WRITE && !node->indirect.multiple) { err = snode_alloc_indirect_page(fs, inst, &node->indirect.multiple); @@ -528,15 +522,14 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, return STORFS_ERR_NOT_FOUND; } - uint32_t single_indirect_page_idx = (cache.idx - si_size) / epp; - err = atomic_read(fs, node->indirect.multiple); + err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; } SNodeMultiple *multiple_extents = (SNodeMultiple *)fs->working_buf; storfs_page_t single_indirect_page = - multiple_extents[single_indirect_page_idx].single_location; + multiple_extents[idx.multiple].single_location; if(op->op == SNODE_WRITE && !single_indirect_page) { err = snode_alloc_new_page(fs, &single_indirect_page); if(err != STORFS_OK) { @@ -550,8 +543,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, } multiple_extents = (SNodeMultiple *)fs->working_buf; - multiple_extents[single_indirect_page_idx].single_location = - single_indirect_page; + multiple_extents[idx.multiple].single_location = single_indirect_page; err = atomic_write(fs, node->indirect.multiple); if(err != STORFS_OK) { @@ -559,14 +551,10 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, } } - uint32_t single_indirect_extent_idx = (cache.idx - si_size) % epp; - storfs_page_t init_single_indirect = single_indirect_page; + storfs_page_t init_single_indirect = single_indirect_page; - err = process_extent_pages(fs, - op, - single_indirect_extent_idx, - &single_indirect_page, - cache); + err = + process_extent_pages(fs, op, idx.indirect, &single_indirect_page, cache); if(err != STORFS_OK) { return err; } @@ -578,7 +566,7 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, } multiple_extents = (SNodeMultiple *)fs->working_buf; - SNodeMultiple *multiple = &multiple_extents[single_indirect_page_idx]; + SNodeMultiple *multiple = &multiple_extents[idx.multiple]; if(op->op == SNODE_WRITE) { multiple->total += op->extent.count; } @@ -634,16 +622,71 @@ static storfs_err_t get_modify_extents(storfs_t *fs, return STORFS_ERR_NO_FREE_BLOCKS; } -static storfs_err_t snode_read_or_write_data(storfs_t *fs, - SNodeOpInst *op, - uint8_t *data, - uint32_t size, - SNodeExtentCache *cache) { +static storfs_err_t create_direct_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + if(!op->is_boundary) { + return STORFS_OK; + } + + SNode *node = &inst->node; + SNodeExtent *extent = &op->extent; + SNodeExtent *direct_extent = &node->direct[cache.idx]; + uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); + storfs_err_t err = + bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); + if(err != STORFS_OK) { + return err; + } + + direct_extent->start = extent->start; + direct_extent->count = extent->count; + + return snode_update(fs, node, inst->page); +} + +static storfs_err_t create_indirect_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + return STORFS_OK; +} +static storfs_err_t create_multiple_extent(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + SNodeExtentCache cache, + void *arg) { + return STORFS_OK; +} + +static storfs_err_t snode_read_or_write_data(storfs_t *fs, + SNodeInst *inst, + SNodeOpInst *op, + uint8_t *data, + uint32_t size) { + SNodeExtent *extent = &op->extent; + SNodeExtentCache *cache = op->op == SNODE_WRITE ? &inst->write : &inst->read; + storfs_err_t err; + if(!extent->start && !extent->count) { + const SNodeHandleExtentCbs cbs = { + create_direct_extent, + create_indirect_extent, + create_multiple_extent, + }; + err = get_modify_extents(fs, inst, op, cbs, NULL); + if(err != STORFS_OK) { + return err; + } + } + storfs_loc_t location; location.pageLoc = op->extent.start + cache->offset_bytes / fs->pageSize; location.byteLoc = cache->offset_bytes % fs->pageSize; - storfs_err_t err = atomic_read(fs, location.pageLoc); + err = atomic_read(fs, location.pageLoc); if(err != STORFS_OK) { return err; } @@ -765,16 +808,13 @@ static storfs_err_t erase_indirect_extent(storfs_t *fs, SNodeExtentCache cache, void *arg) { SNode *node = &inst->node; - uint32_t indirect_idx = cache.idx - DIRECT_EXTENT_SIZE; + uint32_t idx = calc_single_idx(cache.idx); storfs_size_t bytes_erased = *(storfs_size_t *)arg; SNodeExtent extent; storfs_page_t original_single = node->indirect.single; - storfs_err_t err = erase_indirect(fs, - &extent, - indirect_idx, - bytes_erased, - &node->indirect.single); + storfs_err_t err = + erase_indirect(fs, &extent, idx, bytes_erased, &node->indirect.single); if(err != STORFS_OK) { return err; } @@ -791,24 +831,21 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, SNodeOpInst *op, SNodeExtentCache cache, void *arg) { - const uint32_t epp = EXTENTS_PER_PAGE(fs); - SNode *node = &inst->node; - SNodeIdxCount idx = calc_snode_idx(fs); - storfs_size_t indirect_idx = (cache.idx - idx.single) % epp; - storfs_size_t multiple_idx = (cache.idx - idx.single) / epp; - storfs_size_t bytes_erased = *(storfs_size_t *)arg; + SNode *node = &inst->node; + storfs_size_t bytes_erased = *(storfs_size_t *)arg; + SNodeMultipleIdx idx = calc_multiple_idx(fs, cache.idx); storfs_err_t err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; } - SNodeMultiple *multiple = &((SNodeMultiple *)fs->working_buf)[multiple_idx]; + SNodeMultiple *multiple = &((SNodeMultiple *)fs->working_buf)[idx.multiple]; storfs_page_t single_location = multiple->single_location; SNodeExtent extent; err = erase_indirect(fs, &extent, - indirect_idx, + idx.indirect, bytes_erased, &multiple->single_location); if(err != STORFS_OK) { @@ -816,7 +853,7 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, } // Safe to free single-indirect page now that parent is on flash - if(!extent.start && !indirect_idx) { + if(!extent.start && !idx.indirect) { err = bitmap_alloc_page(fs, single_location, PAGE_FREE); if(err != STORFS_OK) { return err; @@ -827,7 +864,7 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, if(err != STORFS_OK) { return err; } - multiple = &((SNodeMultiple *)fs->working_buf)[multiple_idx]; + multiple = &((SNodeMultiple *)fs->working_buf)[idx.multiple]; multiple->total -= bytes_erased / fs->pageSize; if(!multiple->total) { multiple->single_location = 0; @@ -837,7 +874,7 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, return err; } // If the first multiple extent is empty, free it - if(!multiple_idx && !multiple->single_location) { + if(!idx.multiple && !multiple->single_location) { storfs_page_t init_multiple_indirect = node->indirect.multiple; node->indirect.multiple = 0; err = erase_snode_indirect_page(fs, inst, init_multiple_indirect); @@ -846,15 +883,13 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, return err; } -static storfs_err_t erase_data(storfs_t *fs, - SNodeInst *inst, - SNodeOpInst *op, - SNodeExtentCache *cache) { - storfs_err_t err = STORFS_OK; - storfs_page_t freed = calculate_freed(fs, &op->extent, op, *cache); - uint32_t page_start = op->extent.start + op->extent.count - freed; - uint32_t pages_erased = freed; - uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; +static storfs_err_t erase_data(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { + storfs_err_t err = STORFS_OK; + SNodeExtentCache *cache = &inst->write; + storfs_page_t freed = calculate_freed(fs, &op->extent, op, *cache); + uint32_t page_start = op->extent.start + op->extent.count - freed; + uint32_t pages_erased = freed; + uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; if(pages_erased) { err = bitmap_free_contiguous(fs, page_start, &pages_erased, pages_erased); @@ -916,11 +951,7 @@ static storfs_err_t snode_perform_op(storfs_t *fs, SNodeOpInst *op, uint8_t *data, uint32_t *size) { - storfs_err_t err = STORFS_OK; - SNodeExtentCache *cache = &inst->write; - if(op->op == SNODE_READ) { - cache = &inst->read; - } + storfs_err_t err = STORFS_OK; const SNodeHandleExtentCbs cbs = { process_direct_extents, @@ -934,9 +965,9 @@ static storfs_err_t snode_perform_op(storfs_t *fs, } if(op->op != SNODE_ERASE) { - err = snode_read_or_write_data(fs, op, data, *size, cache); + err = snode_read_or_write_data(fs, inst, op, data, *size); } else { - err = erase_data(fs, inst, op, cache); + err = erase_data(fs, inst, op); } } From 95c4a55c80793f0ace9381244431861be5721410 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 15 Aug 2026 16:23:19 -0700 Subject: [PATCH 19/21] Break up the write operation --- src/snode.c | 220 ++++++++++++++++++++++++++-------------------------- 1 file changed, 112 insertions(+), 108 deletions(-) diff --git a/src/snode.c b/src/snode.c index 5e5e90c..7a34da9 100644 --- a/src/snode.c +++ b/src/snode.c @@ -23,7 +23,6 @@ typedef struct { SNodeExtent extent; // Allocated extent on write, filled from flash otherwise SNodeOp op; uint32_t bytes_remaining; - bool is_boundary; } SNodeOpInst; typedef struct { @@ -417,45 +416,17 @@ erase_snode_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t page) { static storfs_err_t process_extent_pages(storfs_t *fs, SNodeOpInst *op, uint32_t indirect_idx, - uint32_t *indirect_page, + uint32_t indirect_page, SNodeExtentCache cache) { - storfs_err_t err = STORFS_OK; - - if(op->op == SNODE_WRITE && op->is_boundary) { - uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); - err = - bitmap_alloc_contiguous(fs, &op->extent.start, &op->extent.count, max); - if(err != STORFS_OK) { - return err; - } - } - - err = atomic_read(fs, *indirect_page); + storfs_err_t err = atomic_read(fs, indirect_page); if(err != STORFS_OK) { return err; } SNodeExtent *indirect_extent = &((SNodeExtent *)fs->working_buf)[indirect_idx]; - - switch(op->op) { - case SNODE_WRITE: - if(!op->is_boundary) { - op->extent.start = indirect_extent->start; - op->extent.count = indirect_extent->count; - // We don't want to update the indirect page here - return STORFS_OK; - } - indirect_extent->start = op->extent.start; - indirect_extent->count = op->extent.count; - err = atomic_write(fs, *indirect_page); - break; - case SNODE_READ: - case SNODE_ERASE: - op->extent.start = indirect_extent->start; - op->extent.count = indirect_extent->count; - break; - } + op->extent.start = indirect_extent->start; + op->extent.count = indirect_extent->count; return err; } @@ -487,18 +458,11 @@ static storfs_err_t process_indirect_extents(storfs_t *fs, uint32_t idx = calc_single_idx(cache.idx); storfs_err_t err = STORFS_OK; - if(op->op == SNODE_WRITE && !node->indirect.single) { - err = snode_alloc_indirect_page(fs, inst, &node->indirect.single); - if(err != STORFS_OK) { - return err; - } - } - if(!node->indirect.single) { - return STORFS_ERR_NOT_FOUND; + return STORFS_OK; } - return process_extent_pages(fs, op, idx, &node->indirect.single, cache); + return process_extent_pages(fs, op, idx, node->indirect.single, cache); } static storfs_err_t process_multiple_extents(storfs_t *fs, @@ -507,22 +471,14 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, SNodeExtentCache cache, void *arg) { (void)arg; - storfs_err_t err = STORFS_OK; SNode *node = &inst->node; SNodeMultipleIdx idx = calc_multiple_idx(fs, cache.idx); - if(op->op == SNODE_WRITE && !node->indirect.multiple) { - err = snode_alloc_indirect_page(fs, inst, &node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - } - if(!node->indirect.multiple) { - return STORFS_ERR_NOT_FOUND; + return STORFS_OK; } - err = atomic_read(fs, node->indirect.multiple); + storfs_err_t err = atomic_read(fs, node->indirect.multiple); if(err != STORFS_OK) { return err; } @@ -530,53 +486,13 @@ static storfs_err_t process_multiple_extents(storfs_t *fs, SNodeMultiple *multiple_extents = (SNodeMultiple *)fs->working_buf; storfs_page_t single_indirect_page = multiple_extents[idx.multiple].single_location; - if(op->op == SNODE_WRITE && !single_indirect_page) { - err = snode_alloc_new_page(fs, &single_indirect_page); - if(err != STORFS_OK) { - return err; - } - - // Must re-read indirect multiple as snode_alloc_new_page clobers buffer - err = atomic_read(fs, node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - - multiple_extents = (SNodeMultiple *)fs->working_buf; - multiple_extents[idx.multiple].single_location = single_indirect_page; - - err = atomic_write(fs, node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - } - storfs_page_t init_single_indirect = single_indirect_page; - err = - process_extent_pages(fs, op, idx.indirect, &single_indirect_page, cache); - if(err != STORFS_OK) { - return err; - } - - if(op->op == SNODE_WRITE && op->is_boundary) { - err = atomic_read(fs, node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - - multiple_extents = (SNodeMultiple *)fs->working_buf; - SNodeMultiple *multiple = &multiple_extents[idx.multiple]; - if(op->op == SNODE_WRITE) { - multiple->total += op->extent.count; - } - err = atomic_write(fs, node->indirect.multiple); - if(err != STORFS_OK) { - return err; - } - } - - return err; + return process_extent_pages(fs, + op, + idx.indirect, + single_indirect_page, + cache); } static storfs_err_t get_modify_extents(storfs_t *fs, @@ -607,10 +523,6 @@ static storfs_err_t get_modify_extents(storfs_t *fs, // Subtract 1 as 1 is inline page extent_cache.idx -= 1; - // Only update the extent if it is on a byte boundary, else the current - // block must be processed - op->is_boundary = !extent_cache.offset_bytes; - if(extent_cache.idx < DIRECT_EXTENT_SIZE) { return cbs.direct(fs, inst, op, extent_cache, arg); } else if(extent_cache.idx < idx.single) { @@ -627,10 +539,6 @@ static storfs_err_t create_direct_extent(storfs_t *fs, SNodeOpInst *op, SNodeExtentCache cache, void *arg) { - if(!op->is_boundary) { - return STORFS_OK; - } - SNode *node = &inst->node; SNodeExtent *extent = &op->extent; SNodeExtent *direct_extent = &node->direct[cache.idx]; @@ -647,19 +555,112 @@ static storfs_err_t create_direct_extent(storfs_t *fs, return snode_update(fs, node, inst->page); } +static storfs_err_t create_indirect(storfs_t *fs, + SNodeOpInst *op, + SNodeExtent *extent, + storfs_size_t idx, + storfs_page_t indirect_page) { + uint32_t max = CALC_CONTIGUOUS_MAX(fs, op); + storfs_err_t err = + bitmap_alloc_contiguous(fs, &extent->start, &extent->count, max); + if(err != STORFS_OK) { + return err; + } + + err = atomic_read(fs, indirect_page); + if(err != STORFS_OK) { + return err; + } + + SNodeExtent *indirect_extent = &((SNodeExtent *)fs->working_buf)[idx]; + indirect_extent->start = extent->start; + indirect_extent->count = extent->count; + + return atomic_write(fs, indirect_page); +} + static storfs_err_t create_indirect_extent(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, SNodeExtentCache cache, void *arg) { - return STORFS_OK; + SNode *node = &inst->node; + SNodeExtent *extent = &op->extent; + uint32_t idx = calc_single_idx(cache.idx); + + if(!node->indirect.single) { + storfs_err_t err = + snode_alloc_indirect_page(fs, inst, &node->indirect.single); + if(err != STORFS_OK) { + return err; + } + } + + return create_indirect(fs, op, extent, idx, node->indirect.single); } static storfs_err_t create_multiple_extent(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op, SNodeExtentCache cache, void *arg) { - return STORFS_OK; + SNode *node = &inst->node; + SNodeExtent *extent = &op->extent; + SNodeMultipleIdx idx = calc_multiple_idx(fs, cache.idx); + storfs_err_t err; + + if(!node->indirect.multiple) { + err = snode_alloc_indirect_page(fs, inst, &node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + } + + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + SNodeMultiple *multiple_extents = (SNodeMultiple *)fs->working_buf; + storfs_page_t single_indirect_page = + multiple_extents[idx.multiple].single_location; + if(!single_indirect_page) { + err = snode_alloc_new_page(fs, &single_indirect_page); + if(err != STORFS_OK) { + return err; + } + + // Must re-read indirect multiple as snode_alloc_new_page clobbers buffer + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + multiple_extents = (SNodeMultiple *)fs->working_buf; + multiple_extents[idx.multiple].single_location = single_indirect_page; + + err = atomic_write(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + } + + err = create_indirect(fs, op, extent, idx.indirect, single_indirect_page); + if(err != STORFS_OK) { + return err; + } + + // Multiple got clobbered again... re-read + err = atomic_read(fs, node->indirect.multiple); + if(err != STORFS_OK) { + return err; + } + + multiple_extents = (SNodeMultiple *)fs->working_buf; + SNodeMultiple *multiple = &multiple_extents[idx.multiple]; + if(op->op == SNODE_WRITE) { + multiple->total += op->extent.count; + } + return atomic_write(fs, node->indirect.multiple); } static storfs_err_t snode_read_or_write_data(storfs_t *fs, @@ -670,7 +671,10 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, SNodeExtent *extent = &op->extent; SNodeExtentCache *cache = op->op == SNODE_WRITE ? &inst->write : &inst->read; storfs_err_t err; - if(!extent->start && !extent->count) { + + // Only update the extent if it is on a byte boundary, else the current + // block must be processed + if(op->op == SNODE_WRITE && !cache->offset_bytes) { const SNodeHandleExtentCbs cbs = { create_direct_extent, create_indirect_extent, From d702dc56c90421840e0fb24c530285227cccd214 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Sat, 15 Aug 2026 16:24:02 -0700 Subject: [PATCH 20/21] Silly little conditional removal --- src/snode.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/snode.c b/src/snode.c index 7a34da9..90c28a9 100644 --- a/src/snode.c +++ b/src/snode.c @@ -657,9 +657,7 @@ static storfs_err_t create_multiple_extent(storfs_t *fs, multiple_extents = (SNodeMultiple *)fs->working_buf; SNodeMultiple *multiple = &multiple_extents[idx.multiple]; - if(op->op == SNODE_WRITE) { - multiple->total += op->extent.count; - } + multiple->total += op->extent.count; return atomic_write(fs, node->indirect.multiple); } From 632cc2da5c0c01ee7e9a770edaf234eb9c703121 Mon Sep 17 00:00:00 2001 From: Matthew Krause Date: Mon, 17 Aug 2026 21:54:43 -0700 Subject: [PATCH 21/21] Getting closer... --- src/snode.c | 117 +++--- test/fake_flash.c | 14 + test/fake_flash.h | 3 + test/test_snode.c | 902 ++++++++++++++++++++++++---------------------- 4 files changed, 556 insertions(+), 480 deletions(-) diff --git a/src/snode.c b/src/snode.c index 90c28a9..74c3ea6 100644 --- a/src/snode.c +++ b/src/snode.c @@ -392,16 +392,15 @@ snode_alloc_indirect_page(storfs_t *fs, SNodeInst *inst, storfs_page_t *page) { return snode_update(fs, &inst->node, inst->page); } -static inline storfs_page_t calculate_freed(const storfs_t *fs, - const SNodeExtent *extent, - const SNodeOpInst *op, - SNodeExtentCache cache) { +static inline storfs_size_t calculate_freed_bytes(const storfs_t *fs, + const SNodeExtent *extent, + const SNodeOpInst *op, + SNodeExtentCache cache) { storfs_size_t extent_bytes = - extent->count * fs->pageSize + cache.offset_bytes; - storfs_page_t bytes_remaining = op->bytes_remaining; - storfs_page_t pages_remaining = bytes_remaining / fs->pageSize; + cache.offset_bytes ? cache.offset_bytes : extent->count * fs->pageSize; + storfs_size_t bytes_remaining = op->bytes_remaining; - return bytes_remaining > extent_bytes ? extent->count : pages_remaining; + return bytes_remaining > extent_bytes ? extent_bytes : bytes_remaining; } static storfs_err_t @@ -757,8 +756,8 @@ static storfs_err_t snode_read_or_write_data(storfs_t *fs, static inline void decrement_extent(storfs_t *fs, SNodeExtent *extent, - storfs_size_t bytes_erased) { - extent->count -= bytes_erased / fs->pageSize; + storfs_page_t pages_erased) { + extent->count -= pages_erased; if(!extent->count) { extent->start = 0; } @@ -771,36 +770,39 @@ static storfs_err_t erase_direct_extent(storfs_t *fs, void *arg) { SNode *node = &inst->node; SNodeExtent *direct_extent = &node->direct[cache.idx]; - storfs_size_t bytes_erased = *(storfs_size_t *)arg; + storfs_page_t pages_erased = *(storfs_page_t *)arg; - decrement_extent(fs, direct_extent, bytes_erased); + decrement_extent(fs, direct_extent, pages_erased); return snode_update(fs, node, inst->page); } -static storfs_err_t erase_indirect(storfs_t *fs, - SNodeExtent *extent, - storfs_size_t idx, - storfs_size_t bytes_erased, - storfs_page_t *indirect_page) { - storfs_err_t err = atomic_read(fs, *indirect_page); +static storfs_err_t erase_indirect(storfs_t *fs, + SNodeExtent *extent, + storfs_size_t idx, + storfs_page_t pages_erased, + storfs_page_t indirect_page) { + storfs_err_t err = atomic_read(fs, indirect_page); if(err != STORFS_OK) { return err; } SNodeExtent *tmp_extent = &((SNodeExtent *)fs->working_buf)[idx]; - decrement_extent(fs, tmp_extent, bytes_erased); + printf("Before: Indirect_page %d, start: %d, finish: %d\n", + indirect_page, + tmp_extent->start, + tmp_extent->count); + decrement_extent(fs, tmp_extent, pages_erased); *extent = *tmp_extent; - err = atomic_write(fs, *indirect_page); + printf("After: Indirect_page %d, start: %d, finish: %d\n", + indirect_page, + tmp_extent->start, + tmp_extent->count); + err = atomic_write(fs, indirect_page); if(err != STORFS_OK) { return err; } - if(!idx && !extent->start) { - // If this is the first indirect index and it is empty - *indirect_page = 0; - } - return err; } @@ -811,18 +813,17 @@ static storfs_err_t erase_indirect_extent(storfs_t *fs, void *arg) { SNode *node = &inst->node; uint32_t idx = calc_single_idx(cache.idx); - storfs_size_t bytes_erased = *(storfs_size_t *)arg; + storfs_page_t pages_erased = *(storfs_page_t *)arg; SNodeExtent extent; - storfs_page_t original_single = node->indirect.single; - storfs_err_t err = - erase_indirect(fs, &extent, idx, bytes_erased, &node->indirect.single); + storfs_err_t err = + erase_indirect(fs, &extent, idx, pages_erased, node->indirect.single); if(err != STORFS_OK) { return err; } - if(!node->indirect.single && !extent.start) { - err = erase_snode_indirect_page(fs, inst, original_single); + if(!extent.count && !extent.start) { + err = erase_snode_indirect_page(fs, inst, node->indirect.single); } return err; @@ -834,7 +835,7 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, SNodeExtentCache cache, void *arg) { SNode *node = &inst->node; - storfs_size_t bytes_erased = *(storfs_size_t *)arg; + storfs_page_t pages_erased = *(storfs_page_t *)arg; SNodeMultipleIdx idx = calc_multiple_idx(fs, cache.idx); storfs_err_t err = atomic_read(fs, node->indirect.multiple); @@ -844,12 +845,12 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, SNodeMultiple *multiple = &((SNodeMultiple *)fs->working_buf)[idx.multiple]; storfs_page_t single_location = multiple->single_location; SNodeExtent extent; + printf("Before erase: Indirect_page %d, idx %d\n", + single_location, + idx.multiple); - err = erase_indirect(fs, - &extent, - idx.indirect, - bytes_erased, - &multiple->single_location); + err = + erase_indirect(fs, &extent, idx.indirect, pages_erased, single_location); if(err != STORFS_OK) { return err; } @@ -867,7 +868,7 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, return err; } multiple = &((SNodeMultiple *)fs->working_buf)[idx.multiple]; - multiple->total -= bytes_erased / fs->pageSize; + multiple->total -= pages_erased; if(!multiple->total) { multiple->single_location = 0; } @@ -886,12 +887,23 @@ static storfs_err_t erase_multiple_extent(storfs_t *fs, } static storfs_err_t erase_data(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { - storfs_err_t err = STORFS_OK; - SNodeExtentCache *cache = &inst->write; - storfs_page_t freed = calculate_freed(fs, &op->extent, op, *cache); - uint32_t page_start = op->extent.start + op->extent.count - freed; - uint32_t pages_erased = freed; - uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; + storfs_err_t err = STORFS_OK; + SNodeExtentCache *cache = &inst->write; + storfs_size_t bytes_erased = + calculate_freed_bytes(fs, &op->extent, op, *cache); + storfs_page_t pages_erased = bytes_erased / fs->pageSize; + uint32_t page_start = op->extent.start + op->extent.count - pages_erased; + uint32_t erase_bytes_offset = op->bytes_remaining % fs->pageSize; + printf("Pages erased: %d, bytes_erased: %d, page_start: %d, page offset %d, " + "idx %d, op remaining %d, extent start %d, extent count %d\n", + pages_erased, + bytes_erased, + page_start, + inst->write.offset_bytes, + inst->write.idx, + op->bytes_remaining, + op->extent.start, + op->extent.count); if(pages_erased) { err = bitmap_free_contiguous(fs, page_start, &pages_erased, pages_erased); @@ -900,23 +912,21 @@ static storfs_err_t erase_data(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { } } - uint32_t bytes_erased = pages_erased * fs->pageSize; - if(bytes_erased) { + if(pages_erased) { const SNodeHandleExtentCbs cbs = { erase_direct_extent, erase_indirect_extent, erase_multiple_extent, }; - err = get_modify_extents(fs, inst, op, cbs, &bytes_erased); + err = get_modify_extents(fs, inst, op, cbs, &pages_erased); if(err != STORFS_OK) { return err; } } - op->bytes_remaining -= bytes_erased; - // Is there a partial page erase needed for this extent? if(err == STORFS_OK && erase_bytes_offset && pages_erased < op->extent.count) { + page_start--; err = atomic_read(fs, page_start); if(err != STORFS_OK) { goto finish; @@ -926,22 +936,21 @@ static storfs_err_t erase_data(storfs_t *fs, SNodeInst *inst, SNodeOpInst *op) { !cache->idx ? fs->pageSize - sizeof(SNode) : fs->pageSize; uint32_t page_offset = page_capacity - erase_bytes_offset; uint32_t trim_bytes = MIN(op->bytes_remaining, page_capacity - page_offset); + printf("Erasing page at: %d, num_bytes: %d\n", page_start, trim_bytes); memset(&fs->working_buf[page_offset], 0, trim_bytes); err = atomic_write(fs, page_start); if(err != STORFS_OK) { goto finish; } - bytes_erased += op->bytes_remaining; - op->bytes_remaining = 0; } cache->processed_bytes -= bytes_erased; + op->bytes_remaining -= bytes_erased; finish: - bool erased_full_extent = bytes_erased == op->extent.count * fs->pageSize; - if(op->bytes_remaining || erased_full_extent) { + if(op->bytes_remaining) { cache->idx = cache->idx ? cache->idx - 1 : 0; cache->offset_bytes = 0; - } else if(err != STORFS_OK || !op->bytes_remaining) { + } else if(err == STORFS_OK) { cache->offset_bytes = op->extent.count * fs->pageSize - bytes_erased; } diff --git a/test/fake_flash.c b/test/fake_flash.c index 0f3f327..008d298 100644 --- a/test/fake_flash.c +++ b/test/fake_flash.c @@ -114,3 +114,17 @@ void fake_storfs_fail_op(FlashOperation op, bool fail, uint32_t count) { ctx.fail_tracker[op] = 0; } } + +void fake_storfs_read_page_raw(storfs_page_t page, + uint8_t *buf, + storfs_size_t size) { + if(page >= fake_storfs_get_page_count() || !buf || !size) { + return; + } + + storfs_size_t page_size = fake_storfs_get_page_size(); + storfs_size_t read_size = size < page_size ? size : page_size; + storfs_page_t page_offset = page * page_size; + + memcpy(buf, &ctx.flash_sim[page_offset], read_size); +} diff --git a/test/fake_flash.h b/test/fake_flash.h index dd51bab..0110567 100644 --- a/test/fake_flash.h +++ b/test/fake_flash.h @@ -15,5 +15,8 @@ storfs_t *fake_storfs_init(void); storfs_page_t fake_storfs_get_page_count(void); storfs_byte_t fake_storfs_get_page_size(void); void fake_storfs_fail_op(FlashOperation op, bool fail, uint32_t count); +void fake_storfs_read_page_raw(storfs_page_t page, + uint8_t *buf, + storfs_size_t size); #endif diff --git a/test/test_snode.c b/test/test_snode.c index 4eaefef..91e1fb7 100644 --- a/test/test_snode.c +++ b/test/test_snode.c @@ -206,6 +206,20 @@ fill_snode(SNodeInst *inst, storfs_size_t *buf_size, storfs_size_t chunk_size) { return buf; } +/*! + @brief Perform a full read on the SNode and compare to the write data + + @details Iteratively performs the read across the file in random or fix sized + chunks and compares each read chunk to the buffer written to the + file. + + @param inst SNode instance to write to + @param write_buf pointer to buffer that was written to file + @param file_size the size of the file, and write_buf, in bytes + @param read_offset offset to begin performing reads on + @param random whether the read chunks should be of random size + @param chunk_read if not random, the chunk size to read in bytes + */ static void full_read_helper(SNodeInst *inst, uint8_t *write_buf, storfs_size_t file_size, @@ -331,40 +345,61 @@ void test_snode_erase(void) { storfs_size_t chunk_size = fs->pageSize * 3; uint32_t buf_size; uint8_t *write_buf; - write_buf = fill_snode(&inst, &buf_size, chunk_size); + // write_buf = fill_snode(&inst, &buf_size, chunk_size); // Perform partial erase not page aligned, read back data, verify it matches - const uint32_t erase_sizes[] = { - fs->pageSize * 3 / 4, - fs->pageSize * 10 / 3, - buf_size / 2, - 612, - }; - - for(uint16_t erase = 0; erase < ARRAY_SIZE(erase_sizes); erase++) { - uint32_t erase_size = erase_sizes[erase]; - uint32_t bytes = erase_size; - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); - TEST_ASSERT_EQUAL(bytes, erase_size); - buf_size -= erase_size; - TEST_ASSERT_EQUAL(inst.node.size, buf_size); - full_read_helper(&inst, write_buf, buf_size, 0, true, 0); + // const uint32_t erase_sizes[] = { + // fs->pageSize * 3 / 4, + // fs->pageSize * 10 / 3, + // buf_size / 2, + // 612, + //}; + + // for(uint16_t erase = 0; erase < ARRAY_SIZE(erase_sizes); erase++) { + // uint32_t erase_size = erase_sizes[erase]; + // uint32_t bytes = erase_size; + // TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); + // TEST_ASSERT_EQUAL(bytes, erase_size); + // buf_size -= erase_size; + // TEST_ASSERT_EQUAL(inst.node.size, buf_size); + // full_read_helper(&inst, write_buf, buf_size, 0, true, 0); + // } + + //// Perform a read at a random offset to ensure reading to end works as + //// expected + // uint32_t offset = buf_size / 2; + // full_read_helper(&inst, write_buf, buf_size, offset, true, 0); + + //// Erase the rest + // uint32_t bytes = buf_size; + // TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); + // TEST_ASSERT_EQUAL(bytes, buf_size); + + storfs_size_t page_size = fake_storfs_get_page_size(); + uint8_t page_buf[page_size]; + for(uint8_t i = 0; i < page; i++) { + fake_storfs_read_page_raw(i, page_buf, page_size); + for(storfs_size_t j = 0; j < page_size; j++) { + printf("0x%X ", page_buf[j]); + } + printf("\n"); } - - // Perform a read at a random offset to ensure reading to end works as - // expected - uint32_t offset = buf_size / 2; - full_read_helper(&inst, write_buf, buf_size, offset, true, 0); - - // Erase the rest - uint32_t bytes = buf_size; - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &bytes), STORFS_OK); - TEST_ASSERT_EQUAL(bytes, buf_size); + printf("END HERE\n\n\n\n"); // Test erasing random sizes write_buf = fill_snode(&inst, &buf_size, chunk_size); storfs_err_t err; uint32_t i = 0; + + for(uint8_t i = 0; i < page; i++) { + fake_storfs_read_page_raw(i, page_buf, page_size); + for(storfs_size_t j = 0; j < page_size; j++) { + printf("0x%X ", page_buf[j]); + } + printf("\n"); + } + printf("END HERE\n\n\n\n"); + do { chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); @@ -377,408 +412,423 @@ void test_snode_erase(void) { full_read_helper(&inst, write_buf, buf_size, 0, true, 0); } } while(inst.node.size && err == STORFS_OK); -} - -void test_snode_find_location(void) { - uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); - uint8_t *write_buf = random_array(buf_size); - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - - uint32_t bytes = buf_size; - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), STORFS_OK); - TEST_ASSERT_EQUAL(bytes, buf_size); - - // Test obtaining the location now, will be in indirect extents which has yet - // to be created - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NOT_FOUND); - - // Erase and then fill the SNode all the way up - bytes = buf_size; - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(bytes, buf_size); - TEST_ASSERT_EQUAL(inst.node.size, 0); - - random_array_free(write_buf); - storfs_size_t chunk_size = fs->pageSize; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - - // Now compare after filling - uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); - bytes = buf_size; - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &buf_size), - STORFS_ERR_END_OF_FILE); - TEST_ASSERT_EQUAL(bytes, buf_size); - TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); - free(read_buf); - - // Test obtaining the location now, will be at the very end of the SNode - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_ERR_NO_SPACE); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size), - STORFS_ERR_NO_SPACE); - - // Test getting the data almost at the end of the file - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), - STORFS_OK); - - bytes = buf_size; - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), - STORFS_ERR_NO_FREE_BLOCKS); - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.node.size, 0); - TEST_ASSERT_EQUAL(inst.write.idx, 0); - - // Test improper parameters - TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), - STORFS_ERR_NULL_POINTER); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), - STORFS_ERR_NULL_POINTER); -} - -void test_snode_misaligned_read(void) { - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - - storfs_size_t chunk_size = fs->pageSize * 6; - uint32_t buf_size; - uint8_t *write_buf; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - - uint32_t read_offset = fs->pageSize * 2; - uint32_t read_size = fs->pageSize * 3; - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, read_offset), - STORFS_OK); - full_read_helper(&inst, write_buf, buf_size, read_offset, false, read_size); - - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.write.idx, 0); -} - -static void -simple_read_helper(SNodeInst *inst, uint8_t *write_buf, uint32_t write_size) { - // Read all data in one go - uint8_t *read_buf = (uint8_t *)calloc(write_size, sizeof(uint8_t)); - uint32_t bytes = write_size; - TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, inst, read_buf, &bytes), - STORFS_ERR_END_OF_FILE); - TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, write_size), 0); - free(read_buf); -} - -void test_read_after_full_write(void) { - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - - storfs_size_t chunk_size = fs->pageSize; - uint32_t buf_size; - uint8_t *write_buf; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - - // Read all of the data to make sure it matches - uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, compare); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - } - - random_array_free(write_buf); - free(read_buf); -} - -void test_read_after_full_write_update_cache(void) { - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - storfs_size_t chunk_size = fs->pageSize; - uint32_t buf_size; - uint8_t *write_buf; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - - // Read all of the data to make sure it matches - uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); - storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, compare); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - } - free(read_buf); - - simple_read_helper(&inst, write_buf, buf_size); - - random_array_free(write_buf); -} - -void test_read_after_random_page_full_write(void) { - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - - storfs_size_t chunk_size = fs->pageSize * 5; - uint32_t buf_size; - uint8_t *write_buf; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - simple_read_helper(&inst, write_buf, buf_size); - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.write.idx, 0); - - chunk_size = fs->pageSize * 12; - write_buf = fill_snode(&inst, &buf_size, chunk_size); - simple_read_helper(&inst, write_buf, buf_size); - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.write.idx, 0); - - // Offset chunk not exactly on a byte boundary - chunk_size = (storfs_size_t)((float)fs->pageSize * 5.25); - write_buf = fill_snode(&inst, &buf_size, chunk_size); - simple_read_helper(&inst, write_buf, buf_size); - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.write.idx, 0); - - // Write random chunk size to the file until it filles up - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - write_buf = NULL; - uint32_t allocated = 0; - storfs_err_t err; - uint32_t file_size = 0; - do { - chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); - uint32_t written = chunk_size; - storfs_size_t offset = allocated; - - allocated += chunk_size; - write_buf = realloc(write_buf, allocated); - - err = snode_write_data(fs, &inst, &write_buf[offset], &written); - if(err == STORFS_OK) { - TEST_ASSERT_EQUAL(written, chunk_size); - } else { - TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); - // The total file size is equal to the last data written - file_size = allocated - (chunk_size - written); + for(uint8_t i = 0; i < page; i++) { + fake_storfs_read_page_raw(i, page_buf, page_size); + for(storfs_size_t j = 0; j < page_size; j++) { + printf("0x%X ", page_buf[j]); } - } while(err == STORFS_OK); - simple_read_helper(&inst, write_buf, file_size); - - // Read back random chunk size - full_read_helper(&inst, write_buf, file_size, 0, true, 0); - random_array_free(write_buf); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &file_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst.write.idx, 0); -} - -void test_fill_stagger(void) { - uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); - uint8_t *write_buf = random_array(buf_size); - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - - storfs_size_t chunk_size = fs->pageSize; - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - uint32_t idx_compare = i / chunk_size + 1; - - storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); - storfs_err_t err_compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, err_compare); - TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); - // Test that finding the location is also correct - snode_find_write_location(fs, &inst); - TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); - } - - // Read all of the data to make sure it matches - uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, compare); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - } - - random_array_free(write_buf); - free(read_buf); -} - -void test_fill_boundary(void) { - storfs_page_t page = 17; - char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; - SNodeInst inst = { 0 }; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); - - // Test writing on boundary - storfs_size_t chunk_size = fs->pageSize; - uint32_t buf_size = chunk_size * calc_max_idx(); - uint32_t leftover = fs->pageSize - sizeof(SNode); - buf_size += leftover; - uint8_t *write_buf = random_array(buf_size); - - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &leftover), - STORFS_OK); - - for(uint32_t i = leftover; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = chunk_size; - - storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], &data_size); - storfs_err_t err_compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, err_compare); - - const uint32_t idx_compare = i / chunk_size + 2; - TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); - - // Test obtaining write location works as expected as well - snode_find_write_location(fs, &inst); - TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); - } - - // Read all of the data to make sure it matches - uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); - storfs_err_t compare = get_err_compare(data_remain, chunk_size); - TEST_ASSERT_EQUAL(err, compare); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); + printf("\n"); } - - random_array_free(write_buf); - free(read_buf); } -void test_snode_write_read_alternate(void) { - const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); - uint8_t *write_buf = random_array(buf_size); - uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); - storfs_page_t snode_1_page = 17; - storfs_page_t snode_2_page = snode_1_page++; - char snode_1_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; - char snode_2_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; - - storfs_crc16_IgnoreAndReturn(FAKE_CRC16); - - TEST_ASSERT_EQUAL( - snode_create(fs, snode_1_name, &snode_1_page, SNODE_TYPE_FILE), - STORFS_OK); - TEST_ASSERT_EQUAL( - snode_create(fs, snode_2_name, &snode_2_page, SNODE_TYPE_FILE), - STORFS_OK); - - // Write all the data from the file, read it and compare - SNodeInst inst_1 = { 0 }; - SNodeInst inst_2 = { 0 }; - TEST_ASSERT_EQUAL(snode_lookup(fs, snode_1_page, &inst_1), STORFS_OK); - TEST_ASSERT_EQUAL(snode_lookup(fs, snode_2_page, &inst_2), STORFS_OK); - - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_1), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_1, 0), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_2), STORFS_OK); - TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_2, 0), STORFS_OK); - - // Chunk read it to check offset paramter - uint32_t chunk_size = 693; - for(uint32_t i = 0; i < buf_size; i += chunk_size) { - uint32_t data_remain = buf_size - i; - uint32_t data_size = MIN(data_remain, chunk_size); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], &data_size), - STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, &data_size), - STORFS_OK); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], &data_size), - STORFS_OK); - TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, &data_size), - STORFS_OK); - TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); - } - TEST_ASSERT_EQUAL(inst_1.node.size, buf_size); - TEST_ASSERT_EQUAL(inst_2.node.size, buf_size); - - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_1, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst_1.node.size, 0); - TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, &buf_size), STORFS_OK); - TEST_ASSERT_EQUAL(inst_2.node.size, 0); - - random_array_free(write_buf); - free(read_buf); -} +// void test_snode_find_location(void) { +// uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); +// uint8_t *write_buf = random_array(buf_size); +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// uint32_t bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), +// STORFS_OK); TEST_ASSERT_EQUAL(bytes, buf_size); +// +// // Test obtaining the location now, will be in indirect extents which has +// yet +// // to be created +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), +// STORFS_ERR_NOT_FOUND); +// +// // Erase and then fill the SNode all the way up +// bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(bytes, buf_size); +// TEST_ASSERT_EQUAL(inst.node.size, 0); +// +// random_array_free(write_buf); +// storfs_size_t chunk_size = fs->pageSize; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// +// // Now compare after filling +// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); +// bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_read_data(fs, &inst, read_buf, &buf_size), +// STORFS_ERR_END_OF_FILE); +// TEST_ASSERT_EQUAL(bytes, buf_size); +// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, buf_size), 0); +// free(read_buf); +// +// // Test obtaining the location now, will be at the very end of the SNode +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), +// STORFS_ERR_NO_SPACE); TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, +// buf_size), +// STORFS_ERR_NO_SPACE); +// +// // Test getting the data almost at the end of the file +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, buf_size - 1), +// STORFS_OK); +// +// bytes = buf_size; +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &bytes), +// STORFS_ERR_NO_FREE_BLOCKS); +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.node.size, 0); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// +// // Test improper parameters +// TEST_ASSERT_EQUAL(snode_find_write_location(NULL, &inst), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, NULL), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_read_location(NULL, &inst, 0), +// STORFS_ERR_NULL_POINTER); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, NULL, 0), +// STORFS_ERR_NULL_POINTER); +// } +// +// void test_snode_misaligned_read(void) { +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// +// storfs_size_t chunk_size = fs->pageSize * 6; +// uint32_t buf_size; +// uint8_t *write_buf; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// +// uint32_t read_offset = fs->pageSize * 2; +// uint32_t read_size = fs->pageSize * 3; +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, read_offset), +// STORFS_OK); +// full_read_helper(&inst, write_buf, buf_size, read_offset, false, +// read_size); +// +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// } +// +// static void +// simple_read_helper(SNodeInst *inst, uint8_t *write_buf, uint32_t write_size) +// { +// // Read all data in one go +// uint8_t *read_buf = (uint8_t *)calloc(write_size, sizeof(uint8_t)); +// uint32_t bytes = write_size; +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, inst, 0), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_read_data(fs, inst, read_buf, &bytes), +// STORFS_ERR_END_OF_FILE); +// TEST_ASSERT_EQUAL(memcmp(write_buf, read_buf, write_size), 0); +// free(read_buf); +// } +// +// void test_read_after_full_write(void) { +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// storfs_size_t chunk_size = fs->pageSize; +// uint32_t buf_size; +// uint8_t *write_buf; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// +// // Read all of the data to make sure it matches +// uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// storfs_err_t err = snode_read_data(fs, &inst, read_buf, +// &data_size); storfs_err_t compare = get_err_compare(data_remain, +// chunk_size); TEST_ASSERT_EQUAL(err, compare); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// } +// +// random_array_free(write_buf); +// free(read_buf); +// } +// +// void test_read_after_full_write_update_cache(void) { +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// storfs_size_t chunk_size = fs->pageSize; +// uint32_t buf_size; +// uint8_t *write_buf; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// +// // Read all of the data to make sure it matches +// uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, i), STORFS_OK); +// storfs_err_t err = snode_read_data(fs, &inst, read_buf, &data_size); +// storfs_err_t compare = get_err_compare(data_remain, chunk_size); +// TEST_ASSERT_EQUAL(err, compare); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// } +// free(read_buf); +// +// simple_read_helper(&inst, write_buf, buf_size); +// +// random_array_free(write_buf); +// } +// +// void test_read_after_random_page_full_write(void) { +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// storfs_size_t chunk_size = fs->pageSize * 5; +// uint32_t buf_size; +// uint8_t *write_buf; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// simple_read_helper(&inst, write_buf, buf_size); +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// +// chunk_size = fs->pageSize * 12; +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// simple_read_helper(&inst, write_buf, buf_size); +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// +// // Offset chunk not exactly on a byte boundary +// chunk_size = (storfs_size_t)((float)fs->pageSize * 5.25); +// write_buf = fill_snode(&inst, &buf_size, chunk_size); +// simple_read_helper(&inst, write_buf, buf_size); +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// +// // Write random chunk size to the file until it filles up +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// write_buf = NULL; +// uint32_t allocated = 0; +// storfs_err_t err; +// uint32_t file_size = 0; +// do { +// chunk_size = random_integer(RANDOM_DATA_MAX_CHUNK); +// uint32_t written = chunk_size; +// storfs_size_t offset = allocated; +// +// allocated += chunk_size; +// write_buf = realloc(write_buf, allocated); +// +// err = snode_write_data(fs, &inst, &write_buf[offset], &written); +// if(err == STORFS_OK) { +// TEST_ASSERT_EQUAL(written, chunk_size); +// } else { +// TEST_ASSERT_EQUAL(err, STORFS_ERR_END_OF_FILE); +// // The total file size is equal to the last data written +// file_size = allocated - (chunk_size - written); +// } +// } while(err == STORFS_OK); +// simple_read_helper(&inst, write_buf, file_size); +// +// // Read back random chunk size +// full_read_helper(&inst, write_buf, file_size, 0, true, 0); +// random_array_free(write_buf); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst, &file_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst.write.idx, 0); +// } +// +// void test_fill_stagger(void) { +// uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); +// uint8_t *write_buf = random_array(buf_size); +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// storfs_size_t chunk_size = fs->pageSize; +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// uint32_t idx_compare = i / chunk_size + 1; +// +// storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], +// &data_size); storfs_err_t err_compare = get_err_compare(data_remain, +// chunk_size); TEST_ASSERT_EQUAL(err, err_compare); +// TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); +// // Test that finding the location is also correct +// snode_find_write_location(fs, &inst); +// TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); +// } +// +// // Read all of the data to make sure it matches +// uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// storfs_err_t err = snode_read_data(fs, &inst, read_buf, +// &data_size); storfs_err_t compare = get_err_compare(data_remain, +// chunk_size); TEST_ASSERT_EQUAL(err, compare); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// } +// +// random_array_free(write_buf); +// free(read_buf); +// } +// +// void test_fill_boundary(void) { +// storfs_page_t page = 17; +// char name[STORFS_MAX_FILE_NAME] = FAKE_NAME; +// SNodeInst inst = { 0 }; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL(snode_create(fs, name, &page, SNODE_TYPE_FILE), +// STORFS_OK); TEST_ASSERT_EQUAL(snode_lookup(fs, page, &inst), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst, 0), STORFS_OK); +// +// // Test writing on boundary +// storfs_size_t chunk_size = fs->pageSize; +// uint32_t buf_size = chunk_size * calc_max_idx(); +// uint32_t leftover = fs->pageSize - sizeof(SNode); +// buf_size += leftover; +// uint8_t *write_buf = random_array(buf_size); +// +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst, write_buf, &leftover), +// STORFS_OK); +// +// for(uint32_t i = leftover; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = chunk_size; +// +// storfs_err_t err = snode_write_data(fs, &inst, &write_buf[i], +// &data_size); storfs_err_t err_compare = get_err_compare(data_remain, +// chunk_size); TEST_ASSERT_EQUAL(err, err_compare); +// +// const uint32_t idx_compare = i / chunk_size + 2; +// TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); +// +// // Test obtaining write location works as expected as well +// snode_find_write_location(fs, &inst); +// TEST_ASSERT_EQUAL(inst.write.idx, idx_compare); +// } +// +// // Read all of the data to make sure it matches +// uint8_t *read_buf = (uint8_t *)calloc(chunk_size, sizeof(uint8_t)); +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// storfs_err_t err = snode_read_data(fs, &inst, read_buf, +// &data_size); storfs_err_t compare = get_err_compare(data_remain, +// chunk_size); TEST_ASSERT_EQUAL(err, compare); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// } +// +// random_array_free(write_buf); +// free(read_buf); +// } +// +// void test_snode_write_read_alternate(void) { +// const uint32_t buf_size = MULTIPLE_INDIRECT_DATA_SIZE(fs); +// uint8_t *write_buf = random_array(buf_size); +// uint8_t *read_buf = (uint8_t *)calloc(buf_size, sizeof(uint8_t)); +// storfs_page_t snode_1_page = 17; +// storfs_page_t snode_2_page = snode_1_page++; +// char snode_1_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; +// char snode_2_name[STORFS_MAX_FILE_NAME] = FAKE_NAME "snode_1"; +// +// storfs_crc16_IgnoreAndReturn(FAKE_CRC16); +// +// TEST_ASSERT_EQUAL( +// snode_create(fs, snode_1_name, &snode_1_page, SNODE_TYPE_FILE), +// STORFS_OK); +// TEST_ASSERT_EQUAL( +// snode_create(fs, snode_2_name, &snode_2_page, SNODE_TYPE_FILE), +// STORFS_OK); +// +// // Write all the data from the file, read it and compare +// SNodeInst inst_1 = { 0 }; +// SNodeInst inst_2 = { 0 }; +// TEST_ASSERT_EQUAL(snode_lookup(fs, snode_1_page, &inst_1), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_lookup(fs, snode_2_page, &inst_2), STORFS_OK); +// +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_1), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_1, 0), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_write_location(fs, &inst_2), STORFS_OK); +// TEST_ASSERT_EQUAL(snode_find_read_location(fs, &inst_2, 0), STORFS_OK); +// +// // Chunk read it to check offset paramter +// uint32_t chunk_size = 693; +// for(uint32_t i = 0; i < buf_size; i += chunk_size) { +// uint32_t data_remain = buf_size - i; +// uint32_t data_size = MIN(data_remain, chunk_size); +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_1, &write_buf[i], +// &data_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_1, read_buf, &data_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// TEST_ASSERT_EQUAL(snode_write_data(fs, &inst_2, &write_buf[i], +// &data_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(snode_read_data(fs, &inst_2, read_buf, &data_size), +// STORFS_OK); +// TEST_ASSERT_EQUAL(memcmp(&write_buf[i], read_buf, data_size), 0); +// } +// TEST_ASSERT_EQUAL(inst_1.node.size, buf_size); +// TEST_ASSERT_EQUAL(inst_2.node.size, buf_size); +// +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_1, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst_1.node.size, 0); +// TEST_ASSERT_EQUAL(snode_erase_data(fs, &inst_2, &buf_size), STORFS_OK); +// TEST_ASSERT_EQUAL(inst_2.node.size, 0); +// +// random_array_free(write_buf); +// free(read_buf); +// }