From f3eba8637278678762b9cadef4907368c9f9861a Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 11:25:35 -0400 Subject: [PATCH 01/73] Rename CDataLogger to CFsDataLogger and add a CRawDataLogger class --- app/backplane/power_module/include/c_power_module.h | 2 +- app/backplane/radio_module/include/c_radio_module.h | 2 +- app/backplane/sensor_module/include/c_sensor_module.h | 2 +- app/samples/benchmark_littlefs_datalogger/src/main.cpp | 2 +- app/samples/datalogger/src/main.cpp | 6 +++--- .../f_core/os/{c_datalogger.h => c_fs_datalogger.h} | 8 ++++---- include/f_core/os/c_raw_datalogger.h | 4 ++++ include/f_core/os/tenants/c_datalogger_tenant.h | 10 +++++----- .../os/{c_datalogger.cpp => c_fs_datalogger.cpp} | 2 +- lib/f_core/os/c_raw_datalogger.cpp | 1 + 10 files changed, 22 insertions(+), 17 deletions(-) rename include/f_core/os/{c_datalogger.h => c_fs_datalogger.h} (92%) create mode 100644 include/f_core/os/c_raw_datalogger.h rename lib/f_core/os/{c_datalogger.cpp => c_fs_datalogger.cpp} (97%) create mode 100644 lib/f_core/os/c_raw_datalogger.cpp diff --git a/app/backplane/power_module/include/c_power_module.h b/app/backplane/power_module/include/c_power_module.h index 5cfe9374d..22e6a9b03 100644 --- a/app/backplane/power_module/include/c_power_module.h +++ b/app/backplane/power_module/include/c_power_module.h @@ -69,7 +69,7 @@ class CPowerModule : public CProjectConfiguration { "Broadcast Tenant", ipAddrStr, downlinkBroadcastPort, downlinkBroadcastPort, sensorDataDownlinkMessagePort }; - CDataLoggerTenant dataLoggerTenant{ + CFsDataLoggerTenant dataLoggerTenant{ "Data Logger Tenant", "/lfs/sensor_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(60), 5 }; CUdpAlertTenant alertTenant{"Alert Tenant", ipAddrStr, NNetworkDefs::ALERT_PORT}; diff --git a/app/backplane/radio_module/include/c_radio_module.h b/app/backplane/radio_module/include/c_radio_module.h index 21fd73d0e..6f69d7f5e 100644 --- a/app/backplane/radio_module/include/c_radio_module.h +++ b/app/backplane/radio_module/include/c_radio_module.h @@ -75,7 +75,7 @@ class CRadioModule : public CProjectConfiguration { CLoraTransmitTenant loraTransmitTenant{"LoRa Transmit Tenant", lora, &loraBroadcastMessagePort}; CLoraReceiveTenant loraReceiveTenant{"LoRa Receive Tenant", loraTransmitTenant, ipAddrStr, radioModuleSourcePort}; #endif - CDataLoggerTenant dataLoggerTenant{"Data Logger Tenant", "/lfs/gps_data.bin", LogMode::Growing, 0, gnssDataLogMessagePort, K_SECONDS(15), 5}; + CFsDataLoggerTenant dataLoggerTenant{"Data Logger Tenant", "/lfs/gps_data.bin", LogMode::Growing, 0, gnssDataLogMessagePort, K_SECONDS(15), 5}; CStateMachineUpdater stateMachineUpdater; // Tasks diff --git a/app/backplane/sensor_module/include/c_sensor_module.h b/app/backplane/sensor_module/include/c_sensor_module.h index a7d9db5c5..cf2efbf50 100644 --- a/app/backplane/sensor_module/include/c_sensor_module.h +++ b/app/backplane/sensor_module/include/c_sensor_module.h @@ -71,7 +71,7 @@ class CSensorModule : public CProjectConfiguration { CUdpBroadcastTenant broadcastTenant{"Broadcast Tenant", ipAddrStr.c_str(), telemetryBroadcastPort, telemetryBroadcastPort, sensorDataBroadcastMessagePort}; CUdpBroadcastTenant downlinkTelemTenant{"Telemetry Downlink Tenant", ipAddrStr.c_str(), telemetryDownlinkPort, telemetryDownlinkPort, downlinkMessagePort}; CUdpBroadcastTenant udpAlertTenant{"UDP Alert Tenant", ipAddrStr.c_str(), alertPort, alertPort, alertMessagePort}; - CDataLoggerTenant dataLoggerTenant{"Data Logger Tenant", "/lfs/sensor_module_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(3), 64}; + CFsDataLoggerTenant dataLoggerTenant{"Data Logger Tenant", "/lfs/sensor_module_data.bin", LogMode::Growing, 0, sensorDataLogMessagePort, K_SECONDS(3), 64}; // Tasks CTask networkTask{"Networking Task", 15, 3072, 0}; diff --git a/app/samples/benchmark_littlefs_datalogger/src/main.cpp b/app/samples/benchmark_littlefs_datalogger/src/main.cpp index f8638904e..9427eac2d 100644 --- a/app/samples/benchmark_littlefs_datalogger/src/main.cpp +++ b/app/samples/benchmark_littlefs_datalogger/src/main.cpp @@ -73,7 +73,7 @@ template void benchmarkDataloggerMode(const char *testName, const char *filePath, LogMode mode, size_t maxPackets = 1000, size_t syncFrequency = 10) { LOG_INF("\n\n=== %s ===", testName); - CDataLogger logger(filePath, mode, maxPackets); + CFsDataLogger logger(filePath, mode, maxPackets); uint64_t totalWriteCycles = 0; uint64_t totalSyncCycles = 0; diff --git a/app/samples/datalogger/src/main.cpp b/app/samples/datalogger/src/main.cpp index 57306627e..406b02fd4 100644 --- a/app/samples/datalogger/src/main.cpp +++ b/app/samples/datalogger/src/main.cpp @@ -6,9 +6,9 @@ struct Packet { uint8_t a; uint8_t b; }; -CDataLogger expand_logger{"/lfs/expand.bin"}; -CDataLogger fill_logger{"/lfs/fill.bin", LogMode::FixedSize, 10}; -CDataLogger wrap_logger{"/lfs/wrap.bin", LogMode::Circular, 10}; +CFsDataLogger expand_logger{"/lfs/expand.bin"}; +CFsDataLogger fill_logger{"/lfs/fill.bin", LogMode::FixedSize, 10}; +CFsDataLogger wrap_logger{"/lfs/wrap.bin", LogMode::Circular, 10}; int main() { for (uint8_t i = 0; i < 100; i++) { diff --git a/include/f_core/os/c_datalogger.h b/include/f_core/os/c_fs_datalogger.h similarity index 92% rename from include/f_core/os/c_datalogger.h rename to include/f_core/os/c_fs_datalogger.h index 296dae631..dfac4eae5 100644 --- a/include/f_core/os/c_datalogger.h +++ b/include/f_core/os/c_fs_datalogger.h @@ -7,7 +7,7 @@ enum class LogMode { Growing, Circular, FixedSize }; -/// @brief Internal, type-unsafe datalogger. Don't use this directly. Use CDataLogger instead +/// @brief Internal, type-unsafe datalogger. Don't use this directly. Use CFsDataLogger instead namespace detail { class datalogger { public: @@ -32,7 +32,7 @@ class datalogger { * This class is implemented as a type safe wrapper to detail::datalogger. */ template -class CDataLogger { +class CFsDataLogger { public: using PacketType = T; static_assert(std::is_trivially_copyable::value, @@ -46,14 +46,14 @@ class CDataLogger { * The logger will use the "Growing" mode and will expand as you write more data until your filesystem runs out of space. * @param filename the name of the file to write to */ - CDataLogger(const char *filename) : internal(filename, LogMode::Growing, 0) {} + CFsDataLogger(const char *filename) : internal(filename, LogMode::Growing, 0) {} /** * Construct a Datalogger for the specified filename, grow mode, and size * @param filename the name of the file to write to * @param mode the logging mode to use * @param the number of packets to log (only used if mode is Circular or FixedSize) */ - CDataLogger(const char *filename, LogMode mode, std::size_t num_packets) : internal(filename, mode, num_packets) {} + CFsDataLogger(const char *filename, LogMode mode, std::size_t num_packets) : internal(filename, mode, num_packets) {} /** * Write a packet to the file diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h new file mode 100644 index 000000000..918723c63 --- /dev/null +++ b/include/f_core/os/c_raw_datalogger.h @@ -0,0 +1,4 @@ +#ifndef C_RAW_DATALOGGER_H +#define C_RAW_DATALOGGER_H + +#endif //C_RAW_DATALOGGER_H \ No newline at end of file diff --git a/include/f_core/os/tenants/c_datalogger_tenant.h b/include/f_core/os/tenants/c_datalogger_tenant.h index d5ea3c1ae..0cf1e08e8 100644 --- a/include/f_core/os/tenants/c_datalogger_tenant.h +++ b/include/f_core/os/tenants/c_datalogger_tenant.h @@ -5,20 +5,20 @@ #include #include -#include +#include #include #include template -class CDataLoggerTenant : public CTenant { +class CFsDataLoggerTenant : public CTenant { public: - explicit CDataLoggerTenant(const char* name, const char* filename, LogMode mode, std::size_t numPackets, + explicit CFsDataLoggerTenant(const char* name, const char* filename, LogMode mode, std::size_t numPackets, CMessagePort& messagePort, k_timeout_t syncTimeout = K_FOREVER, int syncOnCount = 0) : CTenant(name), messagePort(messagePort), dataLogger(filename, mode, numPackets), filename(filename), syncTimeout(syncTimeout), syncOnCount(syncOnCount) {} - ~CDataLoggerTenant() override { + ~CFsDataLoggerTenant() override { Cleanup(); } @@ -55,7 +55,7 @@ class CDataLoggerTenant : public CTenant { private: CMessagePort& messagePort; - CDataLogger dataLogger; + CFsDataLogger dataLogger; const char* filename; // FS Sync after every X time diff --git a/lib/f_core/os/c_datalogger.cpp b/lib/f_core/os/c_fs_datalogger.cpp similarity index 97% rename from lib/f_core/os/c_datalogger.cpp rename to lib/f_core/os/c_fs_datalogger.cpp index f38a13a20..df8f293de 100644 --- a/lib/f_core/os/c_datalogger.cpp +++ b/lib/f_core/os/c_fs_datalogger.cpp @@ -1,4 +1,4 @@ -#include +#include #include LOG_MODULE_REGISTER(datalogger); diff --git a/lib/f_core/os/c_raw_datalogger.cpp b/lib/f_core/os/c_raw_datalogger.cpp new file mode 100644 index 000000000..3c2ae61b0 --- /dev/null +++ b/lib/f_core/os/c_raw_datalogger.cpp @@ -0,0 +1 @@ +#include "c_raw_datalogger.h" \ No newline at end of file From b0b2cca8b3709fe2f356d92e9ccbd5c757f86f46 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 11:54:44 -0400 Subject: [PATCH 02/73] Define functions --- .../sensor_module/src/c_sensor_module.cpp | 1 + include/f_core/os/c_raw_datalogger.h | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/app/backplane/sensor_module/src/c_sensor_module.cpp b/app/backplane/sensor_module/src/c_sensor_module.cpp index 40cb8a7c2..7b0825263 100644 --- a/app/backplane/sensor_module/src/c_sensor_module.cpp +++ b/app/backplane/sensor_module/src/c_sensor_module.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include LOG_MODULE_REGISTER(sensor_module); diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h index 918723c63..dbb603c72 100644 --- a/include/f_core/os/c_raw_datalogger.h +++ b/include/f_core/os/c_raw_datalogger.h @@ -1,4 +1,30 @@ #ifndef C_RAW_DATALOGGER_H #define C_RAW_DATALOGGER_H +#include + +template +class CRawDataLogger { +public: + CRawDataLogger(const device &flash) : flash(flash), ctx(nullptr) {} + + int Write() { + + } + + int Close() { + + } + + int Flush() { + + } + + +private: + const device& flash; + stream_flash_ctx *ctx; + uint8_t buffer[bufferSize]; +}; + #endif //C_RAW_DATALOGGER_H \ No newline at end of file From 31836972a9363e045b0963eb4c017d8887037c52 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 13:00:24 -0400 Subject: [PATCH 03/73] Implementation for CRawDatalogger --- include/f_core/os/c_raw_datalogger.h | 91 +++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h index dbb603c72..36698b8a5 100644 --- a/include/f_core/os/c_raw_datalogger.h +++ b/include/f_core/os/c_raw_datalogger.h @@ -2,29 +2,104 @@ #define C_RAW_DATALOGGER_H #include +#include +#include -template +struct DataloggerMetadata { + char filename[32]; + size_t allocated_size; + uint32_t version; + size_t packet_size; +}; + +constexpr uint32_t DATALOGGER_VERSION = 1; + +template class CRawDataLogger { public: - CRawDataLogger(const device &flash) : flash(flash), ctx(nullptr) {} + CRawDataLogger(const device *flash_dev, size_t flashAddress, size_t fileSize, const std::string &filename) + : flash(flash_dev), flashAddress(flashAddress), fileSize(fileSize), bytesBuffered(0), bytesWritten(0), initialized(false) + { + memset(&ctx, 0, sizeof(ctx)); + memset(buffer, 0, sizeof(buffer)); + + // prep metadata + DataloggerMetadata meta = {}; + strncpy(meta.filename, filename.c_str(), sizeof(meta.filename) - 1); + meta.allocated_size = fileSize; + meta.version = DATALOGGER_VERSION; + meta.packet_size = sizeof(T); - int Write() { + // metadata to flash + int ret = stream_flash_init(&ctx, flash, buffer, bufferSize, flashAddress, fileSize, nullptr); + if (ret < 0) { + lastError = ret; + return; + } + ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&meta), sizeof(meta), true); + if (ret < 0) { + lastError = ret; + return; + } + bytesWritten = sizeof(meta); + initialized = true; + // reinit ctx start after metadata + ret = stream_flash_init(&ctx, flash, buffer, bufferSize, flashAddress + sizeof(meta), fileSize - sizeof(meta), nullptr); + if (ret < 0) { + lastError = ret; + initialized = false; + } } - int Close() { + bool IsInitialized() const { return initialized; } + int Write(const T &data, bool flush = false) { + if (!initialized) return lastError ? lastError : -1; + + if (bytesBuffered + sizeof(T) > bufferSize) { + int ret = Flush(); + if (ret < 0) return ret; + } + + memcpy(buffer + bytesBuffered, &data, sizeof(T)); + bytesBuffered += sizeof(T); + + // Flush if requested or buffer full + if (flush || bytesBuffered == bufferSize) { + int ret = Flush(); + if (ret < 0) return ret; + } + return 0; } int Flush() { - + if (!initialized) return lastError ? lastError : -1; + if (bytesBuffered == 0) return 0; + int ret = stream_flash_buffered_write(&ctx, buffer, bytesBuffered, true); + if (ret < 0) { + lastError = ret; + return ret; + } + bytesWritten += bytesBuffered; + bytesBuffered = 0; + return 0; } + size_t GetBytesWritten() const { return bytesWritten; } + size_t GetBytesBuffered() const { return bytesBuffered; } private: - const device& flash; - stream_flash_ctx *ctx; - uint8_t buffer[bufferSize]; + const device *flash; + stream_flash_ctx ctx; + size_t flashAddress; + size_t fileSize; + uint8_t buffer[packetBufferSize * sizeof(T)]; + size_t bufferSize = sizeof(buffer); + size_t bytesBuffered; + size_t bytesWritten; + int lastError = 0; + bool initialized; }; #endif //C_RAW_DATALOGGER_H \ No newline at end of file From 6674a0238a1a8dd12d49644d44c4859654c31cf7 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 13:08:26 -0400 Subject: [PATCH 04/73] Simplify --- include/f_core/os/c_raw_datalogger.h | 52 +++++++++------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h index 36698b8a5..27d20fb46 100644 --- a/include/f_core/os/c_raw_datalogger.h +++ b/include/f_core/os/c_raw_datalogger.h @@ -18,20 +18,15 @@ template class CRawDataLogger { public: CRawDataLogger(const device *flash_dev, size_t flashAddress, size_t fileSize, const std::string &filename) - : flash(flash_dev), flashAddress(flashAddress), fileSize(fileSize), bytesBuffered(0), bytesWritten(0), initialized(false) + : flash(flash_dev), flashAddress(flashAddress), fileSize(fileSize), lastError(0), initialized(false) { - memset(&ctx, 0, sizeof(ctx)); - memset(buffer, 0, sizeof(buffer)); - // prep metadata DataloggerMetadata meta = {}; strncpy(meta.filename, filename.c_str(), sizeof(meta.filename) - 1); meta.allocated_size = fileSize; meta.version = DATALOGGER_VERSION; meta.packet_size = sizeof(T); - - // metadata to flash - int ret = stream_flash_init(&ctx, flash, buffer, bufferSize, flashAddress, fileSize, nullptr); + int ret = stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress, fileSize, nullptr); if (ret < 0) { lastError = ret; return; @@ -41,11 +36,8 @@ class CRawDataLogger { lastError = ret; return; } - bytesWritten = sizeof(meta); initialized = true; - - // reinit ctx start after metadata - ret = stream_flash_init(&ctx, flash, buffer, bufferSize, flashAddress + sizeof(meta), fileSize - sizeof(meta), nullptr); + ret = stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress + sizeof(meta), fileSize - sizeof(meta), nullptr); if (ret < 0) { lastError = ret; initialized = false; @@ -56,49 +48,39 @@ class CRawDataLogger { int Write(const T &data, bool flush = false) { if (!initialized) return lastError ? lastError : -1; - - if (bytesBuffered + sizeof(T) > bufferSize) { - int ret = Flush(); - if (ret < 0) return ret; - } - - memcpy(buffer + bytesBuffered, &data, sizeof(T)); - bytesBuffered += sizeof(T); - - // Flush if requested or buffer full - if (flush || bytesBuffered == bufferSize) { - int ret = Flush(); - if (ret < 0) return ret; + int ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&data), sizeof(T), flush); + if (ret < 0) { + lastError = ret; + return ret; } return 0; } int Flush() { if (!initialized) return lastError ? lastError : -1; - if (bytesBuffered == 0) return 0; - int ret = stream_flash_buffered_write(&ctx, buffer, bytesBuffered, true); + int ret = stream_flash_buffered_write(&ctx, nullptr, 0, true); if (ret < 0) { lastError = ret; return ret; } - bytesWritten += bytesBuffered; - bytesBuffered = 0; return 0; } - size_t GetBytesWritten() const { return bytesWritten; } - size_t GetBytesBuffered() const { return bytesBuffered; } + size_t GetBytesWritten() const { + return stream_flash_bytes_written(&ctx); + } + size_t GetBytesBuffered() const { + return stream_flash_bytes_buffered(&ctx); + } + int GetLastError() const { return lastError; } private: const device *flash; stream_flash_ctx ctx; size_t flashAddress; size_t fileSize; - uint8_t buffer[packetBufferSize * sizeof(T)]; - size_t bufferSize = sizeof(buffer); - size_t bytesBuffered; - size_t bytesWritten; - int lastError = 0; + uint8_t buffer[packetBufferSize]; + int lastError; bool initialized; }; From 92c4a5be43b4c0ca1d1d9de1d94afcbaa08504ae Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 13:15:33 -0400 Subject: [PATCH 05/73] Introduce several modes for hitting EOF --- include/f_core/os/c_raw_datalogger.h | 80 +++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h index 27d20fb46..03db3405f 100644 --- a/include/f_core/os/c_raw_datalogger.h +++ b/include/f_core/os/c_raw_datalogger.h @@ -5,11 +5,18 @@ #include #include +enum class DataloggerMode { + Rotating, + Fixed, + Linked +}; + struct DataloggerMetadata { char filename[32]; size_t allocated_size; uint32_t version; size_t packet_size; + size_t next_file_address; }; constexpr uint32_t DATALOGGER_VERSION = 1; @@ -17,27 +24,27 @@ constexpr uint32_t DATALOGGER_VERSION = 1; template class CRawDataLogger { public: - CRawDataLogger(const device *flash_dev, size_t flashAddress, size_t fileSize, const std::string &filename) - : flash(flash_dev), flashAddress(flashAddress), fileSize(fileSize), lastError(0), initialized(false) - { - // prep metadata - DataloggerMetadata meta = {}; - strncpy(meta.filename, filename.c_str(), sizeof(meta.filename) - 1); - meta.allocated_size = fileSize; - meta.version = DATALOGGER_VERSION; - meta.packet_size = sizeof(T); + CRawDataLogger(const device* flash_dev, size_t flashAddress, size_t fileSize, const std::string& filename, + DataloggerMode mode) + : flash(flash_dev), flashAddress(flashAddress), fileSize(fileSize), mode(mode), lastError(0), + initialized(false), currentOffset(0), nextFileAddress(0) { + memset(&ctx, 0, sizeof(ctx)); + memset(buffer, 0, sizeof(buffer)); + PrepMetadata(filename, 0); // next_file_address = 0 for first file int ret = stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress, fileSize, nullptr); if (ret < 0) { lastError = ret; return; } - ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&meta), sizeof(meta), true); + ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&metadata), sizeof(metadata), true); if (ret < 0) { lastError = ret; return; } initialized = true; - ret = stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress + sizeof(meta), fileSize - sizeof(meta), nullptr); + currentOffset = sizeof(metadata); + ret = stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress + sizeof(metadata), + fileSize - sizeof(metadata), nullptr); if (ret < 0) { lastError = ret; initialized = false; @@ -45,14 +52,44 @@ class CRawDataLogger { } bool IsInitialized() const { return initialized; } + DataloggerMode GetMode() const { return mode; } - int Write(const T &data, bool flush = false) { + int Write(const T& data, bool flush = false) { if (!initialized) return lastError ? lastError : -1; + size_t bytes_written = stream_flash_bytes_written(&ctx); + size_t space_left = fileSize - sizeof(metadata) - bytes_written; + if (space_left < sizeof(T)) { + switch (mode) { + case DataloggerMode::Rotating: + currentOffset = sizeof(metadata); + stream_flash_init(&ctx, flash, buffer, sizeof(buffer), flashAddress + sizeof(metadata), + fileSize - sizeof(metadata), nullptr); + break; + case DataloggerMode::Fixed: + return -ENOSPC; + case DataloggerMode::Linked: + nextFileAddress = flashAddress + fileSize; + PrepMetadata(metadata.filename, nextFileAddress); + stream_flash_init(&ctx, flash, buffer, sizeof(buffer), nextFileAddress, fileSize, nullptr); + int ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&metadata), + sizeof(metadata), true); + if (ret < 0) { + lastError = ret; + return ret; + } + stream_flash_init(&ctx, flash, buffer, sizeof(buffer), nextFileAddress + sizeof(metadata), + fileSize - sizeof(metadata), nullptr); + flashAddress = nextFileAddress; + currentOffset = sizeof(metadata); + break; + } + } int ret = stream_flash_buffered_write(&ctx, reinterpret_cast(&data), sizeof(T), flush); if (ret < 0) { lastError = ret; return ret; } + currentOffset += sizeof(T); return 0; } @@ -69,19 +106,36 @@ class CRawDataLogger { size_t GetBytesWritten() const { return stream_flash_bytes_written(&ctx); } + size_t GetBytesBuffered() const { return stream_flash_bytes_buffered(&ctx); } + int GetLastError() const { return lastError; } + size_t GetCurrentOffset() const { return currentOffset; } + size_t GetNextFileAddress() const { return nextFileAddress; } private: - const device *flash; + const device* flash; stream_flash_ctx ctx; size_t flashAddress; size_t fileSize; uint8_t buffer[packetBufferSize]; + DataloggerMode mode; int lastError; bool initialized; + DataloggerMetadata metadata; + size_t currentOffset; + size_t nextFileAddress; + + void PrepMetadata(const std::string& filename, size_t next_addr) { + memset(&metadata, 0, sizeof(metadata)); + strncpy(metadata.filename, filename.c_str(), sizeof(metadata.filename) - 1); + metadata.allocated_size = fileSize; + metadata.version = DATALOGGER_VERSION; + metadata.packet_size = sizeof(T); + metadata.next_file_address = next_addr; + } }; #endif //C_RAW_DATALOGGER_H \ No newline at end of file From 223199ba03a5ed4459c296c5662e47853b25fbb5 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Sat, 27 Sep 2025 13:23:57 -0400 Subject: [PATCH 06/73] ReadMetaData and Find functions --- include/f_core/os/c_raw_datalogger.h | 55 +++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/include/f_core/os/c_raw_datalogger.h b/include/f_core/os/c_raw_datalogger.h index 03db3405f..d89106e45 100644 --- a/include/f_core/os/c_raw_datalogger.h +++ b/include/f_core/os/c_raw_datalogger.h @@ -4,11 +4,13 @@ #include #include #include +#include enum class DataloggerMode { Rotating, Fixed, - Linked + LinkedFixed, + LinkedTruncate }; struct DataloggerMetadata { @@ -67,7 +69,8 @@ class CRawDataLogger { break; case DataloggerMode::Fixed: return -ENOSPC; - case DataloggerMode::Linked: + case DataloggerMode::LinkedFixed: + case DataloggerMode::LinkedTruncate: nextFileAddress = flashAddress + fileSize; PrepMetadata(metadata.filename, nextFileAddress); stream_flash_init(&ctx, flash, buffer, sizeof(buffer), nextFileAddress, fileSize, nullptr); @@ -115,6 +118,41 @@ class CRawDataLogger { size_t GetCurrentOffset() const { return currentOffset; } size_t GetNextFileAddress() const { return nextFileAddress; } + std::optional> FindLinkedSpace() { + if (mode != DataloggerMode::LinkedFixed && mode != DataloggerMode::LinkedTruncate) return std::nullopt; + size_t addr = flashAddress; + size_t max_flash_size = /* TODO: get max flash size from device or config */ 0x1000000; // Example: 16MB + size_t file_sz = fileSize; + DataloggerMetadata meta; + while (addr + sizeof(DataloggerMetadata) < max_flash_size) { + int ret = ReadMetadata(addr, meta); + if (ret == 0) { + // Metadata found, follow next pointer or jump to next file boundary + if (meta.next_file_address != 0 && meta.next_file_address > addr) { + addr = meta.next_file_address; + } else { + addr += file_sz; + } + } else { + // No metadata found here + if (mode == DataloggerMode::LinkedFixed) { + // Check if enough space for a new file + if (addr + file_sz < max_flash_size) { + return std::make_pair(addr, file_sz); + } else { + return std::nullopt; + } + } else if (mode == DataloggerMode::LinkedTruncate) { + // Truncate to next metadata or use initial size + size_t next_meta_addr = FindNextMetadata(addr + sizeof(DataloggerMetadata), max_flash_size); + size_t available_size = (next_meta_addr > addr) ? (next_meta_addr - addr) : file_sz; + return std::make_pair(addr, available_size); + } + } + } + return std::nullopt; + } + private: const device* flash; stream_flash_ctx ctx; @@ -136,6 +174,19 @@ class CRawDataLogger { metadata.packet_size = sizeof(T); metadata.next_file_address = next_addr; } + + int ReadMetadata(size_t addr, DataloggerMetadata& out_meta) { + return -1; // TODO + } + size_t FindNextMetadata(size_t start_addr, size_t max_addr) { + DataloggerMetadata meta; + for (size_t addr = start_addr; addr + sizeof(DataloggerMetadata) < max_addr; addr += sizeof(DataloggerMetadata)) { + if (ReadMetadata(addr, meta) == 0) { + return addr; + } + } + return max_addr; + } }; #endif //C_RAW_DATALOGGER_H \ No newline at end of file From c18a7625421ec4fedd3d2df2496e2c25a37e2558 Mon Sep 17 00:00:00 2001 From: Aaron Chan Date: Mon, 29 Sep 2025 18:40:39 -0400 Subject: [PATCH 07/73] grab filesize --- app/backplane/sensor_module/.idea/editor.xml | 1 + app/backplane/sensor_module/.idea/vcs.xml | 2 ++ include/f_core/os/c_raw_datalogger.h | 9 ++++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/backplane/sensor_module/.idea/editor.xml b/app/backplane/sensor_module/.idea/editor.xml index e42167fe8..86232448b 100644 --- a/app/backplane/sensor_module/.idea/editor.xml +++ b/app/backplane/sensor_module/.idea/editor.xml @@ -117,6 +117,7 @@