From 0509c1c929626414413e958cbca4009ae6897d46 Mon Sep 17 00:00:00 2001 From: Pascal Roobrouck Date: Sun, 13 Jul 2025 19:09:22 +0200 Subject: [PATCH 1/4] refactor: update humidity unit representation to "%" across sensors and tests; remove unnecessary comments --- lib/logging/clicommand.hpp | 1 - lib/sensors/bme680.cpp | 2 +- lib/sensors/measurementgroupcollection.cpp | 4 - lib/sensors/scd40.cpp | 2 +- lib/sensors/scd40.md | 3 + lib/sensors/sensorchannel.md | 98 ++++++------------- lib/sensors/sensordevicecollection.cpp | 1 + lib/sensors/sht40.cpp | 2 +- .../test_sensordevicecollection/test.cpp | 4 +- 9 files changed, 39 insertions(+), 78 deletions(-) create mode 100644 lib/sensors/scd40.md diff --git a/lib/logging/clicommand.hpp b/lib/logging/clicommand.hpp index 624d0f3..f2e0943 100644 --- a/lib/logging/clicommand.hpp +++ b/lib/logging/clicommand.hpp @@ -41,7 +41,6 @@ class cliCommand { static constexpr uint32_t setDisplay{'s' * 256 + 'd'}; // set display : sd line# deviceIndex channelIndex static constexpr uint32_t setSensor{'s' * 256 + 's'}; // set sensor : sd deviceIndex channelIndex oversampling prescaler static constexpr uint32_t softwareReset{'s' * 65536 + 'w' * 256 + 'r'}; // restart device - soft reset - static constexpr uint32_t hardwareReset{'h' * 256 + 'w' * 256 + 'r'}; // reset device - arm for hard reset, reset occurs after removing USB to prevent going into bootloader mode uint32_t nmbrOfArguments{0}; char commandAsString[maxCommandLineLength]; diff --git a/lib/sensors/bme680.cpp b/lib/sensors/bme680.cpp index c07e866..ee89869 100644 --- a/lib/sensors/bme680.cpp +++ b/lib/sensors/bme680.cpp @@ -22,7 +22,7 @@ bool mockBME680Present{false}; sensorDeviceState bme680::state{sensorDeviceState::unknown}; sensorChannel bme680::channels[nmbrChannels] = { {1, "temperature", "~C"}, - {0, "relativeHumidity", "%RH"}, + {0, "relativeHumidity", "%"}, {0, "barometricPressure", "hPa"}, }; diff --git a/lib/sensors/measurementgroupcollection.cpp b/lib/sensors/measurementgroupcollection.cpp index a165785..2751712 100644 --- a/lib/sensors/measurementgroupcollection.cpp +++ b/lib/sensors/measurementgroupcollection.cpp @@ -69,8 +69,6 @@ void measurementGroupCollection::get(measurementGroup& aMeasurementGroup, uint32 uint32_t remainingLength{lengthInBytes}; uint32_t currentOffset{offset}; - // TODO : wakeUp NVS if needed - while (remainingLength > 0) { uint32_t bytesInThisPage = nonVolatileStorage::bytesInCurrentPage(getAddressFromOffset(currentOffset), remainingLength); nonVolatileStorage::readInPage(getAddressFromOffset(currentOffset), remainingData, bytesInThisPage); @@ -80,8 +78,6 @@ void measurementGroupCollection::get(measurementGroup& aMeasurementGroup, uint32 } aMeasurementGroup.fromBytes(buffer); - - // TODO : put NVS back to sleep if it was sleeping } uint32_t measurementGroupCollection::getFreeSpace() { diff --git a/lib/sensors/scd40.cpp b/lib/sensors/scd40.cpp index 3471c8f..bd4d45d 100644 --- a/lib/sensors/scd40.cpp +++ b/lib/sensors/scd40.cpp @@ -19,7 +19,7 @@ bool mockSCD40Present{false}; sensorDeviceState scd40::state{sensorDeviceState::unknown}; sensorChannel scd40::channels[nmbrChannels] = { {1, "temperature", "~C"}, - {0, "relativeHumidity", "%RH"}, + {0, "relativeHumidity", "%"}, {0, "CO2", "ppm"}, }; diff --git a/lib/sensors/scd40.md b/lib/sensors/scd40.md new file mode 100644 index 0000000..2dc9b6f --- /dev/null +++ b/lib/sensors/scd40.md @@ -0,0 +1,3 @@ +Important note : I found the temperature readings of the sensor to be very inaccurate : 4 deg C lower than BME680. +It seems this sensor is not factory calibrated and need manual calibration via its offset register. +As we are using the sensor in low power mode, it's maybe dissipating less heat and as such giving a too low reading. \ No newline at end of file diff --git a/lib/sensors/sensorchannel.md b/lib/sensors/sensorchannel.md index d232d33..e865a86 100644 --- a/lib/sensors/sensorchannel.md +++ b/lib/sensors/sensorchannel.md @@ -1,84 +1,46 @@ -# prescaling and oversampling +# sensor oversampling and output frequency -1. Prescaling means that a sensorChannel does not take a sample (measurement) every RTC tick (30s), but rather once per n RTC ticks. -The prescaler value can be set from 0 .. 8190 - prescaler = 0 : this sensorChannel does NOT take any samples = deactivation of the sensorchannel - prescaler = 1 .. 8190 : take a sample every 1..8190 ticks. The prescaleCounter runs from (prescaler-1) to 0 +## oversampling -2. Oversampling means that a sensorChannel takes multiple samples, and then averages them into a single output -The oversampling can be set from 0..7, resulting in averaging 1..8 samples to a single output. -The oversamplingCounter runs from (oversampling) to 0 +You can set any sensorChannel to take multiple samples and average them to a single output. This filters noise and improves the sensors accuracy. +Select oversampling from 4 possible values : +0 : no oversampling, each sample is an output +1 : average 2 samples to an output +2 : average 4 samples to an output +3 : average 10 samples to an output -oversampling (3 bits) and prescaler (13 bits) are compressed into a single uint16 when storing into EEPROM -As this has 0xFF as blank value, a value, the max valid value for prescaler is 8190 (io 8191), so we can recognize uninitialized eeprom. +# output frequency -Note : (minutesBetweenOutput * 2) must be a multiple of numberOfSamplesToAverage -Note : ((minutesBetweenOutput * 2) % numberOfSamplesToAverage) == 0 - - -I could store prescaler more efficiently.. -0 = off +You can select an output frequency for any sensorChannel. Select from 14 possible values : +0 = off : channel will not produce outputs 1 = every minute 2 = every 2 minutes 3 = every 5 minutes 4 = every 10 minutes 5 = every 15 minutes 6 = every 30 minutes -7 = every 60 minutes +7 = every hour 8 = every 2 hours -9 = every 6 hours -10 = every 12 hours -11 = every 24 hours - -prescaler - -0 = off -1 = every minute, prescaler = 2 -2 = every 2 minutes, prescaler = 4 -3 = every 5 minutes, prescaler = 10 -4 = every 10 minutes, prescaler = 20 -5 = every 15 minutes, prescaler = 30 -6 = every 30 minutes, prescaler = 60 -7 = every hour, prescaler = 120 -8 = every 2 hours, prescaler = 240 -9 = every 4 hours, prescaler = 480 -10 = every 6 hours, prescaler = 720 -11 = every 12 hours, prescaler = 1440 -12 = every 24 hours, prescaler = 2880 -13 = every 48 hours, prescaler = 5760 - -4 bits needed - -oversampling -1 -2 -4 -8 - -2 bits needed - -total 6 bits needed, fits in a byte - - - - -# TODO +9 = every 4 hours +10 = every 6 hours +11 = every 12 hours +12 = every 24 hours +13 = every 48 hours -* calibratie coefficienten in array steken -* code refactoren voor meer performantie : meer proberen op voorhand te berekenen -* store the oversampling / prescaling etc for the measurement channels in NVS so the settings remain after reset +# combining oversampling and output frequency -# TODO -* the SX126x also needs two sensorChannels, for RSSI and SNR, as we will measure those and log, store, send them +The basic realTime clock tick is 30 seconds. This means that the sensor cannot be sampled more than once per 30 seconds. +As a result some combinations of oversampling and output frequency are not valid, eg +- oversampling set to 10 +- output frequency set to every minute +This would require 10 samples per minute, and the maximum is 2 samples per minute +When setting incompatible values, the oversampling will be reduced to be compatible with output frequency -We need an additional setting on each sensorChannel what to do with the measurement : -* nothing -* log to UART -* store into EEPROM -* transmit over LoRaWAN +# optimizing oversampling and output frequency -Showing on the display is controlled at the display itself, where we define up to three sensorChannelTypes to be shown on the top 3 lines of the screen. Battery SOC and network is always shown at bottom line +For optimal (lowest) power consumption, it is best to consider 2 things : +* give all sensors settings so that the samples and outputs are multiples of each other (or equal). + * good example : one sensor every 60 minutes, other sensor 15 minutes (60 is multiple of 15) + * bad example : noe sensor every 5 minutes, other sensor every 2 minutes (5 is not a multiple of 2) +* even when these settings are optimized, it is recommended to restart the device (SWR) so all counters are synchronized -# TODO -At startup, as not all samples in the array are written, the averaging won't work. Could solve this by doing a first sample and set all values to this value -Warning : Lux sensor does something wrong on first sample after power up \ No newline at end of file diff --git a/lib/sensors/sensordevicecollection.cpp b/lib/sensors/sensordevicecollection.cpp index 672c95f..02c38ba 100644 --- a/lib/sensors/sensordevicecollection.cpp +++ b/lib/sensors/sensordevicecollection.cpp @@ -141,6 +141,7 @@ bool sensorDeviceCollection::isSamplingReady() { if (scd40::getState() != sensorDeviceState::sleeping) { return false; } + break; // Add more types of sensors here default: break; diff --git a/lib/sensors/sht40.cpp b/lib/sensors/sht40.cpp index a69d950..ffed88b 100644 --- a/lib/sensors/sht40.cpp +++ b/lib/sensors/sht40.cpp @@ -23,7 +23,7 @@ uint8_t sht40::i2cAddress; sensorDeviceState sht40::state{sensorDeviceState::unknown}; sensorChannel sht40::channels[nmbrChannels] = { {1, "temperature", "~C"}, - {0, "relativeHumidity", "%RH"}, + {0, "relativeHumidity", "%"}, }; uint32_t sht40::rawDataTemperature; diff --git a/test/generic/test_sensordevicecollection/test.cpp b/test/generic/test_sensordevicecollection/test.cpp index 17539be..561ec6e 100644 --- a/test/generic/test_sensordevicecollection/test.cpp +++ b/test/generic/test_sensordevicecollection/test.cpp @@ -167,7 +167,7 @@ void test_channelProperties() { TEST_ASSERT_EQUAL_STRING("relativeHumidity", sensorDeviceCollection::name(static_cast(sensorDeviceType::bme680), bme680::relativeHumidity)); TEST_ASSERT_EQUAL_STRING("barometricPressure", sensorDeviceCollection::name(static_cast(sensorDeviceType::bme680), bme680::barometricPressure)); TEST_ASSERT_EQUAL_STRING("~C", sensorDeviceCollection::units(static_cast(sensorDeviceType::bme680), bme680::temperature)); - TEST_ASSERT_EQUAL_STRING("%RH", sensorDeviceCollection::units(static_cast(sensorDeviceType::bme680), bme680::relativeHumidity)); + TEST_ASSERT_EQUAL_STRING("%", sensorDeviceCollection::units(static_cast(sensorDeviceType::bme680), bme680::relativeHumidity)); TEST_ASSERT_EQUAL_STRING("hPa", sensorDeviceCollection::units(static_cast(sensorDeviceType::bme680), bme680::barometricPressure)); // mockTSL2591Present = true; @@ -188,7 +188,7 @@ void test_channelName() { void test_units() { TEST_ASSERT_EQUAL_STRING("lux", sensorDeviceCollection::units(static_cast(sensorDeviceType::tsl2591), tsl2591::visibleLight)); TEST_ASSERT_EQUAL_STRING("~C", sensorDeviceCollection::units(static_cast(sensorDeviceType::sht40), sht40::temperature)); - TEST_ASSERT_EQUAL_STRING("%RH", sensorDeviceCollection::units(static_cast(sensorDeviceType::sht40), sht40::relativeHumidity)); + TEST_ASSERT_EQUAL_STRING("%", sensorDeviceCollection::units(static_cast(sensorDeviceType::sht40), sht40::relativeHumidity)); TEST_ASSERT_EQUAL_STRING("invalid index", sensorDeviceCollection::units(static_cast(sensorDeviceType::nmbrOfKnownDevices), 0)); } From 4c537890c47ba25f5c422b20a23cd2aa0c3f73eb Mon Sep 17 00:00:00 2001 From: Pascal Roobrouck Date: Sun, 13 Jul 2025 20:45:39 +0200 Subject: [PATCH 2/4] added measurements output as CSV --- lib/application/maincontroller.cpp | 74 +++++++++++++++++++++++++++--- lib/application/maincontroller.hpp | 1 + 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/lib/application/maincontroller.cpp b/lib/application/maincontroller.cpp index 976dfd9..6730ac6 100644 --- a/lib/application/maincontroller.cpp +++ b/lib/application/maincontroller.cpp @@ -496,7 +496,22 @@ void mainController::runCli() { break; case cliCommand::getMeasurements: - showMeasurements(); + if (theCommand.nmbrOfArguments == 0) { + showMeasurements(); + } + if (theCommand.nmbrOfArguments == 1) { + if (strncmp(theCommand.arguments[0], "csv", 3) == 0) { + showMeasurementsCsv(); + break; + } else { + cli::sendResponse("invalid argument\n"); + break; + } + cli::sendResponse("invalid number of arguments\n"); + return; + } + + showMeasurementsCsv(); break; case cliCommand::getLoRaWANStatus: @@ -581,6 +596,7 @@ void mainController::showHelp() { cli::sendResponse("? : show help\n"); cli::sendResponse("gds : show device status\n"); cli::sendResponse("gms : show recorded measurements status\n"); + cli::sendResponse("gm (csv) : show recorded measurements, optionally as CSV\n"); cli::sendResponse("gls : show LoRaWAN status\n"); cli::sendResponse("el : enable logging\n"); cli::sendResponse("dl : disable logging\n"); @@ -809,6 +825,7 @@ void mainController::setSensor(const cliCommand& theCommand) { } void mainController::showMeasurementsStatus() { + cli::sendResponse("wait..."); measurementGroup tmpGroup; uint32_t startOffset = measurementGroupCollection::getOldestMeasurementOffset(); uint32_t endOffset = measurementGroupCollection::getNewMeasurementsOffset(); @@ -831,6 +848,7 @@ void mainController::showMeasurementsStatus() { newestMeasurementTime = tmpGroup.getTimeStamp(); offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements()); } + cli::sendResponse("\b\b\b\b\b\b\b"); cli::sendResponse("%u measurements in %u groups\n", nmbrOfMeasurements, nmbrOfGroups); cli::sendResponse("oldoffset %u %s", measurementGroupCollection::getOldestMeasurementOffset(), ctime(&oldestMeasurementTime)); cli::sendResponse("newoffset %u %s", measurementGroupCollection::getNewMeasurementsOffset(), ctime(&newestMeasurementTime)); @@ -847,14 +865,12 @@ void mainController::showMeasurements() { uint32_t offset{startOffset}; uint32_t nmbrOfGroups{0}; uint32_t nmbrOfMeasurements{0}; - time_t oldestMeasurementTime; - - measurementGroupCollection::get(tmpGroup, offset); - oldestMeasurementTime = tmpGroup.getTimeStamp(); + time_t timeStamp; while (offset < endOffset) { measurementGroupCollection::get(tmpGroup, offset); - cli::sendResponse("%d %s", nmbrOfGroups, ctime(&oldestMeasurementTime)); // 1 line per measurementGroup + timeStamp = tmpGroup.getTimeStamp(); + cli::sendResponse("%d %s", nmbrOfGroups, ctime(&timeStamp)); nmbrOfMeasurements = tmpGroup.getNumberOfMeasurements(); for (uint32_t measurementIndex = 0; measurementIndex < nmbrOfMeasurements; measurementIndex++) { // then per measurement part of this group uint32_t tmpDeviceIndex = tmpGroup.getDeviceIndex(measurementIndex); @@ -874,10 +890,54 @@ void mainController::showMeasurements() { } else { cli::sendResponse(" %d", intPart); } - cli::sendResponse(" %s\n", sensorDeviceCollection::units(tmpDeviceIndex, tmpChannelIndex)); } nmbrOfGroups++; offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements()); } +} + +void mainController::showMeasurementsCsv() { + measurementGroup tmpGroup; + uint32_t startOffset = measurementGroupCollection::getOldestMeasurementOffset(); + uint32_t endOffset = measurementGroupCollection::getNewMeasurementsOffset(); + if (endOffset < startOffset) { + endOffset += nonVolatileStorage::getMeasurementsAreaSize(); + } + uint32_t offset{startOffset}; + uint32_t nmbrOfGroups{0}; + uint32_t nmbrOfMeasurements{0}; + time_t timeStamp; + + cli::sendResponse("year,month,day,hours,minutes,seconds,device,channel,value,units\n"); + + while (offset < endOffset) { + measurementGroupCollection::get(tmpGroup, offset); + timeStamp = tmpGroup.getTimeStamp(); + const struct tm* timeInfo = gmtime(&timeStamp); + nmbrOfMeasurements = tmpGroup.getNumberOfMeasurements(); + for (uint32_t measurementIndex = 0; measurementIndex < nmbrOfMeasurements; measurementIndex++) { // then per measurement part of this group + cli::sendResponse("%d,%d,%d,%d,%d,%d,", timeInfo->tm_year + 1900, timeInfo->tm_mon + 1, timeInfo->tm_mday, timeInfo->tm_hour, timeInfo->tm_min, timeInfo->tm_sec); + uint32_t tmpDeviceIndex = tmpGroup.getDeviceIndex(measurementIndex); + uint32_t tmpChannelIndex = tmpGroup.getChannelIndex(measurementIndex); + float tmpValue = tmpGroup.getValue(measurementIndex); + + uint32_t decimals = sensorDeviceCollection::decimals(tmpDeviceIndex, tmpChannelIndex); + uint32_t intPart = integerPart(tmpValue, decimals); + + cli::sendResponse("%s,", sensorDeviceCollection::name(tmpDeviceIndex)); + cli::sendResponse("%s,", sensorDeviceCollection::name(tmpDeviceIndex, tmpChannelIndex)); + + if (decimals > 0) { + uint32_t fracPart; + fracPart = fractionalPart(tmpValue, decimals); + cli::sendResponse("%d.%d,", intPart, fracPart); + } else { + cli::sendResponse("%d,", intPart); + } + cli::sendResponse("%s\n", sensorDeviceCollection::units(tmpDeviceIndex, tmpChannelIndex)); + } + nmbrOfGroups++; + offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements()); + } } \ No newline at end of file diff --git a/lib/application/maincontroller.hpp b/lib/application/maincontroller.hpp index 5004094..c4468a1 100644 --- a/lib/application/maincontroller.hpp +++ b/lib/application/maincontroller.hpp @@ -70,6 +70,7 @@ class mainController { static void showDeviceStatus(); static void showMeasurementsStatus(); static void showMeasurements(); + static void showMeasurementsCsv(); static void showNetworkStatus(); static void setDeviceAddress(const cliCommand& aCommand); static void setNetworkKey(const cliCommand& aCommand); From aa6e2dc4c8f05c20bfbaba23902ce993dd53bb66 Mon Sep 17 00:00:00 2001 From: Pascal Roobrouck Date: Sun, 13 Jul 2025 20:58:16 +0200 Subject: [PATCH 3/4] fix cli argument parsing --- lib/application/maincontroller.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/application/maincontroller.cpp b/lib/application/maincontroller.cpp index 6730ac6..604c667 100644 --- a/lib/application/maincontroller.cpp +++ b/lib/application/maincontroller.cpp @@ -507,11 +507,10 @@ void mainController::runCli() { cli::sendResponse("invalid argument\n"); break; } - cli::sendResponse("invalid number of arguments\n"); return; + } else { + cli::sendResponse("invalid number of arguments\n"); } - - showMeasurementsCsv(); break; case cliCommand::getLoRaWANStatus: From 8b9dc625dcb23237ea7c061ee2ccb40e48c8ef74 Mon Sep 17 00:00:00 2001 From: Pascal Roobrouck Date: Sun, 13 Jul 2025 21:08:44 +0200 Subject: [PATCH 4/4] fix cli command parsing 2 --- lib/application/maincontroller.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/application/maincontroller.cpp b/lib/application/maincontroller.cpp index 604c667..8f82718 100644 --- a/lib/application/maincontroller.cpp +++ b/lib/application/maincontroller.cpp @@ -507,7 +507,6 @@ void mainController::runCli() { cli::sendResponse("invalid argument\n"); break; } - return; } else { cli::sendResponse("invalid number of arguments\n"); }