Merge Develop into Main

lib/application/maincontroller.cpp

@@ -496,7 +496,20 @@ void mainController::runCli() {
                             break;
 
                         case cliCommand::getMeasurements:
-                            showMeasurements();
+                            if (theCommand.nmbrOfArguments == 0) {
+                                showMeasurements();
+                            }
+                            if (theCommand.nmbrOfArguments == 1) {
+                                if (strncmp(theCommand.arguments[0], "csv", 3) == 0) {
+                                    showMeasurementsCsv();
+                                    break;
+                                } else {
+                                    cli::sendResponse("invalid argument\n");
+                                    break;
+                                }
+                            } else {
+                                cli::sendResponse("invalid number of arguments\n");
+                            }
                             break;
 
                         case cliCommand::getLoRaWANStatus:
@@ -581,6 +594,7 @@ void mainController::showHelp() {
     cli::sendResponse("? : show help\n");
     cli::sendResponse("gds : show device status\n");
     cli::sendResponse("gms : show recorded measurements status\n");
+    cli::sendResponse("gm (csv) : show recorded measurements, optionally as CSV\n");
     cli::sendResponse("gls : show LoRaWAN status\n");
     cli::sendResponse("el : enable logging\n");
     cli::sendResponse("dl : disable logging\n");
@@ -809,6 +823,7 @@ void mainController::setSensor(const cliCommand& theCommand) {
 }
 
 void mainController::showMeasurementsStatus() {
+    cli::sendResponse("wait...");
     measurementGroup tmpGroup;
     uint32_t startOffset = measurementGroupCollection::getOldestMeasurementOffset();
     uint32_t endOffset   = measurementGroupCollection::getNewMeasurementsOffset();
@@ -831,6 +846,7 @@ void mainController::showMeasurementsStatus() {
         newestMeasurementTime = tmpGroup.getTimeStamp();
         offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements());
     }
+    cli::sendResponse("\b\b\b\b\b\b\b");
     cli::sendResponse("%u measurements in %u groups\n", nmbrOfMeasurements, nmbrOfGroups);
     cli::sendResponse("oldoffset %u %s", measurementGroupCollection::getOldestMeasurementOffset(), ctime(&oldestMeasurementTime));
     cli::sendResponse("newoffset %u %s", measurementGroupCollection::getNewMeasurementsOffset(), ctime(&newestMeasurementTime));
@@ -847,14 +863,12 @@ void mainController::showMeasurements() {
     uint32_t offset{startOffset};
     uint32_t nmbrOfGroups{0};
     uint32_t nmbrOfMeasurements{0};
-    time_t oldestMeasurementTime;
-
-    measurementGroupCollection::get(tmpGroup, offset);
-    oldestMeasurementTime = tmpGroup.getTimeStamp();
+    time_t timeStamp;
 
     while (offset < endOffset) {
         measurementGroupCollection::get(tmpGroup, offset);
-        cli::sendResponse("%d %s", nmbrOfGroups, ctime(&oldestMeasurementTime));        // 1 line per measurementGroup
+        timeStamp = tmpGroup.getTimeStamp();
+        cli::sendResponse("%d %s", nmbrOfGroups, ctime(&timeStamp));
         nmbrOfMeasurements = tmpGroup.getNumberOfMeasurements();
         for (uint32_t measurementIndex = 0; measurementIndex < nmbrOfMeasurements; measurementIndex++) {        // then per measurement part of this group
             uint32_t tmpDeviceIndex  = tmpGroup.getDeviceIndex(measurementIndex);
@@ -874,10 +888,54 @@ void mainController::showMeasurements() {
             } else {
                 cli::sendResponse(" %d", intPart);
             }
-
             cli::sendResponse(" %s\n", sensorDeviceCollection::units(tmpDeviceIndex, tmpChannelIndex));
         }
         nmbrOfGroups++;
         offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements());
     }
+}
+
+void mainController::showMeasurementsCsv() {
+    measurementGroup tmpGroup;
+    uint32_t startOffset = measurementGroupCollection::getOldestMeasurementOffset();
+    uint32_t endOffset   = measurementGroupCollection::getNewMeasurementsOffset();
+    if (endOffset < startOffset) {
+        endOffset += nonVolatileStorage::getMeasurementsAreaSize();
+    }
+    uint32_t offset{startOffset};
+    uint32_t nmbrOfGroups{0};
+    uint32_t nmbrOfMeasurements{0};
+    time_t timeStamp;
+
+    cli::sendResponse("year,month,day,hours,minutes,seconds,device,channel,value,units\n");
+
+    while (offset < endOffset) {
+        measurementGroupCollection::get(tmpGroup, offset);
+        timeStamp                 = tmpGroup.getTimeStamp();
+        const struct tm* timeInfo = gmtime(&timeStamp);
+        nmbrOfMeasurements        = tmpGroup.getNumberOfMeasurements();
+        for (uint32_t measurementIndex = 0; measurementIndex < nmbrOfMeasurements; measurementIndex++) {        // then per measurement part of this group
+            cli::sendResponse("%d,%d,%d,%d,%d,%d,", timeInfo->tm_year + 1900, timeInfo->tm_mon + 1, timeInfo->tm_mday, timeInfo->tm_hour, timeInfo->tm_min, timeInfo->tm_sec);
+            uint32_t tmpDeviceIndex  = tmpGroup.getDeviceIndex(measurementIndex);
+            uint32_t tmpChannelIndex = tmpGroup.getChannelIndex(measurementIndex);
+            float tmpValue           = tmpGroup.getValue(measurementIndex);
+
+            uint32_t decimals = sensorDeviceCollection::decimals(tmpDeviceIndex, tmpChannelIndex);
+            uint32_t intPart  = integerPart(tmpValue, decimals);
+
+            cli::sendResponse("%s,", sensorDeviceCollection::name(tmpDeviceIndex));
+            cli::sendResponse("%s,", sensorDeviceCollection::name(tmpDeviceIndex, tmpChannelIndex));
+
+            if (decimals > 0) {
+                uint32_t fracPart;
+                fracPart = fractionalPart(tmpValue, decimals);
+                cli::sendResponse("%d.%d,", intPart, fracPart);
+            } else {
+                cli::sendResponse("%d,", intPart);
+            }
+            cli::sendResponse("%s\n", sensorDeviceCollection::units(tmpDeviceIndex, tmpChannelIndex));
+        }
+        nmbrOfGroups++;
+        offset += measurementGroup::lengthInBytes(tmpGroup.getNumberOfMeasurements());
+    }
 }

lib/application/maincontroller.hpp

@@ -70,6 +70,7 @@ class mainController {
     static void showDeviceStatus();
     static void showMeasurementsStatus();
     static void showMeasurements();
+    static void showMeasurementsCsv();
     static void showNetworkStatus();
     static void setDeviceAddress(const cliCommand& aCommand);
     static void setNetworkKey(const cliCommand& aCommand);

lib/logging/clicommand.hpp

@@ -41,7 +41,6 @@ class cliCommand {
     static constexpr uint32_t setDisplay{'s' * 256 + 'd'};                                 // set display         : sd line# deviceIndex channelIndex
     static constexpr uint32_t setSensor{'s' * 256 + 's'};                                  // set sensor          : sd deviceIndex channelIndex oversampling prescaler
     static constexpr uint32_t softwareReset{'s' * 65536 + 'w' * 256 + 'r'};                // restart device - soft reset
-    static constexpr uint32_t hardwareReset{'h' * 256 + 'w' * 256 + 'r'};                  // reset device - arm for hard reset, reset occurs after removing USB to prevent going into bootloader mode
 
     uint32_t nmbrOfArguments{0};
     char commandAsString[maxCommandLineLength];

lib/sensors/bme680.cpp

@@ -22,7 +22,7 @@ bool mockBME680Present{false};
 sensorDeviceState bme680::state{sensorDeviceState::unknown};
 sensorChannel bme680::channels[nmbrChannels] = {
     {1, "temperature", "~C"},
-    {0, "relativeHumidity", "%RH"},
+    {0, "relativeHumidity", "%"},
     {0, "barometricPressure", "hPa"},
 };
 

lib/sensors/measurementgroupcollection.cpp

@@ -69,8 +69,6 @@ void measurementGroupCollection::get(measurementGroup& aMeasurementGroup, uint32
     uint32_t remainingLength{lengthInBytes};
     uint32_t currentOffset{offset};
 
-    // TODO : wakeUp NVS if needed
-
     while (remainingLength > 0) {
         uint32_t bytesInThisPage = nonVolatileStorage::bytesInCurrentPage(getAddressFromOffset(currentOffset), remainingLength);
         nonVolatileStorage::readInPage(getAddressFromOffset(currentOffset), remainingData, bytesInThisPage);
@@ -80,8 +78,6 @@ void measurementGroupCollection::get(measurementGroup& aMeasurementGroup, uint32
     }
 
     aMeasurementGroup.fromBytes(buffer);
-
-    // TODO : put NVS back to sleep if it was sleeping
 }
 
 uint32_t measurementGroupCollection::getFreeSpace() {

lib/sensors/scd40.cpp

@@ -19,7 +19,7 @@ bool mockSCD40Present{false};
 sensorDeviceState scd40::state{sensorDeviceState::unknown};
 sensorChannel scd40::channels[nmbrChannels] = {
     {1, "temperature", "~C"},
-    {0, "relativeHumidity", "%RH"},
+    {0, "relativeHumidity", "%"},
     {0, "CO2", "ppm"},
 };
 

lib/sensors/scd40.md

@@ -0,0 +1,3 @@
+Important note : I found the temperature readings of the sensor to be very inaccurate : 4 deg C lower than BME680. 
+It seems this sensor is not factory calibrated and need manual calibration via its offset register. 
+As we are using the sensor in low power mode, it's maybe dissipating less heat and as such giving a too low reading.

lib/sensors/sensorchannel.md

@@ -1,84 +1,46 @@
-# prescaling and oversampling
+# sensor oversampling and output frequency
 
-1. Prescaling means that a sensorChannel does not take a sample (measurement) every RTC tick (30s), but rather once per n RTC ticks.
-The prescaler value can be set from 0 .. 8190
-    prescaler = 0 : this sensorChannel does NOT take any samples = deactivation of the sensorchannel
-    prescaler = 1 .. 8190 : take a sample every 1..8190 ticks. The prescaleCounter runs from (prescaler-1) to 0
+## oversampling
 
-2. Oversampling means that a sensorChannel takes multiple samples, and then averages them into a single output
-The oversampling can be set from 0..7, resulting in averaging 1..8 samples to a single output.
-The oversamplingCounter runs from (oversampling) to 0
+You can set any sensorChannel to take multiple samples and average them to a single output. This filters noise and improves the sensors accuracy.
+Select oversampling from 4 possible values :
+0 : no oversampling, each sample is an output 
+1 : average 2 samples to an output
+2 : average 4 samples to an output
+3 : average 10 samples to an output
 
-oversampling (3 bits) and prescaler (13 bits) are compressed into a single uint16 when storing into EEPROM
-As this has 0xFF as blank value, a value, the max valid value for prescaler is 8190 (io 8191), so we can recognize uninitialized eeprom.
+# output frequency
 
-Note : (minutesBetweenOutput * 2) must be a multiple of numberOfSamplesToAverage
-Note : ((minutesBetweenOutput * 2) % numberOfSamplesToAverage) == 0
-
-
-I could store prescaler more efficiently..
-0 = off
+You can select an output frequency for any sensorChannel. Select from 14 possible values :
+0 = off : channel will not produce outputs
 1 = every minute
 2 = every 2 minutes
 3 = every 5 minutes
 4 = every 10 minutes
 5 = every 15 minutes
 6 = every 30 minutes
-7 = every 60 minutes
+7 = every hour
 8 = every 2 hours
-9 = every 6 hours
-10 = every 12 hours
-11 = every 24 hours
-
-prescaler
-
-0 = off
-1 = every minute, prescaler = 2
-2 = every 2 minutes, prescaler = 4
-3 = every 5 minutes, prescaler = 10
-4 = every 10 minutes, prescaler = 20
-5 = every 15 minutes, prescaler = 30
-6 = every 30 minutes, prescaler = 60
-7 = every hour, prescaler = 120
-8 = every 2 hours, prescaler = 240
-9 = every 4 hours, prescaler = 480
-10 = every 6 hours, prescaler = 720
-11 = every 12 hours, prescaler = 1440
-12 = every 24 hours, prescaler = 2880
-13 = every 48 hours, prescaler = 5760
-
-4 bits needed
-
-oversampling
-1
-2
-4
-8
-
-2 bits needed
-
-total 6 bits needed, fits in a byte
-
-
-
-
-# TODO
+9 = every 4 hours
+10 = every 6 hours
+11 = every 12 hours
+12 = every 24 hours
+13 = every 48 hours
 
-* calibratie coefficienten in array steken
-* code refactoren voor meer performantie : meer proberen op voorhand te berekenen
-* store the oversampling / prescaling etc for the measurement channels in NVS so the settings remain after reset
+# combining oversampling and output frequency
 
-# TODO
-* the SX126x also needs two sensorChannels, for RSSI and SNR, as we will measure those and log, store, send them
+The basic realTime clock tick is 30 seconds. This means that the sensor cannot be sampled more than once per 30 seconds.
+As a result some combinations of oversampling and output frequency are not valid, eg
+- oversampling set to 10
+- output frequency set to every minute
+This would require 10 samples per minute, and the maximum is 2 samples per minute
+When setting incompatible values, the oversampling will be reduced to be compatible with output frequency
 
-We need an additional setting on each sensorChannel what to do with the measurement :
-* nothing
-* log to UART
-* store into EEPROM
-* transmit over LoRaWAN
+# optimizing oversampling and output frequency
 
-Showing on the display is controlled at the display itself, where we define up to three sensorChannelTypes to be shown on the top 3 lines of the screen. Battery SOC and network is always shown at bottom line
+For optimal (lowest) power consumption, it is best to consider 2 things :
+* give all sensors settings so that the samples and outputs are multiples of each other (or equal). 
+  * good example : one sensor every 60 minutes, other sensor 15 minutes (60 is multiple of 15)
+  * bad example : noe sensor every 5 minutes, other sensor every 2 minutes (5 is not a multiple of 2)
+* even when these settings are optimized, it is recommended to restart the device (SWR) so all counters are synchronized 
 
-# TODO
-At startup, as not all samples in the array are written, the averaging won't work. Could solve this by doing a first sample and set all values to this value
-Warning : Lux sensor does something wrong on first sample after power up

lib/sensors/sensordevicecollection.cpp

@@ -141,6 +141,7 @@ bool sensorDeviceCollection::isSamplingReady() {
                     if (scd40::getState() != sensorDeviceState::sleeping) {
                         return false;
                     }
+                    break;
                 // Add more types of sensors here
                 default:
                     break;

lib/sensors/sht40.cpp

@@ -23,7 +23,7 @@ uint8_t sht40::i2cAddress;
 sensorDeviceState sht40::state{sensorDeviceState::unknown};
 sensorChannel sht40::channels[nmbrChannels] = {
     {1, "temperature", "~C"},
-    {0, "relativeHumidity", "%RH"},
+    {0, "relativeHumidity", "%"},
 };
 
 uint32_t sht40::rawDataTemperature;

test/generic/test_sensordevicecollection/test.cpp

@@ -167,7 +167,7 @@ void test_channelProperties() {
     TEST_ASSERT_EQUAL_STRING("relativeHumidity", sensorDeviceCollection::name(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::relativeHumidity));
     TEST_ASSERT_EQUAL_STRING("barometricPressure", sensorDeviceCollection::name(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::barometricPressure));
     TEST_ASSERT_EQUAL_STRING("~C", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::temperature));
-    TEST_ASSERT_EQUAL_STRING("%RH", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::relativeHumidity));
+    TEST_ASSERT_EQUAL_STRING("%", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::relativeHumidity));
     TEST_ASSERT_EQUAL_STRING("hPa", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::bme680), bme680::barometricPressure));
 
     // mockTSL2591Present = true;
@@ -188,7 +188,7 @@ void test_channelName() {
 void test_units() {
     TEST_ASSERT_EQUAL_STRING("lux", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::tsl2591), tsl2591::visibleLight));
     TEST_ASSERT_EQUAL_STRING("~C", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::sht40), sht40::temperature));
-    TEST_ASSERT_EQUAL_STRING("%RH", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::sht40), sht40::relativeHumidity));
+    TEST_ASSERT_EQUAL_STRING("%", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::sht40), sht40::relativeHumidity));
     TEST_ASSERT_EQUAL_STRING("invalid index", sensorDeviceCollection::units(static_cast<uint32_t>(sensorDeviceType::nmbrOfKnownDevices), 0));
 }