diff --git a/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java b/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java index 3ecce07..e128274 100644 --- a/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java +++ b/src/integration/java/org/jlab/myquery/MyStatsQueryTest.java @@ -17,12 +17,150 @@ public class MyStatsQueryTest { @Test public void basicUsageTest() throws IOException, InterruptedException { + /* + This test runs a basic myStats query across multiple channels and bins. Only channel1 bin + "2019-08-12T00:24:00", was fully checked for accuracy. This same bin is then queried again in basicUsageTest2 + as the only bin to ensure that the binning logic is correct for both single and multiple bins. This is OK as + the FloatAnalysisStream handles should be tested more thoroughly as part of jmyapi and that handles all the + analysis logic outside bin creation. + */ HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() .uri( URI.create( - "http://localhost:8080/myquery/mystats?c=channel1,channel4&b=2019-08-12&e=2019-08-12+01%3A00%3A00&n=5&m=docker&f=3&v=6")) + "http://localhost:8080/myquery/mystats?c=channel1,channel4&b=2019-08-12+00%3A23%3A50&e=2019-08-12+00%3A24%3A20&n=3&m=docker&f=&v=")) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + + String jsonString = + """ + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12T00:23:50", + "eventCount": 4, + "updateCount": 3, + "duration": 10, + "integration": 958.25815691380921634845435619354248046875, + "max": 95.9059, + "mean": 95.8258, + "min": 95.3710, + "rms": 95.8259, + "stdev": 0.131686 + }, + { + "begin": "2019-08-12T00:24:00", + "eventCount": 6, + "updateCount": 5, + "duration": 10, + "integration": 955.6611437809115159325301647186279296875, + "max": 95.6961, + "mean": 95.5661, + "min": 95.3710, + "rms": 95.5662, + "stdev": 0.110596 + }, + { + "begin": "2019-08-12T00:24:10", + "eventCount": 7, + "updateCount": 6, + "duration": 10, + "integration": 959.089333145524506107904016971588134765625, + "max": 96.2163, + "mean": 95.9089, + "min": 95.6568, + "rms": 95.9090, + "stdev": 0.144037 + } + ], + "returnCount": 3 + }, + "channel4": { + "metadata": { + "name": "channel4", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12T00:23:50", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12T00:24:00", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + }, + { + "begin": "2019-08-12T00:24:10", + "eventCount": 0, + "updateCount": 0, + "duration": null, + "integration": null, + "max": null, + "mean": null, + "min": null, + "rms": null, + "stdev": null + } + ], + "returnCount": 3 + } + } + }"""; + String exp; + try (JsonReader r = Json.createReader(new StringReader(jsonString))) { + exp = r.readObject().toString(); + } + + try (JsonReader reader = Json.createReader(new StringReader(response.body()))) { + JsonObject json = reader.readObject(); + assertEquals(exp, json.toString()); + } + } + + @Test + public void basicUsageTest2() throws IOException, InterruptedException { + // This test should match the corresponding bin in basicUsageTest. This helps ensure that the + // binning logic is + // correct. See basicUsageTest comments for more details. + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri( + URI.create( + "http://localhost:8080/myquery/mystats?c=channel1&b=2019-08-12+00%3A24%3A00&e=2019-08-12+00%3A24%3A10&n=1&m=docker&f=3&v=6")) .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); @@ -42,141 +180,20 @@ public void basicUsageTest() throws IOException, InterruptedException { "active": true }, "data": [ - { - "begin": "2019-08-12 00:00:00.000", - "eventCount": 335, - "updateCount": 334, - "duration": 719.1279857158660888671875, - "integration": 68755.144563262627343647181987762451171875, - "max": 96.8135, - "mean": 95.6091, - "min": 94.4322, - "rms": 95.6100, - "stdev": 0.436171 - }, - { - "begin": "2019-08-12 00:12:00.000", - "eventCount": 369, - "updateCount": 368, - "duration": 719.1194341182708740234375, - "integration": 68944.138235634469310753047466278076171875, - "max": 96.8513, - "mean": 95.8730, - "min": 94.9427, - "rms": 95.8738, - "stdev": 0.390679 - }, { "begin": "2019-08-12 00:24:00.000", - "eventCount": 343, - "updateCount": 342, - "duration": 718.11330127716064453125, - "integration": 68751.66884353742352686822414398193359375, - "max": 96.8915, - "mean": 95.7393, - "min": 95.0116, - "rms": 95.7400, - "stdev": 0.352170 - }, - { - "begin": "2019-08-12 00:36:00.000", - "eventCount": 317, - "updateCount": 316, - "duration": 718.073926448822021484375, - "integration": 65907.665449329026159830391407012939453125, - "max": 96.9524, - "mean": 91.7840, - "min": 0, - "rms": 93.2710, - "stdev": 16.5887 - }, - { - "begin": "2019-08-12 00:48:00.000", - "eventCount": 352, - "updateCount": 351, - "duration": 714.1164848804473876953125, - "integration": 68422.188102417494519613683223724365234375, - "max": 96.9037, - "mean": 95.8138, - "min": 94.8486, - "rms": 95.8148, - "stdev": 0.453055 + "eventCount": 6, + "updateCount": 5, + "duration": 10, + "integration": 955.6611437809115159325301647186279296875, + "max": 95.6961, + "mean": 95.5661, + "min": 95.3710, + "rms": 95.5662, + "stdev": 0.110596 } ], - "returnCount": 5 - }, - "channel4": { - "metadata": { - "name": "channel4", - "datatype": "DBR_DOUBLE", - "datasize": 1, - "datahost": "mya", - "ioc": null, - "active": true - }, - "data": [ - { - "begin": "2019-08-12 00:00:00.000", - "eventCount": 0, - "updateCount": 0, - "duration": null, - "integration": null, - "max": null, - "mean": null, - "min": null, - "rms": null, - "stdev": null - }, - { - "begin": "2019-08-12 00:12:00.000", - "eventCount": 0, - "updateCount": 0, - "duration": null, - "integration": null, - "max": null, - "mean": null, - "min": null, - "rms": null, - "stdev": null - }, - { - "begin": "2019-08-12 00:24:00.000", - "eventCount": 0, - "updateCount": 0, - "duration": null, - "integration": null, - "max": null, - "mean": null, - "min": null, - "rms": null, - "stdev": null - }, - { - "begin": "2019-08-12 00:36:00.000", - "eventCount": 0, - "updateCount": 0, - "duration": null, - "integration": null, - "max": null, - "mean": null, - "min": null, - "rms": null, - "stdev": null - }, - { - "begin": "2019-08-12 00:48:00.000", - "eventCount": 0, - "updateCount": 0, - "duration": null, - "integration": null, - "max": null, - "mean": null, - "min": null, - "rms": null, - "stdev": null - } - ], - "returnCount": 5 + "returnCount": 1 } } }"""; @@ -192,13 +209,18 @@ public void basicUsageTest() throws IOException, InterruptedException { } @Test - public void unsupportedTypeTest() throws IOException, InterruptedException { + public void basicUsageTest3() throws IOException, InterruptedException { + // This test checks that the binning logic is correct the binning moves past the last event in + // the time range. + // At one point bins after the last event were producing null values even though the previous + // event had a value. + // See basicUsageTest comments for more details. HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() .uri( URI.create( - "http://localhost:8080/myquery/mystats?c=channel1,channel2&b=2019-08-12+00%3A01%3A00&e=2019-08-19+02%3A00%3A00&n=2&m=docker&f=&v=")) + "http://localhost:8080/myquery/mystats?c=channel1&b=2019-08-12+23%3A59%3A50&e=2019-08-13+00%3A00%3A20&n=3&m=docker&f=&v=")) .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); @@ -206,7 +228,7 @@ public void unsupportedTypeTest() throws IOException, InterruptedException { String jsonString = """ - { + { "channels": { "channel1": { "metadata": { @@ -219,37 +241,118 @@ public void unsupportedTypeTest() throws IOException, InterruptedException { }, "data": [ { - "begin": "2019-08-12T00:01:00", - "eventCount": 32963, - "updateCount": 32962, - "duration": 305970, - "integration": 27214184.8855658471584320068359375, - "max": 103.997, - "mean": 88.9440, - "min": 0, - "rms": 91.1415, - "stdev": 19.8931 + "begin": "2019-08-12T23:59:50", + "eventCount": 7, + "updateCount": 6, + "duration": 10, + "integration": 949.006091671968533773906528949737548828125, + "max": 95.1860, + "mean": 94.9006, + "min": 94.5131, + "rms": 94.9009, + "stdev": 0.244360 + }, + { + "begin": "2019-08-13T00:00:00", + "eventCount": 2, + "updateCount": 1, + "duration": 10, + "integration": 951.797027587890625, + "max": 95.1797, + "mean": 95.1797, + "min": 95.1797, + "rms": 95.1797, + "stdev": 0 }, { - "begin": "2019-08-15T13:00:30", + "begin": "2019-08-13T00:00:10", "eventCount": 2, "updateCount": 1, - "duration": 305970, - "integration": 29282429.886932373046875, - "max": 95.7036, - "mean": 95.7036, - "min": 95.7036, - "rms": 95.7036, + "duration": 10, + "integration": 951.797027587890625, + "max": 95.1797, + "mean": 95.1797, + "min": 95.1797, + "rms": 95.1797, "stdev": 0 } ], - "returnCount": 2 - }, - "channel2": { - "error": "This myStats only supports FloatEvents - not 'org.jlab.mya.event.IntEvent'." + "returnCount": 3 } } }"""; + String exp; + try (JsonReader r = Json.createReader(new StringReader(jsonString))) { + exp = r.readObject().toString(); + } + + try (JsonReader reader = Json.createReader(new StringReader(response.body()))) { + JsonObject json = reader.readObject(); + assertEquals(exp, json.toString()); + } + } + + @Test + public void unsupportedTypeTest() throws IOException, InterruptedException { + // Check that we can handle a channel with an unsupported event type without losing data for + // other channels. + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri( + URI.create( + "http://localhost:8080/myquery/mystats?c=channel1,channel2&b=2019-08-12+00%3A01%3A00&e=2019-08-19+02%3A00%3A00&n=2&m=docker&f=&v=")) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + + String jsonString = + """ + { + "channels": { + "channel1": { + "metadata": { + "name": "channel1", + "datatype": "DBR_DOUBLE", + "datasize": 1, + "datahost": "mya", + "ioc": null, + "active": true + }, + "data": [ + { + "begin": "2019-08-12T00:01:00", + "eventCount": 32963, + "updateCount": 32962, + "duration": 305970, + "integration": 27214184.8855658471584320068359375, + "max": 103.997, + "mean": 88.9440, + "min": 0, + "rms": 91.1415, + "stdev": 19.8931 + }, + { + "begin": "2019-08-15T13:00:30", + "eventCount": 2, + "updateCount": 1, + "duration": 305970, + "integration": 29122133.653106689453125, + "max": 95.1797, + "mean": 95.1797, + "min": 95.1797, + "rms": 95.1797, + "stdev": 0 + } + ], + "returnCount": 2 + }, + "channel2": { + "error": "This myStats only supports FloatEvents - not 'org.jlab.mya.event.IntEvent'." + } + } + }"""; String exp; try (JsonReader r = Json.createReader(new StringReader(jsonString))) { diff --git a/src/main/java/org/jlab/myquery/MyStatsController.java b/src/main/java/org/jlab/myquery/MyStatsController.java index 7c4c79f..a6fe107 100644 --- a/src/main/java/org/jlab/myquery/MyStatsController.java +++ b/src/main/java/org/jlab/myquery/MyStatsController.java @@ -16,6 +16,7 @@ import java.util.logging.Logger; import java.util.regex.PatternSyntaxException; import org.jlab.mya.Metadata; +import org.jlab.mya.RunningStatistics; import org.jlab.mya.event.*; import org.jlab.mya.stream.FloatAnalysisStream; @@ -136,20 +137,12 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } PointWebService pws = new PointWebService(deployment); - Map priorEvents = new HashMap<>(); - for (Metadata metadata : metadatas) { - priorEvents.put( - metadata.getName(), pws.findEvent(metadata, updatesOnly, begin, true, true, false)); - } - - Event priorEvent; for (Metadata metadata : metadatas) { if (metadata.getType() != FloatEvent.class) { continue; } - priorEvent = priorEvents.get(metadata.getName()); double interval = ((end.getEpochSecond() + end.getNano() / 1_000_000_000d) - (begin.getEpochSecond() + begin.getNano() / 1_000_000_000d)) @@ -162,16 +155,10 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } else { binEnd = binBegin.plusSeconds((long) interval); } - // Since we provide a priorPoint, the underlying stream should be a BoundaryAwareStream. - try (FloatAnalysisStream fas = - new FloatAnalysisStream( - service.openEventStream( - metadata, updatesOnly, binBegin, binEnd, priorEvent, metadata.getType()))) { - while (fas.read() != null) { - // Read through the entire stream. We only want statistics from it - } - results.add(metadata.getName(), binBegin, fas.getLatestStats()); - } + results.add( + metadata.getName(), + binBegin, + getBinStats(binBegin, binEnd, metadata, service, pws, updatesOnly)); } } @@ -237,4 +224,43 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) } } } + + /** + * Gets the statistics for a bin. Handle this as a separate method to ensure proper handling of + * prior events. + * + * @implNote There was also an intention to make testing easier, but mocking the web services was + * not straightforward. + * @param binBegin The start of the bin. + * @param binEnd The end of the bin. + * @param metadata The metadata for the channel. + * @param intervalService The interval web service. + * @param pointService The point web service. + * @param updatesOnly Whether to only include updates. + * @return The statistics for the bin. + * @throws Exception If an error occurs. + */ + private static RunningStatistics getBinStats( + Instant binBegin, + Instant binEnd, + Metadata metadata, + IntervalWebService intervalService, + PointWebService pointService, + boolean updatesOnly) + throws Exception { + Event priorEvent = pointService.findEvent(metadata, updatesOnly, binBegin, true, true, false); + RunningStatistics stats; + + // Since we provide a priorPoint, the underlying stream should be a BoundaryAwareStream. + try (FloatAnalysisStream fas = + new FloatAnalysisStream( + intervalService.openEventStream( + metadata, updatesOnly, binBegin, binEnd, priorEvent, metadata.getType()))) { + while (fas.read() != null) { + // Read through the entire stream. We only want statistics from it + } + stats = fas.getLatestStats(); + } + return stats; + } }