From e85321cda183bc5278f838b7c128457204e15e73 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 20 Feb 2026 14:46:50 +1100 Subject: [PATCH 01/12] Add shape file format --- .../enumeration/DatasetDownloadEnums.java | 4 +- .../server/core/model/ogc/FeatureRequest.java | 39 +++++++++++++++++++ .../service/wfs/DownloadWfsDataService.java | 18 +++++++-- .../aodn/ogcapi/server/processes/RestApi.java | 8 +++- .../ogcapi/server/processes/RestServices.java | 8 ++-- .../wfs/DownloadWfsDataServiceTest.java | 12 +++--- 6 files changed, 73 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java index ea6d7878..72a8e25d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -20,8 +20,8 @@ public enum Parameter { COLLECTION_TITLE("collection_title"), FULL_METADATA_LINK("full_metadata_link"), SUGGESTED_CITATION("suggested_citation"), - KEY("key") - ; + KEY("key"), + OUTPUT_FORMAT("output_format"); private final String value; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index a419be9e..fa6112de 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -3,6 +3,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.*; import lombok.experimental.SuperBuilder; +import org.springframework.http.MediaType; import java.io.Serializable; import java.math.BigDecimal; @@ -28,6 +29,41 @@ public static PropertyName fromString(String input) { } } + @Getter + public enum GeoServerOutputFormat { + GML2("GML2"), + GML3("GML3"), + GML32("gml32"), + SHAPE_ZIP("shape-zip"), // also accepted as "SHAPE-ZIP" + CSV("csv"), + JSON(MediaType.APPLICATION_JSON_VALUE), // also "json" for backward compatibility + GEOJSON("application/geo+json"), + KML("KML"), + UNKNOWN("unknown"); + + private final String value; + + GeoServerOutputFormat(String value) { + this.value = value; + } + + public static GeoServerOutputFormat fromString(String s) { + if (s == null) return UNKNOWN; + String key = s.trim().toUpperCase().replace("-", "_"); + try { + return valueOf(key); + } + catch (IllegalArgumentException e) { + return UNKNOWN; // or throw custom exception + } + } + + @Override + public String toString() { + return value; + } + } + @Schema(description = "Property to be return") private List properties; @@ -52,6 +88,9 @@ public static PropertyName fromString(String input) { @Schema(description = "Y") private BigDecimal y; + @Schema(description = "Data output format") + private GeoServerOutputFormat outputFormat; + @Schema(description = "Wave buoy name") private String waveBuoy; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java index e3e901e8..1f3a2c3d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java @@ -101,7 +101,8 @@ public String prepareWfsRequestUrl( String endDate, Object multiPolygon, List fields, - String layerName) { + String layerName, + String outputFormat) { DescribeLayerResponse describeLayerResponse = wmsServer.describeLayer(uuid, FeatureRequest.builder().layerName(layerName).build()); @@ -114,7 +115,13 @@ public String prepareWfsRequestUrl( wfsServerUrl = describeLayerResponse.getLayerDescription().getWfs(); wfsTypeName = describeLayerResponse.getLayerDescription().getQuery().getTypeName(); - wfsFieldModel = wfsServer.getDownloadableFields(uuid, WfsServer.WfsFeatureRequest.builder().layerName(wfsTypeName).server(wfsServerUrl).build()); + wfsFieldModel = wfsServer.getDownloadableFields( + uuid, + WfsServer.WfsFeatureRequest.builder() + .layerName(wfsTypeName) + .server(wfsServerUrl) + .build() + ); log.info("WFSFieldModel by describeLayer: {}", wfsFieldModel); } else { Optional featureServerUrl = wfsServer.getFeatureServerUrlByTitle(uuid, layerName); @@ -137,7 +144,12 @@ public String prepareWfsRequestUrl( String cqlFilter = buildCqlFilter(validStartDate, validEndDate, multiPolygon, wfsFieldModel); // Build final WFS request URL - String wfsRequestUrl = wfsServer.createWfsRequestUrl(wfsServerUrl, wfsTypeName, fields, cqlFilter, null); + String wfsRequestUrl = wfsServer.createWfsRequestUrl( + wfsServerUrl, + wfsTypeName, + fields, + cqlFilter, + outputFormat); log.info("Prepared WFS request URL: {}", wfsRequestUrl); return wfsRequestUrl; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index ff9fe909..36ed6d73 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import au.org.aodn.ogcapi.server.core.model.enumeration.InlineResponseKeyEnum; import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; +import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.media.Schema; @@ -125,14 +126,19 @@ public SseEmitter downloadWfsSse( String startDate = body.getInputs().get(DatasetDownloadEnums.Parameter.START_DATE.getValue()).toString(); String endDate = body.getInputs().get(DatasetDownloadEnums.Parameter.END_DATE.getValue()).toString(); String layerName = body.getInputs().get(DatasetDownloadEnums.Parameter.LAYER_NAME.getValue()).toString(); + String outputFormat = body.getInputs().get(DatasetDownloadEnums.Parameter.OUTPUT_FORMAT.getValue()).toString(); var multiPolygon = body.getInputs().get(DatasetDownloadEnums.Parameter.MULTI_POLYGON.getValue()); List fields = body.getInputs().get(DatasetDownloadEnums.Parameter.FIELDS.getValue()) instanceof List list ? list.stream().map(String::valueOf).toList() : null; + if(FeatureRequest.GeoServerOutputFormat.fromString(outputFormat) == FeatureRequest.GeoServerOutputFormat.UNKNOWN) { + throw new IllegalArgumentException(String.format("Output format %s not supported", outputFormat)); + } + return restServices.downloadWfsDataWithSse( - uuid, startDate, endDate, multiPolygon, fields, layerName, emitter + uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat, emitter ); } catch (Exception e) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java index fa338eb9..b0734382 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java @@ -182,6 +182,7 @@ public SseEmitter downloadWfsDataWithSse( Object multiPolygon, List fields, String layerName, + String outputFormat, SseEmitter emitter) { // Set up references for resources that need to be cleaned up @@ -219,9 +220,7 @@ public SseEmitter downloadWfsDataWithSse( cleanupWfsResources.run(); }); - emitter.onError(throwable -> { - WfsErrorHandler.handleError((Exception) throwable, uuid, emitter, cleanupWfsResources); - }); + emitter.onError(throwable -> WfsErrorHandler.handleError((Exception) throwable, uuid, emitter, cleanupWfsResources)); // Validate parameters if (uuid == null || layerName == null || layerName.trim().isEmpty()) { @@ -270,7 +269,8 @@ public SseEmitter downloadWfsDataWithSse( // STEP 3: Do preparation work: Collection lookup from Elasticsearch, WFS validation, Field retrieval, URL building String wfsRequestUrl = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, multiPolygon, fields, layerName); + uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat + ); emitter.send(SseEmitter.event() .name("wfs-request-ready") diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java index 9e3d1adc..489d69d6 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java @@ -113,7 +113,7 @@ public void testPrepareWfsRequestUrl_WithNullDates() { // Test with null dates (non-specified dates from frontend) String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, null, null, null, null, layerName + uuid, null, null, null, null, layerName, null ); // Verify URL doesn't contain temporal filter when dates are null @@ -145,7 +145,7 @@ public void testPrepareWfsRequestUrl_WithEmptyDates() { // Test with empty string dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, "", "", null, null, layerName + uuid, "", "", null, null, layerName, null ); // Verify URL doesn't contain temporal filter when dates are empty @@ -179,7 +179,7 @@ public void testPrepareWfsRequestUrl_WithValidDates() { // Test with valid dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, null, null, layerName + uuid, startDate, endDate, null, null, layerName, null ); // Verify URL contains temporal filter when valid dates are provided @@ -215,7 +215,7 @@ public void testPrepareWfsRequestUrl_WithOnlyStartDate() { // Test with only start date (end date is null) String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, null, null, null, layerName + uuid, startDate, null, null, null, layerName, null ); // Verify URL doesn't contain temporal filter when only one date is provided @@ -249,7 +249,7 @@ public void testPrepareWfsRequestUrl_WithMMYYYYFormat() { // Test with MM-YYYY format dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, null, null, layerName + uuid, startDate, endDate, null, null, layerName, null ); // Verify URL contains temporal filter with converted dates @@ -276,7 +276,7 @@ public void testPrepareWfsRequestUrl_NoWfsServerUrl() { // Test with no WFS server URL available Exception exception = assertThrows(IllegalArgumentException.class, () -> downloadWfsDataService.prepareWfsRequestUrl( - uuid, null, null, null, null, layerName + uuid, null, null, null, null, layerName, null )); assertTrue(exception.getMessage().contains("No WFS server URL found")); From 573961b573b8844abdce05f786440ca9338a4113 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 20 Feb 2026 15:00:00 +1100 Subject: [PATCH 02/12] Update test --- .../server/core/model/ogc/FeatureRequest.java | 14 ++++---- .../wfs/DownloadWfsDataServiceTest.java | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index fa6112de..9ff4792d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -35,7 +35,7 @@ public enum GeoServerOutputFormat { GML3("GML3"), GML32("gml32"), SHAPE_ZIP("shape-zip"), // also accepted as "SHAPE-ZIP" - CSV("csv"), + CSV("text/csv"), JSON(MediaType.APPLICATION_JSON_VALUE), // also "json" for backward compatibility GEOJSON("application/geo+json"), KML("KML"), @@ -48,14 +48,12 @@ public enum GeoServerOutputFormat { } public static GeoServerOutputFormat fromString(String s) { - if (s == null) return UNKNOWN; - String key = s.trim().toUpperCase().replace("-", "_"); - try { - return valueOf(key); - } - catch (IllegalArgumentException e) { - return UNKNOWN; // or throw custom exception + for(GeoServerOutputFormat f : GeoServerOutputFormat.values()) { + if(f.value.equalsIgnoreCase(s)) { + return f; + } } + return UNKNOWN; // or throw custom exception } @Override diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java index 489d69d6..b8f7de85 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java @@ -281,4 +281,40 @@ public void testPrepareWfsRequestUrl_NoWfsServerUrl() { assertTrue(exception.getMessage().contains("No WFS server URL found")); } + + @Test + public void verifyRequestUrlGenerateCorrect() { + // Setup + String uuid = "test-uuid"; + String layerName = "test:layer"; + String startDate = "01-2024"; // MM-YYYY format + String endDate = "12-2024"; // MM-YYYY format + WfsFields wfsFieldModel = createTestWFSFieldModel(); + + DescribeLayerResponse describeLayerResponse = mock(DescribeLayerResponse.class); + DescribeLayerResponse.LayerDescription layerDescription = mock(DescribeLayerResponse.LayerDescription.class); + DescribeLayerResponse.Query query = mock(DescribeLayerResponse.Query.class); + + when(describeLayerResponse.getLayerDescription()).thenReturn(layerDescription); + when(layerDescription.getWfs()).thenReturn("https://test.com/geoserver/wfs"); + when(layerDescription.getQuery()).thenReturn(query); + when(query.getTypeName()).thenReturn(layerName); + + doReturn(describeLayerResponse) + .when(wmsServer).describeLayer(eq(uuid), any(FeatureRequest.class)); + doReturn(wfsFieldModel) + .when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class)); + + // Test with MM-YYYY format dates + String result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, null, null, layerName, null + ); + + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z", result, "Correct url 1"); + + result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, null, null, layerName, "shape-zip" + ); + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=shape-zip&cql_filter=timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z", result, "Correct url 1"); + } } From 1f5fe322f9184376382d4a20b7060a79091702da Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 09:19:16 +1100 Subject: [PATCH 03/12] Add media type --- .../server/core/model/ogc/FeatureRequest.java | 22 ++++++++++--------- .../ogcapi/server/processes/RestServices.java | 2 ++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index 9ff4792d..a8920f42 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -31,20 +31,22 @@ public static PropertyName fromString(String input) { @Getter public enum GeoServerOutputFormat { - GML2("GML2"), - GML3("GML3"), - GML32("gml32"), - SHAPE_ZIP("shape-zip"), // also accepted as "SHAPE-ZIP" - CSV("text/csv"), - JSON(MediaType.APPLICATION_JSON_VALUE), // also "json" for backward compatibility - GEOJSON("application/geo+json"), - KML("KML"), - UNKNOWN("unknown"); + GML2("GML2", "application/gml+xml"), + GML3("GML3", "application/gml+xml"), + GML32("gml32", "application/gml+xml"), + SHAPE_ZIP("shape-zip", "application/zip"), // also accepted as "SHAPE-ZIP" + CSV("text/csv", "text/csv"), + JSON(MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_JSON_VALUE), // also "json" for backward compatibility + GEOJSON("application/geo+json", "application/geo+json"), + KML("KML", "application/vnd.google-earth.kml+xml"), + UNKNOWN("unknown", "application/octet-stream"); private final String value; + private final String mediaType; - GeoServerOutputFormat(String value) { + GeoServerOutputFormat(String value, String mediaType) { this.value = value; + this.mediaType = mediaType; } public static GeoServerOutputFormat fromString(String s) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java index b0734382..dc779e00 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java @@ -2,6 +2,7 @@ import au.org.aodn.ogcapi.server.core.exception.wfs.WfsErrorHandler; import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; +import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import au.org.aodn.ogcapi.server.core.service.wfs.DownloadWfsDataService; import au.org.aodn.ogcapi.server.core.util.EmailUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -275,6 +276,7 @@ public SseEmitter downloadWfsDataWithSse( emitter.send(SseEmitter.event() .name("wfs-request-ready") .data(Map.of( + "media-type", FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getMediaType(), "message", "Connecting to WFS server...", "timestamp", System.currentTimeMillis() ))); From 4dc07aa3e5ad3387dfe513997563e5d6f1d237cb Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 14:46:17 +1100 Subject: [PATCH 04/12] Reduce chunk complexity --- .../server/core/model/ogc/FeatureRequest.java | 22 ++++--- .../service/wfs/DownloadWfsDataService.java | 64 ++++++------------- .../ogcapi/server/features/RestServices.java | 37 ++++++----- .../ogcapi/server/processes/RestServices.java | 3 +- 4 files changed, 55 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index a8920f42..63dac9ca 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -31,22 +31,24 @@ public static PropertyName fromString(String input) { @Getter public enum GeoServerOutputFormat { - GML2("GML2", "application/gml+xml"), - GML3("GML3", "application/gml+xml"), - GML32("gml32", "application/gml+xml"), - SHAPE_ZIP("shape-zip", "application/zip"), // also accepted as "SHAPE-ZIP" - CSV("text/csv", "text/csv"), - JSON(MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_JSON_VALUE), // also "json" for backward compatibility - GEOJSON("application/geo+json", "application/geo+json"), - KML("KML", "application/vnd.google-earth.kml+xml"), - UNKNOWN("unknown", "application/octet-stream"); + GML2("GML2", "application/gml+xml", "gml"), + GML3("GML3", "application/gml+xml", "gml"), + GML32("gml32", "application/gml+xml", "gml"), + SHAPE_ZIP("shape-zip", "application/zip", "shp"), // also accepted as "SHAPE-ZIP" + CSV("text/csv", "text/csv", "csv"), + JSON(MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_JSON_VALUE, "json"), // also "json" for backward compatibility + GEOJSON("application/geo+json", "application/geo+json", "geojson"), + KML("KML", "application/vnd.google-earth.kml+xml", "kml"), + UNKNOWN("unknown", "application/octet-stream", "bin"); private final String value; private final String mediaType; + private final String fileExtension; - GeoServerOutputFormat(String value, String mediaType) { + GeoServerOutputFormat(String value, String mediaType, String fileExtension) { this.value = value; this.mediaType = mediaType; + this.fileExtension = fileExtension; } public static GeoServerOutputFormat fromString(String s) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java index 1f3a2c3d..eda61043 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java @@ -161,6 +161,7 @@ public void executeWfsRequestWithSse( String wfsRequestUrl, String uuid, String layerName, + String outputFormat, SseEmitter emitter, AtomicBoolean wfsServerResponded) { restTemplate.execute( @@ -187,59 +188,35 @@ public void executeWfsRequestWithSse( int bytesRead; long totalBytes = 0; int chunkNumber = 0; - long lastProgressTime = System.currentTimeMillis(); ByteArrayOutputStream chunkBuffer = new ByteArrayOutputStream(); while ((bytesRead = inputStream.read(buffer)) != -1) { chunkBuffer.write(buffer, 0, bytesRead); totalBytes += bytesRead; - long currentTime = System.currentTimeMillis(); - - // Send chunk when buffer is full OR every 2 seconds - if (chunkBuffer.size() >= 16384 || - (currentTime - lastProgressTime >= 2000)) { - + // Send when buffer >= 16KB **and** size divisible by 3 (for clean Base64) + while (chunkBuffer.size() >= 16384 && chunkBuffer.size() % 3 == 0) { byte[] chunkBytes = chunkBuffer.toByteArray(); - - // Ensure Base64 alignment - // Base64 works in 3-byte groups, so chunk size should be divisible by 3 - int alignedSize = (chunkBytes.length / 3) * 3; - - if (alignedSize > 0) { - // Send the aligned portion - byte[] alignedChunk = Arrays.copyOf(chunkBytes, alignedSize); - String encodedData = Base64.getEncoder().encodeToString(alignedChunk); - - emitter.send(SseEmitter.event() - .name("file-chunk") - .data(Map.of( - "chunkNumber", ++chunkNumber, - "data", encodedData, - "chunkSize", alignedChunk.length, - "totalBytes", totalBytes, - "timestamp", currentTime - )) - .id(String.valueOf(chunkNumber))); - - // Keep the remaining bytes for next chunk - if (alignedSize < chunkBytes.length) { - byte[] remainder = Arrays.copyOfRange(chunkBytes, alignedSize, chunkBytes.length); - chunkBuffer.reset(); - chunkBuffer.write(remainder); - } else { - chunkBuffer.reset(); - } - - lastProgressTime = currentTime; - } + String encodedData = Base64.getEncoder().encodeToString(chunkBytes); + + emitter.send(SseEmitter.event() + .name("file-chunk") + .data(Map.of( + "chunkNumber", ++chunkNumber, + "data", encodedData, + "chunkSize", chunkBytes.length, + "totalBytes", totalBytes, + "timestamp", System.currentTimeMillis() + )) + .id(String.valueOf(chunkNumber))); + + chunkBuffer.reset(); } } - // Send final chunk if any remains + // Final chunk (may not be %3==0, but client Base64 decoder usually handles it) if (chunkBuffer.size() > 0) { - String encodedData = Base64.getEncoder() - .encodeToString(chunkBuffer.toByteArray()); + String encodedData = Base64.getEncoder().encodeToString(chunkBuffer.toByteArray()); emitter.send(SseEmitter.event() .name("file-chunk") .data(Map.of( @@ -258,7 +235,8 @@ public void executeWfsRequestWithSse( "totalBytes", totalBytes, "totalChunks", chunkNumber, "message", "WFS data download completed successfully", - "filename", layerName + "_" + uuid + ".csv" + "media-type", FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getMediaType(), + "filename", String.format("%s_%s.%s", layerName, uuid, FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getFileExtension()) ))); // Close SSE connection with completion diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 03a3f7c3..ed03b1a1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -144,25 +144,30 @@ public ResponseEntity getWfsFieldValue(String collectionId, FeatureRequest re String result = wfsServer.getFieldValues(collectionId, wfsFeatureRequest, new ParameterizedTypeReference<>() {}); - FeatureJSON json = new FeatureJSON(); - try { - @SuppressWarnings("unchecked") - FeatureCollection collection = json.readFeatureCollection(new StringReader(result)); - try(FeatureIterator i = collection.features()) { - Map> results = new HashMap<>(); - while(i.hasNext()) { - SimpleFeature s = i.next(); - s.getProperties() - .forEach(property -> { - results.computeIfAbsent(property.getName().toString(), k -> new ArrayList<>()); - results.get(property.getName().toString()).add(s.getAttribute(property.getName())); - }); + if(result != null) { + FeatureJSON json = new FeatureJSON(); + try { + @SuppressWarnings("unchecked") + FeatureCollection collection = json.readFeatureCollection(new StringReader(result)); + try (FeatureIterator i = collection.features()) { + Map> results = new HashMap<>(); + while (i.hasNext()) { + SimpleFeature s = i.next(); + s.getProperties() + .forEach(property -> { + results.computeIfAbsent(property.getName().toString(), k -> new ArrayList<>()); + results.get(property.getName().toString()).add(s.getAttribute(property.getName())); + }); + } + return ResponseEntity.ok().body(results); } - return ResponseEntity.ok().body(results); + } catch (IOException e) { + return ResponseEntity.badRequest().body(e.getMessage()); } } - catch (IOException e) { - return ResponseEntity.badRequest().body(e.getMessage()); + else { + // Field not found + return ResponseEntity.notFound().build(); } } /** diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java index dc779e00..cbe1e3aa 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java @@ -2,7 +2,6 @@ import au.org.aodn.ogcapi.server.core.exception.wfs.WfsErrorHandler; import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; -import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import au.org.aodn.ogcapi.server.core.service.wfs.DownloadWfsDataService; import au.org.aodn.ogcapi.server.core.util.EmailUtils; import com.fasterxml.jackson.core.JsonProcessingException; @@ -276,7 +275,6 @@ public SseEmitter downloadWfsDataWithSse( emitter.send(SseEmitter.event() .name("wfs-request-ready") .data(Map.of( - "media-type", FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getMediaType(), "message", "Connecting to WFS server...", "timestamp", System.currentTimeMillis() ))); @@ -286,6 +284,7 @@ public SseEmitter downloadWfsDataWithSse( wfsRequestUrl, uuid, layerName, + outputFormat, emitter, wfsServerResponded ); From 9b6c7b93f5a6b6dfa7a86ecaae3b96e41f64a268 Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 17:06:49 +1100 Subject: [PATCH 05/12] Add test case to verify the SSE chunk split for text --- .../service/wfs/DownloadWfsDataService.java | 8 +- .../wfs/DownloadWfsDataServiceTest.java | 2 +- .../wfs/DownloadWfsDataServiceTest.java | 99 +++++++++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java index eda61043..68929752 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.server.core.util.GeometryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Service; @@ -27,17 +28,20 @@ public class DownloadWfsDataService { private final WfsServer wfsServer; private final RestTemplate restTemplate; private final HttpEntity pretendUserEntity; + private final int chunkSize; public DownloadWfsDataService( WmsServer wmsServer, WfsServer wfsServer, RestTemplate restTemplate, - @Qualifier("pretendUserEntity") HttpEntity pretendUserEntity + @Qualifier("pretendUserEntity") HttpEntity pretendUserEntity, + @Value("${app.sse.chunkSize:16384}") int chunkSize ) { this.wmsServer = wmsServer; this.wfsServer = wfsServer; this.restTemplate = restTemplate; this.pretendUserEntity = pretendUserEntity; + this.chunkSize = chunkSize; } /** @@ -195,7 +199,7 @@ public void executeWfsRequestWithSse( totalBytes += bytesRead; // Send when buffer >= 16KB **and** size divisible by 3 (for clean Base64) - while (chunkBuffer.size() >= 16384 && chunkBuffer.size() % 3 == 0) { + while (chunkBuffer.size() >= chunkSize && chunkBuffer.size() % 3 == 0) { byte[] chunkBytes = chunkBuffer.toByteArray(); String encodedData = Base64.getEncoder().encodeToString(chunkBytes); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java index b8f7de85..5d61d15c 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java @@ -60,7 +60,7 @@ public void setUp() { wmsServer = Mockito.spy(new WmsServer(search, wfsServer, pretendUserEntity)); downloadWfsDataService = new DownloadWfsDataService( - wmsServer, wfsServer, restTemplate, pretendUserEntity + wmsServer, wfsServer, restTemplate, pretendUserEntity, 16384 ); } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java new file mode 100644 index 00000000..813ece5f --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -0,0 +1,99 @@ +package au.org.aodn.ogcapi.server.service.wfs; + +import au.org.aodn.ogcapi.server.core.service.wfs.DownloadWfsDataService; +import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; +import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.web.client.ResponseExtractor; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +public class DownloadWfsDataServiceTest { + + @Mock + protected WfsServer wfsServer; + + @Mock + protected WmsServer wmsServer; + + @Mock + protected HttpEntity entity; + + @Test + void verifyDecodeTextCorrectlyForSSE() throws Exception { + final CountDownLatch countDownLatch = new CountDownLatch(1); + RestTemplate restTemplateMock = mock(RestTemplate.class); + // Intended to set a very small chunk size to test the edge case + DownloadWfsDataService service = new DownloadWfsDataService( + wmsServer, wfsServer, restTemplateMock, entity, 10); + + String original = "id,name,age,city\n1,Alice,30,Sydney\n2,Bob,25,Melbourne\n3,„Café“,42,Perth\n"; + byte[] originalBytes = original.getBytes(StandardCharsets.UTF_8); + + // Mock WFS response + doAnswer(inv -> { + ResponseExtractor extractor = inv.getArgument(3); + ClientHttpResponse resp = mock(ClientHttpResponse.class); + when(resp.getBody()).thenReturn(new ByteArrayInputStream(originalBytes)); + extractor.extractData(resp); + return null; + }).when(restTemplateMock).execute(anyString(), eq(HttpMethod.GET), any(), any()); + + SseEmitter emitter = spy(new SseEmitter(Long.MAX_VALUE)); + List base64Chunks = new CopyOnWriteArrayList<>(); + + doAnswer(answer -> { + SseEmitter.SseEventBuilder builder = answer.getArgument(0); + var d = builder.build(); + + d.forEach(s -> { + if(s.getData() instanceof Map) { + @SuppressWarnings("unchecked") + Map data = (Map)s.getData(); + + if(data.containsKey("data")) { + base64Chunks.add((String) data.get("data")); + } + if (data.containsKey("filename")) { + // All item proceeded, we can continue the verification + countDownLatch.countDown(); + } + } + }); + return null; + }).when(emitter).send(any(SseEmitter.SseEventBuilder.class)); + + + service.executeWfsRequestWithSse( + "http://mock/wfs?...", "uuid-123", "layer:test", "csv", + emitter, new AtomicBoolean()); + + // Wait for processing (use Awaitility in real tests) + countDownLatch.await(); + + // Reconstruct like browser (atob + utf-8 decode) + ByteArrayOutputStream reconstructed = new ByteArrayOutputStream(); + for (String b64 : base64Chunks) { + reconstructed.write(Base64.getDecoder().decode(b64)); + } + + String result = reconstructed.toString(StandardCharsets.UTF_8); + + assertEquals(original, result); + } +} From 0659533dc728b7bc3f71522408ac92d5af6564f8 Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 17:10:28 +1100 Subject: [PATCH 06/12] Add test case to verify the SSE chunk split for text --- .../ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index 813ece5f..0ad6ac03 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -71,6 +71,7 @@ void verifyDecodeTextCorrectlyForSSE() throws Exception { } if (data.containsKey("filename")) { // All item proceeded, we can continue the verification + assertEquals("layer:test_uuid-123.csv", data.get("filename")); countDownLatch.countDown(); } } @@ -80,7 +81,7 @@ void verifyDecodeTextCorrectlyForSSE() throws Exception { service.executeWfsRequestWithSse( - "http://mock/wfs?...", "uuid-123", "layer:test", "csv", + "http://mock/wfs?...", "uuid-123", "layer:test", "text/csv", emitter, new AtomicBoolean()); // Wait for processing (use Awaitility in real tests) From 2ef28907ea727ef4fcc89f38d30c9c1513128b43 Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 17:14:19 +1100 Subject: [PATCH 07/12] Add verification to binary file --- .../wfs/DownloadWfsDataServiceTest.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index 0ad6ac03..fc1b75c3 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -20,6 +20,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; @@ -97,4 +98,98 @@ void verifyDecodeTextCorrectlyForSSE() throws Exception { assertEquals(original, result); } + + @Test + void verifyDecodeBinaryCorrectlyForSSE() throws Exception { + final CountDownLatch countDownLatch = new CountDownLatch(1); + RestTemplate restTemplateMock = mock(RestTemplate.class); + // Intended to set a very small chunk size to test the edge case + DownloadWfsDataService service = new DownloadWfsDataService( + wmsServer, wfsServer, restTemplateMock, entity, 10); + + byte[] originalBytes = new byte[] { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + (byte)0xFF, (byte)0xFE, (byte)0xFD, (byte)0xFC, (byte)0xFB, (byte)0xFA, (byte)0xF9, (byte)0xF8, + (byte)0x80, (byte)0x81, (byte)0x82, (byte)0x83, 0x7F, 0x7E, 0x7D, 0x7C, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, + (byte)0xA0, (byte)0xA1, (byte)0xA2, (byte)0xA3, (byte)0xA4, (byte)0xA5, (byte)0xA6, (byte)0xA7, + (byte)0xB0, (byte)0xB1, (byte)0xB2, (byte)0xB3, (byte)0xB4, (byte)0xB5, (byte)0xB6, (byte)0xB7, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + (byte)0xC0, (byte)0xC1, (byte)0xC2, (byte)0xC3, (byte)0xC4, (byte)0xC5, (byte)0xC6, (byte)0xC7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + (byte)0xFF, (byte)0xFE, (byte)0xFD, (byte)0xFC, (byte)0xFB, (byte)0xFA, (byte)0xF9, (byte)0xF8, + (byte)0x80, (byte)0x81, (byte)0x82, (byte)0x83, 0x7F, 0x7E, 0x7D, 0x7C, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, + (byte)0xA0, (byte)0xA1, (byte)0xA2, (byte)0xA3, (byte)0xA4, (byte)0xA5, (byte)0xA6, (byte)0xA7, + (byte)0xB0, (byte)0xB1, (byte)0xB2, (byte)0xB3, (byte)0xB4, (byte)0xB5, (byte)0xB6, (byte)0xB7, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + (byte)0xC0, (byte)0xC1, (byte)0xC2, (byte)0xC3, (byte)0xC4, (byte)0xC5, (byte)0xC6, (byte)0xC7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7 + }; + + // Mock WFS response + doAnswer(inv -> { + ResponseExtractor extractor = inv.getArgument(3); + ClientHttpResponse resp = mock(ClientHttpResponse.class); + when(resp.getBody()).thenReturn(new ByteArrayInputStream(originalBytes)); + extractor.extractData(resp); + return null; + }).when(restTemplateMock).execute(anyString(), eq(HttpMethod.GET), any(), any()); + + SseEmitter emitter = spy(new SseEmitter(Long.MAX_VALUE)); + List base64Chunks = new CopyOnWriteArrayList<>(); + + doAnswer(answer -> { + SseEmitter.SseEventBuilder builder = answer.getArgument(0); + var d = builder.build(); + + d.forEach(s -> { + if(s.getData() instanceof Map) { + @SuppressWarnings("unchecked") + Map data = (Map)s.getData(); + + if(data.containsKey("data")) { + base64Chunks.add((String) data.get("data")); + } + if (data.containsKey("filename")) { + // All item proceeded, we can continue the verification + assertEquals("layer:test_uuid-123.shp", data.get("filename")); + countDownLatch.countDown(); + } + } + }); + return null; + }).when(emitter).send(any(SseEmitter.SseEventBuilder.class)); + + + service.executeWfsRequestWithSse( + "http://mock/wfs?...", "uuid-123", "layer:test", "shape-zip", + emitter, new AtomicBoolean()); + + // Wait for processing (use Awaitility in real tests) + countDownLatch.await(); + + // Reconstruct like browser (atob + utf-8 decode) + ByteArrayOutputStream reconstructed = new ByteArrayOutputStream(); + for (String b64 : base64Chunks) { + reconstructed.write(Base64.getDecoder().decode(b64)); + } + + assertArrayEquals(originalBytes, reconstructed.toByteArray()); + } } From 23c8f0e83cfa64757b7d07a6dbfc5e995c8ca366 Mon Sep 17 00:00:00 2001 From: rng Date: Mon, 23 Feb 2026 17:15:51 +1100 Subject: [PATCH 08/12] Add comments --- .../server/service/wfs/DownloadWfsDataServiceTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index fc1b75c3..e7d6cfc5 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -34,7 +34,10 @@ public class DownloadWfsDataServiceTest { @Mock protected HttpEntity entity; - + /** + * Test a text file from source is break down into chunk correct and reconstruct correctly + * @throws Exception - Not expected + */ @Test void verifyDecodeTextCorrectlyForSSE() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); @@ -98,7 +101,10 @@ void verifyDecodeTextCorrectlyForSSE() throws Exception { assertEquals(original, result); } - + /** + * Test a binary file from source is break down into chunk correct and reconstruct correctly + * @throws Exception - Not expected + */ @Test void verifyDecodeBinaryCorrectlyForSSE() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); From 5ba67f78062f86a941e766d926486e9e988213fb Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 24 Feb 2026 09:22:33 +1100 Subject: [PATCH 09/12] Minor update to use Enum --- .../wfs/DownloadWfsDataServiceTest.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index e7d6cfc5..17b654d0 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.service.wfs; +import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import au.org.aodn.ogcapi.server.core.service.wfs.DownloadWfsDataService; import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; @@ -85,7 +86,10 @@ void verifyDecodeTextCorrectlyForSSE() throws Exception { service.executeWfsRequestWithSse( - "http://mock/wfs?...", "uuid-123", "layer:test", "text/csv", + "http://mock/wfs?...", + "uuid-123", + "layer:test", + FeatureRequest.GeoServerOutputFormat.CSV.getValue(), emitter, new AtomicBoolean()); // Wait for processing (use Awaitility in real tests) @@ -113,6 +117,7 @@ void verifyDecodeBinaryCorrectlyForSSE() throws Exception { DownloadWfsDataService service = new DownloadWfsDataService( wmsServer, wfsServer, restTemplateMock, entity, 10); + // Just some big enough random binary to trigger chunk split byte[] originalBytes = new byte[] { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, @@ -145,7 +150,12 @@ void verifyDecodeBinaryCorrectlyForSSE() throws Exception { 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, - (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7 + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, }; // Mock WFS response @@ -184,7 +194,10 @@ void verifyDecodeBinaryCorrectlyForSSE() throws Exception { service.executeWfsRequestWithSse( - "http://mock/wfs?...", "uuid-123", "layer:test", "shape-zip", + "http://mock/wfs?...", + "uuid-123", + "layer:test", + FeatureRequest.GeoServerOutputFormat.SHAPE_ZIP.getValue(), emitter, new AtomicBoolean()); // Wait for processing (use Awaitility in real tests) From 0ebd424e3050c25186512fb0c6032f3aae6672e9 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 24 Feb 2026 10:51:51 +1100 Subject: [PATCH 10/12] Fix all test --- .../server/core/model/ogc/FeatureRequest.java | 2 +- .../service/wfs/DownloadWfsDataService.java | 31 ++++++---- .../wfs/DownloadWfsDataServiceTest.java | 60 ++++++++++++++++--- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index 63dac9ca..227d0d0b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -34,7 +34,7 @@ public enum GeoServerOutputFormat { GML2("GML2", "application/gml+xml", "gml"), GML3("GML3", "application/gml+xml", "gml"), GML32("gml32", "application/gml+xml", "gml"), - SHAPE_ZIP("shape-zip", "application/zip", "shp"), // also accepted as "SHAPE-ZIP" + SHAPE_ZIP("shape-zip", "application/zip", "zip"), // also accepted as "SHAPE-ZIP" CSV("text/csv", "text/csv", "csv"), JSON(MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_JSON_VALUE, "json"), // also "json" for backward compatibility GEOJSON("application/geo+json", "application/geo+json", "geojson"), diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java index 68929752..03e914b1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java @@ -43,11 +43,10 @@ public DownloadWfsDataService( this.pretendUserEntity = pretendUserEntity; this.chunkSize = chunkSize; } - /** * Build CQL filter for temporal and spatial constraints */ - private String buildCqlFilter(String startDate, String endDate, Object multiPolygon, WfsFields wfsFieldModel) { + protected String buildCqlFilter(String startDate, String endDate, Object multiPolygon, WfsFields wfsFieldModel) { StringBuilder cqlFilter = new StringBuilder(); if (wfsFieldModel == null || wfsFieldModel.getFields() == null) { @@ -56,23 +55,23 @@ private String buildCqlFilter(String startDate, String endDate, Object multiPoly List fields = wfsFieldModel.getFields(); - // Find temporal field - Optional temporalField = fields.stream() + // Possible to have multiple days, better to consider all + List temporalField = fields.stream() .filter(field -> "dateTime".equals(field.getType()) || "date".equals(field.getType())) - .findFirst(); + .toList(); // Add temporal filter only if both dates are specified - if (temporalField.isPresent() && startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { - String fieldName = temporalField.get().getName(); - cqlFilter.append(fieldName) - .append(" DURING ") - .append(startDate).append("T00:00:00Z/") - .append(endDate).append("T23:59:59Z"); + if (!temporalField.isEmpty() && startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { + List cqls = new ArrayList<>(); + temporalField.forEach(temp -> + cqls.add(String.format("(%s DURING %sT00:00:00Z/%sT23:59:59Z)", temp.getName(), startDate, endDate)) + ); + cqlFilter.append("(").append(String.join(" OR ", cqls)).append(")"); } // Find geometry field Optional geometryField = fields.stream() - .filter(field -> "geometrypropertytype".equals(field.getType())) + .filter(field -> "geometrypropertytype".equalsIgnoreCase(field.getType())) .findFirst(); // Add spatial filter @@ -133,7 +132,13 @@ public String prepareWfsRequestUrl( if (featureServerUrl.isPresent()) { wfsServerUrl = featureServerUrl.get(); wfsTypeName = layerName; - wfsFieldModel = wfsServer.getDownloadableFields(uuid, WfsServer.WfsFeatureRequest.builder().layerName(wfsTypeName).server(wfsServerUrl).build()); + wfsFieldModel = wfsServer.getDownloadableFields( + uuid, + WfsServer.WfsFeatureRequest.builder() + .layerName(wfsTypeName) + .server(wfsServerUrl) + .build() + ); log.info("WFSFieldModel by wfs typename: {}", wfsFieldModel); } else { throw new IllegalArgumentException("No WFS server URL found for the given UUID and layer name"); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java index 5d61d15c..ef8024aa 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java @@ -7,6 +7,8 @@ import au.org.aodn.ogcapi.server.core.service.Search; import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -21,6 +23,7 @@ import org.springframework.web.client.RestTemplate; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -254,13 +257,10 @@ public void testPrepareWfsRequestUrl_WithMMYYYYFormat() { // Verify URL contains temporal filter with converted dates assertNotNull(result); - assertTrue(result.contains("typeName=" + layerName)); - assertTrue(result.contains("cql_filter"), "URL should contain cql_filter"); - assertTrue(result.contains("DURING"), "CQL filter should contain DURING operator"); - // Start date should be first day of January 2023 - assertTrue(result.contains("2023-01-01T00:00:00Z"), "Start date should be converted to first day of month"); - // End date should be last day of December 2023 - assertTrue(result.contains("2023-12-31T23:59:59Z"), "End date should be converted to last day of month"); + assertEquals( + "https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2023-01-01T00:00:00Z/2023-12-31T23:59:59Z))", + result + ); } @Test @@ -310,11 +310,53 @@ public void verifyRequestUrlGenerateCorrect() { uuid, startDate, endDate, null, null, layerName, null ); - assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z", result, "Correct url 1"); + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))", result, "Correct url 1"); result = downloadWfsDataService.prepareWfsRequestUrl( uuid, startDate, endDate, null, null, layerName, "shape-zip" ); - assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=shape-zip&cql_filter=timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z", result, "Correct url 1"); + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=shape-zip&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))", result, "Correct url 1"); + } + /** + * Make sure the url generated contains the correct polygon + * @throws JsonProcessingException - Not expected + */ + @Test + public void verifyRequestUrlGenerateCorrectWithPolygon() throws JsonProcessingException { + // Setup + String uuid = "test-uuid"; + String layerName = "test:layer"; + String startDate = "01-2024"; // MM-YYYY format + String endDate = "12-2024"; // MM-YYYY format + WfsFields wfsFieldModel = createTestWFSFieldModel(); + + DescribeLayerResponse describeLayerResponse = mock(DescribeLayerResponse.class); + DescribeLayerResponse.LayerDescription layerDescription = mock(DescribeLayerResponse.LayerDescription.class); + DescribeLayerResponse.Query query = mock(DescribeLayerResponse.Query.class); + + when(describeLayerResponse.getLayerDescription()).thenReturn(layerDescription); + when(layerDescription.getWfs()).thenReturn("https://test.com/geoserver/wfs"); + when(layerDescription.getQuery()).thenReturn(query); + when(query.getTypeName()).thenReturn(layerName); + + doReturn(describeLayerResponse) + .when(wmsServer).describeLayer(eq(uuid), any(FeatureRequest.class)); + doReturn(wfsFieldModel) + .when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class)); + + Object multiPolygon = new ObjectMapper().readValue( + "{ \"type\": \"MultiPolygon\", \"coordinates\": [[[[112.01192842942288, -22.393450547704845], [129.68986083498982, -22.393450547704845], [129.68986083498982, -12.647778557898718], [112.01192842942288, -12.647778557898718], [112.01192842942288, -22.393450547704845]]], [[[128.29423459244452, 3.5283082597303377], [143.95626242544682, 3.5283082597303377], [143.95626242544682, 13.182067934641196], [128.29423459244452, 13.182067934641196], [128.29423459244452, 3.5283082597303377]]]] }", + HashMap.class + ); + + // Test with MM-YYYY format dates + String result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, multiPolygon, null, layerName, null + ); + + assertEquals( + "https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z)) AND INTERSECTS(geom,MULTIPOLYGON (((112.01192842942288 -22.393450547704845, 129.68986083498982 -22.393450547704845, 129.68986083498982 -12.647778557898718, 112.01192842942288 -12.647778557898718, 112.01192842942288 -22.393450547704845)), ((128.29423459244452 3.5283082597303377, 143.95626242544682 3.5283082597303377, 143.95626242544682 13.182067934641196, 128.29423459244452 13.182067934641196, 128.29423459244452 3.5283082597303377))))", + result, + "Correct url 1"); } } From 63b4048fd1ae6ff38342efcb113c5b73da115cc7 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 24 Feb 2026 10:56:39 +1100 Subject: [PATCH 11/12] Fix typo --- .../ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index 17b654d0..c0e76caf 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -155,7 +155,7 @@ void verifyDecodeBinaryCorrectlyForSSE() throws Exception { 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, - (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5 }; // Mock WFS response From 6db9b8429babe370dc7a987316d9dd5eed446ec1 Mon Sep 17 00:00:00 2001 From: rng Date: Tue, 24 Feb 2026 10:59:57 +1100 Subject: [PATCH 12/12] Fix typo --- .../ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java index c0e76caf..c10bc669 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -184,7 +184,7 @@ void verifyDecodeBinaryCorrectlyForSSE() throws Exception { } if (data.containsKey("filename")) { // All item proceeded, we can continue the verification - assertEquals("layer:test_uuid-123.shp", data.get("filename")); + assertEquals("layer:test_uuid-123.zip", data.get("filename")); countDownLatch.countDown(); } }