diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java index ea6d7878..72a8e25d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java @@ -20,8 +20,8 @@ public enum Parameter { COLLECTION_TITLE("collection_title"), FULL_METADATA_LINK("full_metadata_link"), SUGGESTED_CITATION("suggested_citation"), - KEY("key") - ; + KEY("key"), + OUTPUT_FORMAT("output_format"); private final String value; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java index a419be9e..227d0d0b 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java @@ -3,6 +3,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.*; import lombok.experimental.SuperBuilder; +import org.springframework.http.MediaType; import java.io.Serializable; import java.math.BigDecimal; @@ -28,6 +29,43 @@ public static PropertyName fromString(String input) { } } + @Getter + public enum GeoServerOutputFormat { + GML2("GML2", "application/gml+xml", "gml"), + GML3("GML3", "application/gml+xml", "gml"), + GML32("gml32", "application/gml+xml", "gml"), + SHAPE_ZIP("shape-zip", "application/zip", "zip"), // also accepted as "SHAPE-ZIP" + CSV("text/csv", "text/csv", "csv"), + JSON(MediaType.APPLICATION_JSON_VALUE, MediaType.APPLICATION_JSON_VALUE, "json"), // also "json" for backward compatibility + GEOJSON("application/geo+json", "application/geo+json", "geojson"), + KML("KML", "application/vnd.google-earth.kml+xml", "kml"), + UNKNOWN("unknown", "application/octet-stream", "bin"); + + private final String value; + private final String mediaType; + private final String fileExtension; + + GeoServerOutputFormat(String value, String mediaType, String fileExtension) { + this.value = value; + this.mediaType = mediaType; + this.fileExtension = fileExtension; + } + + public static GeoServerOutputFormat fromString(String s) { + for(GeoServerOutputFormat f : GeoServerOutputFormat.values()) { + if(f.value.equalsIgnoreCase(s)) { + return f; + } + } + return UNKNOWN; // or throw custom exception + } + + @Override + public String toString() { + return value; + } + } + @Schema(description = "Property to be return") private List properties; @@ -52,6 +90,9 @@ public static PropertyName fromString(String input) { @Schema(description = "Y") private BigDecimal y; + @Schema(description = "Data output format") + private GeoServerOutputFormat outputFormat; + @Schema(description = "Wave buoy name") private String waveBuoy; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java index e3e901e8..03e914b1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.server.core.util.GeometryUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Service; @@ -27,23 +28,25 @@ public class DownloadWfsDataService { private final WfsServer wfsServer; private final RestTemplate restTemplate; private final HttpEntity pretendUserEntity; + private final int chunkSize; public DownloadWfsDataService( WmsServer wmsServer, WfsServer wfsServer, RestTemplate restTemplate, - @Qualifier("pretendUserEntity") HttpEntity pretendUserEntity + @Qualifier("pretendUserEntity") HttpEntity pretendUserEntity, + @Value("${app.sse.chunkSize:16384}") int chunkSize ) { this.wmsServer = wmsServer; this.wfsServer = wfsServer; this.restTemplate = restTemplate; this.pretendUserEntity = pretendUserEntity; + this.chunkSize = chunkSize; } - /** * Build CQL filter for temporal and spatial constraints */ - private String buildCqlFilter(String startDate, String endDate, Object multiPolygon, WfsFields wfsFieldModel) { + protected String buildCqlFilter(String startDate, String endDate, Object multiPolygon, WfsFields wfsFieldModel) { StringBuilder cqlFilter = new StringBuilder(); if (wfsFieldModel == null || wfsFieldModel.getFields() == null) { @@ -52,23 +55,23 @@ private String buildCqlFilter(String startDate, String endDate, Object multiPoly List fields = wfsFieldModel.getFields(); - // Find temporal field - Optional temporalField = fields.stream() + // Possible to have multiple days, better to consider all + List temporalField = fields.stream() .filter(field -> "dateTime".equals(field.getType()) || "date".equals(field.getType())) - .findFirst(); + .toList(); // Add temporal filter only if both dates are specified - if (temporalField.isPresent() && startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { - String fieldName = temporalField.get().getName(); - cqlFilter.append(fieldName) - .append(" DURING ") - .append(startDate).append("T00:00:00Z/") - .append(endDate).append("T23:59:59Z"); + if (!temporalField.isEmpty() && startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) { + List cqls = new ArrayList<>(); + temporalField.forEach(temp -> + cqls.add(String.format("(%s DURING %sT00:00:00Z/%sT23:59:59Z)", temp.getName(), startDate, endDate)) + ); + cqlFilter.append("(").append(String.join(" OR ", cqls)).append(")"); } // Find geometry field Optional geometryField = fields.stream() - .filter(field -> "geometrypropertytype".equals(field.getType())) + .filter(field -> "geometrypropertytype".equalsIgnoreCase(field.getType())) .findFirst(); // Add spatial filter @@ -101,7 +104,8 @@ public String prepareWfsRequestUrl( String endDate, Object multiPolygon, List fields, - String layerName) { + String layerName, + String outputFormat) { DescribeLayerResponse describeLayerResponse = wmsServer.describeLayer(uuid, FeatureRequest.builder().layerName(layerName).build()); @@ -114,7 +118,13 @@ public String prepareWfsRequestUrl( wfsServerUrl = describeLayerResponse.getLayerDescription().getWfs(); wfsTypeName = describeLayerResponse.getLayerDescription().getQuery().getTypeName(); - wfsFieldModel = wfsServer.getDownloadableFields(uuid, WfsServer.WfsFeatureRequest.builder().layerName(wfsTypeName).server(wfsServerUrl).build()); + wfsFieldModel = wfsServer.getDownloadableFields( + uuid, + WfsServer.WfsFeatureRequest.builder() + .layerName(wfsTypeName) + .server(wfsServerUrl) + .build() + ); log.info("WFSFieldModel by describeLayer: {}", wfsFieldModel); } else { Optional featureServerUrl = wfsServer.getFeatureServerUrlByTitle(uuid, layerName); @@ -122,7 +132,13 @@ public String prepareWfsRequestUrl( if (featureServerUrl.isPresent()) { wfsServerUrl = featureServerUrl.get(); wfsTypeName = layerName; - wfsFieldModel = wfsServer.getDownloadableFields(uuid, WfsServer.WfsFeatureRequest.builder().layerName(wfsTypeName).server(wfsServerUrl).build()); + wfsFieldModel = wfsServer.getDownloadableFields( + uuid, + WfsServer.WfsFeatureRequest.builder() + .layerName(wfsTypeName) + .server(wfsServerUrl) + .build() + ); log.info("WFSFieldModel by wfs typename: {}", wfsFieldModel); } else { throw new IllegalArgumentException("No WFS server URL found for the given UUID and layer name"); @@ -137,7 +153,12 @@ public String prepareWfsRequestUrl( String cqlFilter = buildCqlFilter(validStartDate, validEndDate, multiPolygon, wfsFieldModel); // Build final WFS request URL - String wfsRequestUrl = wfsServer.createWfsRequestUrl(wfsServerUrl, wfsTypeName, fields, cqlFilter, null); + String wfsRequestUrl = wfsServer.createWfsRequestUrl( + wfsServerUrl, + wfsTypeName, + fields, + cqlFilter, + outputFormat); log.info("Prepared WFS request URL: {}", wfsRequestUrl); return wfsRequestUrl; @@ -149,6 +170,7 @@ public void executeWfsRequestWithSse( String wfsRequestUrl, String uuid, String layerName, + String outputFormat, SseEmitter emitter, AtomicBoolean wfsServerResponded) { restTemplate.execute( @@ -175,59 +197,35 @@ public void executeWfsRequestWithSse( int bytesRead; long totalBytes = 0; int chunkNumber = 0; - long lastProgressTime = System.currentTimeMillis(); ByteArrayOutputStream chunkBuffer = new ByteArrayOutputStream(); while ((bytesRead = inputStream.read(buffer)) != -1) { chunkBuffer.write(buffer, 0, bytesRead); totalBytes += bytesRead; - long currentTime = System.currentTimeMillis(); - - // Send chunk when buffer is full OR every 2 seconds - if (chunkBuffer.size() >= 16384 || - (currentTime - lastProgressTime >= 2000)) { - + // Send when buffer >= 16KB **and** size divisible by 3 (for clean Base64) + while (chunkBuffer.size() >= chunkSize && chunkBuffer.size() % 3 == 0) { byte[] chunkBytes = chunkBuffer.toByteArray(); - - // Ensure Base64 alignment - // Base64 works in 3-byte groups, so chunk size should be divisible by 3 - int alignedSize = (chunkBytes.length / 3) * 3; - - if (alignedSize > 0) { - // Send the aligned portion - byte[] alignedChunk = Arrays.copyOf(chunkBytes, alignedSize); - String encodedData = Base64.getEncoder().encodeToString(alignedChunk); - - emitter.send(SseEmitter.event() - .name("file-chunk") - .data(Map.of( - "chunkNumber", ++chunkNumber, - "data", encodedData, - "chunkSize", alignedChunk.length, - "totalBytes", totalBytes, - "timestamp", currentTime - )) - .id(String.valueOf(chunkNumber))); - - // Keep the remaining bytes for next chunk - if (alignedSize < chunkBytes.length) { - byte[] remainder = Arrays.copyOfRange(chunkBytes, alignedSize, chunkBytes.length); - chunkBuffer.reset(); - chunkBuffer.write(remainder); - } else { - chunkBuffer.reset(); - } - - lastProgressTime = currentTime; - } + String encodedData = Base64.getEncoder().encodeToString(chunkBytes); + + emitter.send(SseEmitter.event() + .name("file-chunk") + .data(Map.of( + "chunkNumber", ++chunkNumber, + "data", encodedData, + "chunkSize", chunkBytes.length, + "totalBytes", totalBytes, + "timestamp", System.currentTimeMillis() + )) + .id(String.valueOf(chunkNumber))); + + chunkBuffer.reset(); } } - // Send final chunk if any remains + // Final chunk (may not be %3==0, but client Base64 decoder usually handles it) if (chunkBuffer.size() > 0) { - String encodedData = Base64.getEncoder() - .encodeToString(chunkBuffer.toByteArray()); + String encodedData = Base64.getEncoder().encodeToString(chunkBuffer.toByteArray()); emitter.send(SseEmitter.event() .name("file-chunk") .data(Map.of( @@ -246,7 +244,8 @@ public void executeWfsRequestWithSse( "totalBytes", totalBytes, "totalChunks", chunkNumber, "message", "WFS data download completed successfully", - "filename", layerName + "_" + uuid + ".csv" + "media-type", FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getMediaType(), + "filename", String.format("%s_%s.%s", layerName, uuid, FeatureRequest.GeoServerOutputFormat.fromString(outputFormat).getFileExtension()) ))); // Close SSE connection with completion diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java index 03a3f7c3..ed03b1a1 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/features/RestServices.java @@ -144,25 +144,30 @@ public ResponseEntity getWfsFieldValue(String collectionId, FeatureRequest re String result = wfsServer.getFieldValues(collectionId, wfsFeatureRequest, new ParameterizedTypeReference<>() {}); - FeatureJSON json = new FeatureJSON(); - try { - @SuppressWarnings("unchecked") - FeatureCollection collection = json.readFeatureCollection(new StringReader(result)); - try(FeatureIterator i = collection.features()) { - Map> results = new HashMap<>(); - while(i.hasNext()) { - SimpleFeature s = i.next(); - s.getProperties() - .forEach(property -> { - results.computeIfAbsent(property.getName().toString(), k -> new ArrayList<>()); - results.get(property.getName().toString()).add(s.getAttribute(property.getName())); - }); + if(result != null) { + FeatureJSON json = new FeatureJSON(); + try { + @SuppressWarnings("unchecked") + FeatureCollection collection = json.readFeatureCollection(new StringReader(result)); + try (FeatureIterator i = collection.features()) { + Map> results = new HashMap<>(); + while (i.hasNext()) { + SimpleFeature s = i.next(); + s.getProperties() + .forEach(property -> { + results.computeIfAbsent(property.getName().toString(), k -> new ArrayList<>()); + results.get(property.getName().toString()).add(s.getAttribute(property.getName())); + }); + } + return ResponseEntity.ok().body(results); } - return ResponseEntity.ok().body(results); + } catch (IOException e) { + return ResponseEntity.badRequest().body(e.getMessage()); } } - catch (IOException e) { - return ResponseEntity.badRequest().body(e.getMessage()); + else { + // Field not found + return ResponseEntity.notFound().build(); } } /** diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java index ff9fe909..36ed6d73 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java @@ -9,6 +9,7 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.DatasetDownloadEnums; import au.org.aodn.ogcapi.server.core.model.enumeration.InlineResponseKeyEnum; import au.org.aodn.ogcapi.server.core.model.enumeration.ProcessIdEnum; +import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.media.Schema; @@ -125,14 +126,19 @@ public SseEmitter downloadWfsSse( String startDate = body.getInputs().get(DatasetDownloadEnums.Parameter.START_DATE.getValue()).toString(); String endDate = body.getInputs().get(DatasetDownloadEnums.Parameter.END_DATE.getValue()).toString(); String layerName = body.getInputs().get(DatasetDownloadEnums.Parameter.LAYER_NAME.getValue()).toString(); + String outputFormat = body.getInputs().get(DatasetDownloadEnums.Parameter.OUTPUT_FORMAT.getValue()).toString(); var multiPolygon = body.getInputs().get(DatasetDownloadEnums.Parameter.MULTI_POLYGON.getValue()); List fields = body.getInputs().get(DatasetDownloadEnums.Parameter.FIELDS.getValue()) instanceof List list ? list.stream().map(String::valueOf).toList() : null; + if(FeatureRequest.GeoServerOutputFormat.fromString(outputFormat) == FeatureRequest.GeoServerOutputFormat.UNKNOWN) { + throw new IllegalArgumentException(String.format("Output format %s not supported", outputFormat)); + } + return restServices.downloadWfsDataWithSse( - uuid, startDate, endDate, multiPolygon, fields, layerName, emitter + uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat, emitter ); } catch (Exception e) { diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java index fa338eb9..cbe1e3aa 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java @@ -182,6 +182,7 @@ public SseEmitter downloadWfsDataWithSse( Object multiPolygon, List fields, String layerName, + String outputFormat, SseEmitter emitter) { // Set up references for resources that need to be cleaned up @@ -219,9 +220,7 @@ public SseEmitter downloadWfsDataWithSse( cleanupWfsResources.run(); }); - emitter.onError(throwable -> { - WfsErrorHandler.handleError((Exception) throwable, uuid, emitter, cleanupWfsResources); - }); + emitter.onError(throwable -> WfsErrorHandler.handleError((Exception) throwable, uuid, emitter, cleanupWfsResources)); // Validate parameters if (uuid == null || layerName == null || layerName.trim().isEmpty()) { @@ -270,7 +269,8 @@ public SseEmitter downloadWfsDataWithSse( // STEP 3: Do preparation work: Collection lookup from Elasticsearch, WFS validation, Field retrieval, URL building String wfsRequestUrl = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, multiPolygon, fields, layerName); + uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat + ); emitter.send(SseEmitter.event() .name("wfs-request-ready") @@ -284,6 +284,7 @@ public SseEmitter downloadWfsDataWithSse( wfsRequestUrl, uuid, layerName, + outputFormat, emitter, wfsServerResponded ); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java index 9e3d1adc..ef8024aa 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataServiceTest.java @@ -7,6 +7,8 @@ import au.org.aodn.ogcapi.server.core.service.Search; import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -21,6 +23,7 @@ import org.springframework.web.client.RestTemplate; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -60,7 +63,7 @@ public void setUp() { wmsServer = Mockito.spy(new WmsServer(search, wfsServer, pretendUserEntity)); downloadWfsDataService = new DownloadWfsDataService( - wmsServer, wfsServer, restTemplate, pretendUserEntity + wmsServer, wfsServer, restTemplate, pretendUserEntity, 16384 ); } @@ -113,7 +116,7 @@ public void testPrepareWfsRequestUrl_WithNullDates() { // Test with null dates (non-specified dates from frontend) String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, null, null, null, null, layerName + uuid, null, null, null, null, layerName, null ); // Verify URL doesn't contain temporal filter when dates are null @@ -145,7 +148,7 @@ public void testPrepareWfsRequestUrl_WithEmptyDates() { // Test with empty string dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, "", "", null, null, layerName + uuid, "", "", null, null, layerName, null ); // Verify URL doesn't contain temporal filter when dates are empty @@ -179,7 +182,7 @@ public void testPrepareWfsRequestUrl_WithValidDates() { // Test with valid dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, null, null, layerName + uuid, startDate, endDate, null, null, layerName, null ); // Verify URL contains temporal filter when valid dates are provided @@ -215,7 +218,7 @@ public void testPrepareWfsRequestUrl_WithOnlyStartDate() { // Test with only start date (end date is null) String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, null, null, null, layerName + uuid, startDate, null, null, null, layerName, null ); // Verify URL doesn't contain temporal filter when only one date is provided @@ -249,18 +252,15 @@ public void testPrepareWfsRequestUrl_WithMMYYYYFormat() { // Test with MM-YYYY format dates String result = downloadWfsDataService.prepareWfsRequestUrl( - uuid, startDate, endDate, null, null, layerName + uuid, startDate, endDate, null, null, layerName, null ); // Verify URL contains temporal filter with converted dates assertNotNull(result); - assertTrue(result.contains("typeName=" + layerName)); - assertTrue(result.contains("cql_filter"), "URL should contain cql_filter"); - assertTrue(result.contains("DURING"), "CQL filter should contain DURING operator"); - // Start date should be first day of January 2023 - assertTrue(result.contains("2023-01-01T00:00:00Z"), "Start date should be converted to first day of month"); - // End date should be last day of December 2023 - assertTrue(result.contains("2023-12-31T23:59:59Z"), "End date should be converted to last day of month"); + assertEquals( + "https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2023-01-01T00:00:00Z/2023-12-31T23:59:59Z))", + result + ); } @Test @@ -276,9 +276,87 @@ public void testPrepareWfsRequestUrl_NoWfsServerUrl() { // Test with no WFS server URL available Exception exception = assertThrows(IllegalArgumentException.class, () -> downloadWfsDataService.prepareWfsRequestUrl( - uuid, null, null, null, null, layerName + uuid, null, null, null, null, layerName, null )); assertTrue(exception.getMessage().contains("No WFS server URL found")); } + + @Test + public void verifyRequestUrlGenerateCorrect() { + // Setup + String uuid = "test-uuid"; + String layerName = "test:layer"; + String startDate = "01-2024"; // MM-YYYY format + String endDate = "12-2024"; // MM-YYYY format + WfsFields wfsFieldModel = createTestWFSFieldModel(); + + DescribeLayerResponse describeLayerResponse = mock(DescribeLayerResponse.class); + DescribeLayerResponse.LayerDescription layerDescription = mock(DescribeLayerResponse.LayerDescription.class); + DescribeLayerResponse.Query query = mock(DescribeLayerResponse.Query.class); + + when(describeLayerResponse.getLayerDescription()).thenReturn(layerDescription); + when(layerDescription.getWfs()).thenReturn("https://test.com/geoserver/wfs"); + when(layerDescription.getQuery()).thenReturn(query); + when(query.getTypeName()).thenReturn(layerName); + + doReturn(describeLayerResponse) + .when(wmsServer).describeLayer(eq(uuid), any(FeatureRequest.class)); + doReturn(wfsFieldModel) + .when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class)); + + // Test with MM-YYYY format dates + String result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, null, null, layerName, null + ); + + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))", result, "Correct url 1"); + + result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, null, null, layerName, "shape-zip" + ); + assertEquals("https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=shape-zip&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))", result, "Correct url 1"); + } + /** + * Make sure the url generated contains the correct polygon + * @throws JsonProcessingException - Not expected + */ + @Test + public void verifyRequestUrlGenerateCorrectWithPolygon() throws JsonProcessingException { + // Setup + String uuid = "test-uuid"; + String layerName = "test:layer"; + String startDate = "01-2024"; // MM-YYYY format + String endDate = "12-2024"; // MM-YYYY format + WfsFields wfsFieldModel = createTestWFSFieldModel(); + + DescribeLayerResponse describeLayerResponse = mock(DescribeLayerResponse.class); + DescribeLayerResponse.LayerDescription layerDescription = mock(DescribeLayerResponse.LayerDescription.class); + DescribeLayerResponse.Query query = mock(DescribeLayerResponse.Query.class); + + when(describeLayerResponse.getLayerDescription()).thenReturn(layerDescription); + when(layerDescription.getWfs()).thenReturn("https://test.com/geoserver/wfs"); + when(layerDescription.getQuery()).thenReturn(query); + when(query.getTypeName()).thenReturn(layerName); + + doReturn(describeLayerResponse) + .when(wmsServer).describeLayer(eq(uuid), any(FeatureRequest.class)); + doReturn(wfsFieldModel) + .when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class)); + + Object multiPolygon = new ObjectMapper().readValue( + "{ \"type\": \"MultiPolygon\", \"coordinates\": [[[[112.01192842942288, -22.393450547704845], [129.68986083498982, -22.393450547704845], [129.68986083498982, -12.647778557898718], [112.01192842942288, -12.647778557898718], [112.01192842942288, -22.393450547704845]]], [[[128.29423459244452, 3.5283082597303377], [143.95626242544682, 3.5283082597303377], [143.95626242544682, 13.182067934641196], [128.29423459244452, 13.182067934641196], [128.29423459244452, 3.5283082597303377]]]] }", + HashMap.class + ); + + // Test with MM-YYYY format dates + String result = downloadWfsDataService.prepareWfsRequestUrl( + uuid, startDate, endDate, multiPolygon, null, layerName, null + ); + + assertEquals( + "https://test.com/geoserver/wfs?VERSION=1.0.0&typeName=test:layer&SERVICE=WFS&REQUEST=GetFeature&outputFormat=text/csv&cql_filter=((timestamp DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z)) AND INTERSECTS(geom,MULTIPOLYGON (((112.01192842942288 -22.393450547704845, 129.68986083498982 -22.393450547704845, 129.68986083498982 -12.647778557898718, 112.01192842942288 -12.647778557898718, 112.01192842942288 -22.393450547704845)), ((128.29423459244452 3.5283082597303377, 143.95626242544682 3.5283082597303377, 143.95626242544682 13.182067934641196, 128.29423459244452 13.182067934641196, 128.29423459244452 3.5283082597303377))))", + result, + "Correct url 1"); + } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java new file mode 100644 index 00000000..c10bc669 --- /dev/null +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/wfs/DownloadWfsDataServiceTest.java @@ -0,0 +1,214 @@ +package au.org.aodn.ogcapi.server.service.wfs; + +import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest; +import au.org.aodn.ogcapi.server.core.service.wfs.DownloadWfsDataService; +import au.org.aodn.ogcapi.server.core.service.wfs.WfsServer; +import au.org.aodn.ogcapi.server.core.service.wms.WmsServer; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.web.client.ResponseExtractor; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +public class DownloadWfsDataServiceTest { + + @Mock + protected WfsServer wfsServer; + + @Mock + protected WmsServer wmsServer; + + @Mock + protected HttpEntity entity; + /** + * Test a text file from source is break down into chunk correct and reconstruct correctly + * @throws Exception - Not expected + */ + @Test + void verifyDecodeTextCorrectlyForSSE() throws Exception { + final CountDownLatch countDownLatch = new CountDownLatch(1); + RestTemplate restTemplateMock = mock(RestTemplate.class); + // Intended to set a very small chunk size to test the edge case + DownloadWfsDataService service = new DownloadWfsDataService( + wmsServer, wfsServer, restTemplateMock, entity, 10); + + String original = "id,name,age,city\n1,Alice,30,Sydney\n2,Bob,25,Melbourne\n3,„Café“,42,Perth\n"; + byte[] originalBytes = original.getBytes(StandardCharsets.UTF_8); + + // Mock WFS response + doAnswer(inv -> { + ResponseExtractor extractor = inv.getArgument(3); + ClientHttpResponse resp = mock(ClientHttpResponse.class); + when(resp.getBody()).thenReturn(new ByteArrayInputStream(originalBytes)); + extractor.extractData(resp); + return null; + }).when(restTemplateMock).execute(anyString(), eq(HttpMethod.GET), any(), any()); + + SseEmitter emitter = spy(new SseEmitter(Long.MAX_VALUE)); + List base64Chunks = new CopyOnWriteArrayList<>(); + + doAnswer(answer -> { + SseEmitter.SseEventBuilder builder = answer.getArgument(0); + var d = builder.build(); + + d.forEach(s -> { + if(s.getData() instanceof Map) { + @SuppressWarnings("unchecked") + Map data = (Map)s.getData(); + + if(data.containsKey("data")) { + base64Chunks.add((String) data.get("data")); + } + if (data.containsKey("filename")) { + // All item proceeded, we can continue the verification + assertEquals("layer:test_uuid-123.csv", data.get("filename")); + countDownLatch.countDown(); + } + } + }); + return null; + }).when(emitter).send(any(SseEmitter.SseEventBuilder.class)); + + + service.executeWfsRequestWithSse( + "http://mock/wfs?...", + "uuid-123", + "layer:test", + FeatureRequest.GeoServerOutputFormat.CSV.getValue(), + emitter, new AtomicBoolean()); + + // Wait for processing (use Awaitility in real tests) + countDownLatch.await(); + + // Reconstruct like browser (atob + utf-8 decode) + ByteArrayOutputStream reconstructed = new ByteArrayOutputStream(); + for (String b64 : base64Chunks) { + reconstructed.write(Base64.getDecoder().decode(b64)); + } + + String result = reconstructed.toString(StandardCharsets.UTF_8); + + assertEquals(original, result); + } + /** + * Test a binary file from source is break down into chunk correct and reconstruct correctly + * @throws Exception - Not expected + */ + @Test + void verifyDecodeBinaryCorrectlyForSSE() throws Exception { + final CountDownLatch countDownLatch = new CountDownLatch(1); + RestTemplate restTemplateMock = mock(RestTemplate.class); + // Intended to set a very small chunk size to test the edge case + DownloadWfsDataService service = new DownloadWfsDataService( + wmsServer, wfsServer, restTemplateMock, entity, 10); + + // Just some big enough random binary to trigger chunk split + byte[] originalBytes = new byte[] { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + (byte)0xFF, (byte)0xFE, (byte)0xFD, (byte)0xFC, (byte)0xFB, (byte)0xFA, (byte)0xF9, (byte)0xF8, + (byte)0x80, (byte)0x81, (byte)0x82, (byte)0x83, 0x7F, 0x7E, 0x7D, 0x7C, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, + (byte)0xA0, (byte)0xA1, (byte)0xA2, (byte)0xA3, (byte)0xA4, (byte)0xA5, (byte)0xA6, (byte)0xA7, + (byte)0xB0, (byte)0xB1, (byte)0xB2, (byte)0xB3, (byte)0xB4, (byte)0xB5, (byte)0xB6, (byte)0xB7, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + (byte)0xC0, (byte)0xC1, (byte)0xC2, (byte)0xC3, (byte)0xC4, (byte)0xC5, (byte)0xC6, (byte)0xC7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, + (byte)0xFF, (byte)0xFE, (byte)0xFD, (byte)0xFC, (byte)0xFB, (byte)0xFA, (byte)0xF9, (byte)0xF8, + (byte)0x80, (byte)0x81, (byte)0x82, (byte)0x83, 0x7F, 0x7E, 0x7D, 0x7C, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, + (byte)0xA0, (byte)0xA1, (byte)0xA2, (byte)0xA3, (byte)0xA4, (byte)0xA5, (byte)0xA6, (byte)0xA7, + (byte)0xB0, (byte)0xB1, (byte)0xB2, (byte)0xB3, (byte)0xB4, (byte)0xB5, (byte)0xB6, (byte)0xB7, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, + (byte)0xC0, (byte)0xC1, (byte)0xC2, (byte)0xC3, (byte)0xC4, (byte)0xC5, (byte)0xC6, (byte)0xC7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7, + (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, + (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, + (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5 + }; + + // Mock WFS response + doAnswer(inv -> { + ResponseExtractor extractor = inv.getArgument(3); + ClientHttpResponse resp = mock(ClientHttpResponse.class); + when(resp.getBody()).thenReturn(new ByteArrayInputStream(originalBytes)); + extractor.extractData(resp); + return null; + }).when(restTemplateMock).execute(anyString(), eq(HttpMethod.GET), any(), any()); + + SseEmitter emitter = spy(new SseEmitter(Long.MAX_VALUE)); + List base64Chunks = new CopyOnWriteArrayList<>(); + + doAnswer(answer -> { + SseEmitter.SseEventBuilder builder = answer.getArgument(0); + var d = builder.build(); + + d.forEach(s -> { + if(s.getData() instanceof Map) { + @SuppressWarnings("unchecked") + Map data = (Map)s.getData(); + + if(data.containsKey("data")) { + base64Chunks.add((String) data.get("data")); + } + if (data.containsKey("filename")) { + // All item proceeded, we can continue the verification + assertEquals("layer:test_uuid-123.zip", data.get("filename")); + countDownLatch.countDown(); + } + } + }); + return null; + }).when(emitter).send(any(SseEmitter.SseEventBuilder.class)); + + + service.executeWfsRequestWithSse( + "http://mock/wfs?...", + "uuid-123", + "layer:test", + FeatureRequest.GeoServerOutputFormat.SHAPE_ZIP.getValue(), + emitter, new AtomicBoolean()); + + // Wait for processing (use Awaitility in real tests) + countDownLatch.await(); + + // Reconstruct like browser (atob + utf-8 decode) + ByteArrayOutputStream reconstructed = new ByteArrayOutputStream(); + for (String b64 : base64Chunks) { + reconstructed.write(Base64.getDecoder().decode(b64)); + } + + assertArrayEquals(originalBytes, reconstructed.toByteArray()); + } +}