Skip to content

Bugfix/7939 multi wfs download #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
utas-raymondng merged 4 commits into from
Feb 24, 2026
Merged

Bugfix/7939 multi wfs download #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
14640d0
update wfs prepare download url
NekoLyn Feb 24, 2026
1c4fe0f
refactor and update test
NekoLyn Feb 24, 2026
e10c0ca
fix warning
NekoLyn Feb 24, 2026
bfea71c
fix wrong server url that contains unexpected params
NekoLyn Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 12 additions & 29 deletions server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/DownloadWfsDataService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest;
import au.org.aodn.ogcapi.server.core.model.ogc.wfs.WfsField;
import au.org.aodn.ogcapi.server.core.model.ogc.wfs.WfsFields;
import au.org.aodn.ogcapi.server.core.model.ogc.wms.DescribeLayerResponse;
import au.org.aodn.ogcapi.server.core.service.wms.WmsServer;
import au.org.aodn.ogcapi.server.core.util.DatetimeUtils;
import au.org.aodn.ogcapi.server.core.util.GeometryUtils;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -24,25 +22,23 @@
@Slf4j
@Service
public class DownloadWfsDataService {
private final WmsServer wmsServer;
private final WfsServer wfsServer;
private final RestTemplate restTemplate;
private final HttpEntity<?> pretendUserEntity;
private final int chunkSize;

public DownloadWfsDataService(
WmsServer wmsServer,
WfsServer wfsServer,
RestTemplate restTemplate,
@Qualifier("pretendUserEntity") HttpEntity<?> pretendUserEntity,
@Value("${app.sse.chunkSize:16384}") int chunkSize
) {
this.wmsServer = wmsServer;
this.wfsServer = wfsServer;
this.restTemplate = restTemplate;
this.pretendUserEntity = pretendUserEntity;
this.chunkSize = chunkSize;
}

/**
* Build CQL filter for temporal and spatial constraints
*/
Expand All @@ -64,7 +60,7 @@ protected String buildCqlFilter(String startDate, String endDate, Object multiPo
if (!temporalField.isEmpty() && startDate != null && !startDate.isEmpty() && endDate != null && !endDate.isEmpty()) {
List<String> cqls = new ArrayList<>();
temporalField.forEach(temp ->
cqls.add(String.format("(%s DURING %sT00:00:00Z/%sT23:59:59Z)", temp.getName(), startDate, endDate))
cqls.add(String.format("(%s DURING %sT00:00:00Z/%sT23:59:59Z)", temp.getName(), startDate, endDate))
);
cqlFilter.append("(").append(String.join(" OR ", cqls)).append(")");
}
Expand Down Expand Up @@ -95,6 +91,7 @@ protected String buildCqlFilter(String startDate, String endDate, Object multiPo

return cqlFilter.toString();
}

/**
* Does collection lookup, WFS validation, field retrieval, and URL building
*/
Expand All @@ -107,42 +104,27 @@ public String prepareWfsRequestUrl(
String layerName,
String outputFormat) {

DescribeLayerResponse describeLayerResponse = wmsServer.describeLayer(uuid, FeatureRequest.builder().layerName(layerName).build());

String wfsServerUrl;
String wfsTypeName;
WfsFields wfsFieldModel;

// Try to get WFS details from DescribeLayer first, then fallback to searching by layer name
if (describeLayerResponse != null && describeLayerResponse.getLayerDescription().getWfs() != null) {
wfsServerUrl = describeLayerResponse.getLayerDescription().getWfs();
wfsTypeName = describeLayerResponse.getLayerDescription().getQuery().getTypeName();
// Get WFS server URL and field model for the given UUID and layer name
Optional<String> featureServerUrl = wfsServer.getFeatureServerUrl(uuid, layerName);

// Get the wfs fields to build the CQL filter
if (featureServerUrl.isPresent()) {
wfsServerUrl = featureServerUrl.get();
wfsTypeName = layerName;
wfsFieldModel = wfsServer.getDownloadableFields(
uuid,
WfsServer.WfsFeatureRequest.builder()
.layerName(wfsTypeName)
.server(wfsServerUrl)
.build()
);
log.info("WFSFieldModel by describeLayer: {}", wfsFieldModel);
log.debug("WFSFieldModel by wfs typename: {}", wfsFieldModel);
} else {
Optional<String> featureServerUrl = wfsServer.getFeatureServerUrlByTitle(uuid, layerName);

if (featureServerUrl.isPresent()) {
wfsServerUrl = featureServerUrl.get();
wfsTypeName = layerName;
wfsFieldModel = wfsServer.getDownloadableFields(
uuid,
WfsServer.WfsFeatureRequest.builder()
.layerName(wfsTypeName)
.server(wfsServerUrl)
.build()
);
log.info("WFSFieldModel by wfs typename: {}", wfsFieldModel);
} else {
throw new IllegalArgumentException("No WFS server URL found for the given UUID and layer name");
}
throw new IllegalArgumentException("No WFS server URL found for the given UUID and layer name");
}

// Validate start and end dates
Expand All @@ -163,6 +145,7 @@ public String prepareWfsRequestUrl(
log.info("Prepared WFS request URL: {}", wfsRequestUrl);
return wfsRequestUrl;
}

/**
* Execute WFS request with SSE support
*/
Expand Down
67 changes: 39 additions & 28 deletions server/src/main/java/au/org/aodn/ogcapi/server/core/service/wfs/WfsServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,18 +79,26 @@ public WfsServer(Search search,
this.pretendUserEntity = entity;
this.wfsDefaultParam = wfsDefaultParam;
}

/**
* Build WFS GetFeature URL
*/
protected String createWfsRequestUrl(String wfsUrl, String layerName, List<String> fields, String cqlFilter, String outputFormat) {
UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(wfsUrl)
.scheme("https"); // Force HTTPS to fix redirect
UriComponents components = UriComponentsBuilder.fromUriString(wfsUrl).build();
UriComponentsBuilder builder = UriComponentsBuilder.newInstance()
.scheme("https") // Force HTTPS to fix redirect
.host(components.getHost())
.path(Objects.requireNonNull(components.getPath()));

if (components.getPort() != -1) {
builder.port(components.getPort());
}

Map<String, String> param = new HashMap<>(wfsDefaultParam.getDownload());
param.put("typeName", layerName);
param.put("outputFormat", outputFormat == null ? "text/csv" : outputFormat);

if(fields != null) {
if (fields != null) {
param.put("propertyName", String.join(",", fields));
}
// Add general query parameters
Expand All @@ -107,6 +115,7 @@ protected String createWfsRequestUrl(String wfsUrl, String layerName, List<Strin

return builder.build().toUriString();
}

/**
* Get all WFS links from a collection.
*
Expand Down Expand Up @@ -194,7 +203,7 @@ protected String createFeatureValueQueryUrl(String url, FeatureRequest request)
param.put("TYPENAME", request.getLayerName());
param.put("outputFormat", "application/json");

if(request.getProperties() != null && !request.getProperties().contains(FeatureRequest.PropertyName.wildcard)) {
if (request.getProperties() != null && !request.getProperties().contains(FeatureRequest.PropertyName.wildcard)) {
param.put("propertyName", String.join(
",",
request.getProperties().stream().map(Enum::name).toList())
Expand Down Expand Up @@ -276,7 +285,7 @@ public <T> T getFieldValues(String collectionId, WfsFeatureRequest request, Para
if (uri != null) {
ResponseEntity<T> response =
restTemplate.exchange(uri, HttpMethod.GET, pretendUserEntity, tClass
);
);

if (response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
return response.getBody();
Expand All @@ -289,11 +298,12 @@ public <T> T getFieldValues(String collectionId, WfsFeatureRequest request, Para
}
return null;
}

/**
* Get the downloadable fields for a given collection id and layer name
*
* @param collectionId - The uuid of the collection
* @param request - The feature request containing the layer name
* @param collectionId - The uuid of the collection
* @param request - The feature request containing the layer name
* @return - WFSFieldModel containing typename and fields
*/
@Cacheable(value = DOWNLOADABLE_FIELDS)
Expand Down Expand Up @@ -402,53 +412,54 @@ protected Optional<List<String>> getAllFeatureServerUrls(String collectionId) {
}
}


/**
* Find the url that is able to get WFS call, this can be found in ai:Group
*
* @param collectionId - The uuid
* @param layerName - The layer name to match the title
* @return - The first wfs server link if found
*/
public Optional<String> getFeatureServerUrlByTitle(String collectionId, String layerName) {
public Optional<String> getFeatureServerUrlByTitleOrQueryParam(String collectionId, String layerName) {
ElasticSearchBase.SearchResult<StacCollectionModel> result = search.searchCollections(collectionId);
if (!result.getCollections().isEmpty()) {
StacCollectionModel model = result.getCollections().get(0);
log.info("start to find wfs link for collectionId {} with layerName {}, total links to check {}", collectionId, layerName, model.getLinks().size());
return model.getLinks()
.stream()
.filter(link -> link.getAiGroup() != null)
.filter(link -> link.getAiGroup().contains(WFS_LINK_MARKER) && link.getTitle().equalsIgnoreCase(layerName))
.filter(link -> link.getAiGroup().contains(WFS_LINK_MARKER))
.filter(link -> {
Optional<String> name = extractLayernameOrTypenameFromUrl(link.getHref());
return roughlyMatch(link.getTitle(), layerName) ||
(name.isPresent() && roughlyMatch(name.get(), layerName));
})
.map(LinkModel::getHref)
.findFirst();
} else {
return Optional.empty();
}
}


/**
* Find the url that is able to get WFS call, this can be found in ai:Group
* Find the WFS server URL for a given collection and layer name.
* First tries to match by title or query param, then falls back to the first available WFS link.
*
* @param collectionId - The uuid
* @param layerName - The layer name to match the title
* @return - The first wfs server link if found
* @return - The matched wfs server link, or the first available one if no match found
*/
public Optional<String> getFeatureServerUrlByTitleOrQueryParam(String collectionId, String layerName) {
ElasticSearchBase.SearchResult<StacCollectionModel> result = search.searchCollections(collectionId);
if (!result.getCollections().isEmpty()) {
StacCollectionModel model = result.getCollections().get(0);
return model.getLinks()
.stream()
.filter(link -> link.getAiGroup() != null)
.filter(link -> link.getAiGroup().contains(WFS_LINK_MARKER))
.filter(link -> {
Optional<String> name = extractLayernameOrTypenameFromUrl(link.getHref());
return link.getTitle().equalsIgnoreCase(layerName) ||
(name.isPresent() && roughlyMatch(name.get(), layerName));
})
.map(LinkModel::getHref)
.findFirst();
} else {
return Optional.empty();
public Optional<String> getFeatureServerUrl(String collectionId, String layerName) {
Optional<String> url = getFeatureServerUrlByTitleOrQueryParam(collectionId, layerName);
if (url.isPresent()) {
log.debug("Found WFS link by title/query param for collectionId {} with layerName {}: {}", collectionId, layerName, url.get());
return url;
}

log.debug("No WFS link matched by title/query param for collectionId {} with layerName {}, falling back to first available WFS link", collectionId, layerName);
Optional<List<String>> allUrls = getAllFeatureServerUrls(collectionId);
return allUrls.filter(list -> !list.isEmpty()).map(list -> list.get(0));
}

/**
Expand Down
Loading
Loading