Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5,617 changes: 4,700 additions & 917 deletions package-lock.json

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"license": "ISC",
"dependencies": {
"@godaddy/terminus": "^4.12.1",
"@map-colonies/config": "^1.3.2",
"@map-colonies/config": "^2.2.1",
"@map-colonies/error-express-handler": "^2.1.0",
"@map-colonies/error-types": "^1.2.0",
"@map-colonies/express-access-log-middleware": "^2.0.1",
Expand All @@ -41,7 +41,7 @@
"@map-colonies/openapi-express-viewer": "^3.0.0",
"@map-colonies/raster-shared": "^3.1.4",
"@map-colonies/read-pkg": "0.0.1",
"@map-colonies/schemas": "v1.3.0",
"@map-colonies/schemas": "v1.8.0",
"@map-colonies/telemetry": "^7.0.1",
"@mapbox/geojsonhint": "^3.3.0",
"@opentelemetry/api": "^1.7.0",
Expand Down
2 changes: 2 additions & 0 deletions src/clients/jobManagerWrapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,13 @@ export class JobManagerWrapper extends JobManagerClient {
};
}),
};

const res = await this.createJob<ExportJobParameters, ITaskParameters>(createJobRequest);
const createJobResponse: ICreateExportJobResponse = {
jobId: res.id,
status: OperationStatus.PENDING,
};

return createJobResponse;
}

Expand Down
2 changes: 2 additions & 0 deletions src/export/models/exportManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ export class ExportManager {
}

const gpkgEstimatedSize = calculateEstimatedGpkgSize(featuresRecords, layerMetadata.tileOutputFormat);

await this.validationManager.validateFreeSpace(gpkgEstimatedSize, this.gpkgsLocation);

//creation of params
Expand All @@ -109,6 +110,7 @@ export class ExportManager {
jobTrackerUrl: this.jobTrackerUrl,
polygonPartsEntityName,
};

const jobCreated = await this.jobManagerClient.createExportJob(exportInitRequest);
return jobCreated;
}
Expand Down
12 changes: 11 additions & 1 deletion src/export/models/validationManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import { checkRoiFeatureCollectionSimilarity, sanitizeBbox } from '../../utils/g
@injectable()
export class ValidationManager {
private readonly storageEstimation: IStorageEstimation;
private readonly roiBufferMeter: number;
private readonly minContainedPercentage: number;

public constructor(
@inject(SERVICES.CONFIG) private readonly config: IConfig,
Expand All @@ -39,6 +41,8 @@ export class ValidationManager {
@inject(RasterCatalogManagerClient) private readonly rasterCatalogManager: RasterCatalogManagerClient
) {
this.storageEstimation = config.get<IStorageEstimation>('storageEstimation');
this.roiBufferMeter = config.get<number>('roiBufferMeter');
this.minContainedPercentage = config.get<number>('minContainedPercentage');
}

@withSpanAsyncV4
Expand Down Expand Up @@ -154,7 +158,13 @@ export class ValidationManager {
job.internalId === jobParams.catalogId &&
job.version === jobParams.version &&
job.parameters.exportInputParams.crs === jobParams.crs &&
checkRoiFeatureCollectionSimilarity(job.parameters.exportInputParams.roi, jobParams.roi, { config: this.config })
checkRoiFeatureCollectionSimilarity(
jobParams.roi,
job.parameters.exportInputParams.roi,
this.roiBufferMeter,
this.minContainedPercentage,
this.logger
)
);
return duplicateJob;
}
Expand Down
123 changes: 85 additions & 38 deletions src/utils/geometry.ts
Original file line number Diff line number Diff line change
@@ -1,80 +1,127 @@
/* eslint-disable @typescript-eslint/no-magic-numbers */
import { Logger } from '@map-colonies/js-logger';
import { container } from 'tsyringe';
import { area, buffer, feature, featureCollection, intersect } from '@turf/turf';
import { area, booleanContains, buffer, feature, featureCollection, intersect } from '@turf/turf';
import PolygonBbox from '@turf/bbox';
import { BBox, Feature, MultiPolygon, Polygon } from 'geojson';
import booleanContains from '@turf/boolean-contains';
import booleanEqual from '@turf/boolean-equal';
import { snapBBoxToTileGrid } from '@map-colonies/mc-utils';
import { RoiFeatureCollection, RoiProperties } from '@map-colonies/raster-shared';
import { SERVICES } from '@src/common/constants';
import { IConfig } from '@src/common/interfaces';

const areRoiPropertiesEqual = (props1: RoiProperties, props2: RoiProperties): boolean => {
return props1.maxResolutionDeg === props2.maxResolutionDeg && props1.minResolutionDeg === props2.minResolutionDeg;
};

const areGeometriesSimilar = (feature1: Feature, feature2: Feature, options: { minContainedPercentage: number; bufferMeter: number }): boolean => {
// Check direct containment
const feature1ContainsFeature2 = booleanContains(feature1, feature2);
const feature2ContainsFeature1 = booleanContains(feature2, feature1);
/**
* Check if containment exists and area ratio is within threshold
*/
const isContainedWithAreaThreshold = (containerFeature: Feature, containedFeature: Feature, thresholdRatio: number): boolean => {
if (!isGeometryContained(containerFeature, containedFeature)) {
return false;
}

if (feature1ContainsFeature2 || feature2ContainsFeature1) {
// Even with containment, check if areas are within threshold
const area1 = area(feature1);
const area2 = area(feature2);
const areaRatio = calculateAreaRatio(containedFeature, containerFeature);
return areaRatio >= thresholdRatio;
};

const areaRatio = Math.min(area1, area2) / Math.max(area1, area2);
const thresholdRatio = options.minContainedPercentage / 100;
const areGeometriesSimilar = (
requestRoiFeature: Feature,
jobRoiFeature: Feature,
options: { minContainedPercentage: number; bufferMeter: number }
): boolean => {
const thresholdRatio = options.minContainedPercentage / 100;

// If the area ratio is below threshold, they're too different in size
if (areaRatio < thresholdRatio) {
return false;
}
// Check if job ROI contains the request ROI with area threshold
if (isContainedWithAreaThreshold(jobRoiFeature, requestRoiFeature, thresholdRatio)) {
return true;
}

// Create buffered features
const bufferedFeature1 = buffer(feature1, options.bufferMeter, { units: 'meters' });
const bufferedFeature2 = buffer(feature2, options.bufferMeter, { units: 'meters' });
// If no direct containment, try with buffered job feature
const bufferedJobFeature = buffer(jobRoiFeature, options.bufferMeter, { units: 'meters' });

if (bufferedFeature1 === undefined || bufferedFeature2 === undefined) {
if (bufferedJobFeature === undefined) {
return false;
}
// Check if buffered feature1 contains feature2 or vice versa
return booleanContains(bufferedFeature1, feature2) || booleanContains(bufferedFeature2, feature1);

// Check if buffered job ROI contains the request ROI with area threshold
return isContainedWithAreaThreshold(bufferedJobFeature, requestRoiFeature, thresholdRatio);
};

export const checkRoiFeatureCollectionSimilarity = (fc1: RoiFeatureCollection, fc2: RoiFeatureCollection, options: { config: IConfig }): boolean => {
const roiBufferMeter = options.config.get<number>('roiBufferMeter');
const minContainedPercentage = options.config.get<number>('minContainedPercentage');
const logger: Logger = container.resolve(SERVICES.LOGGER);
const calculateAreaRatio = (feature1: Feature, feature2: Feature): number => {
const feature1Area = area(feature1);
const feature2Area = area(feature2);
const areaRatio = feature1Area / feature2Area;
return areaRatio;
};

/**
* Helper function to handle booleanContains with MultiPolygon geometries
* Works around Turf.js bug with MultiPolygon containment checks
*/
export const isGeometryContained = (completedJobRoi: Feature, requestedRoi: Feature): boolean => {
try {
if (booleanEqual(completedJobRoi, requestedRoi)) {
return true;
}

if (completedJobRoi.geometry.type === 'Polygon' && requestedRoi.geometry.type === 'MultiPolygon') {
const multiPolygon = requestedRoi.geometry;
return multiPolygon.coordinates.every((coords) => {
const polygon = { type: 'Polygon', coordinates: coords } as Polygon;
const polygonFeature = feature(polygon, requestedRoi.properties);
return booleanContains(completedJobRoi as Feature<Polygon>, polygonFeature);
});
}

if (completedJobRoi.geometry.type === 'Polygon' && requestedRoi.geometry.type === 'Polygon') {
return booleanContains(completedJobRoi as Feature<Polygon>, requestedRoi as Feature<Polygon>);
}

return false;
} catch (error) {
// If there's any error with the containment check, return false
return false;
}
};

export const checkRoiFeatureCollectionSimilarity = (
requestRoi: RoiFeatureCollection,
jobRoi: RoiFeatureCollection,
roiBufferMeter: number,
minContainedPercentage: number,
logger: Logger
): boolean => {
// If feature counts differ, they're not similar
if (fc1.features.length !== fc2.features.length) {
logger.debug({ msg: 'Feature counts differ, not similar', fc1Count: fc1.features.length, fc2Count: fc2.features.length });
if (requestRoi.features.length !== jobRoi.features.length) {
logger.debug({ msg: 'Feature counts differ, not similar', requestRoiCount: requestRoi.features.length, jobRoiCount: jobRoi.features.length });
return false;
}

// Track which features have found a match
const fc1Matched = new Array<boolean>(fc1.features.length).fill(false);
const fc2Matched = new Array<boolean>(fc2.features.length).fill(false);
const fc1Matched = new Array<boolean>(requestRoi.features.length).fill(false);
const fc2Matched = new Array<boolean>(jobRoi.features.length).fill(false);

for (let i = 0; i < fc1.features.length; i++) {
const feature1 = fc1.features[i];
for (let i = 0; i < requestRoi.features.length; i++) {
const feature1 = requestRoi.features[i];

for (let j = 0; j < fc2.features.length; j++) {
for (let j = 0; j < jobRoi.features.length; j++) {
// Skip already matched features in fc2
if (fc2Matched[j]) {
continue;
}

const feature2 = fc2.features[j];
const feature2 = jobRoi.features[j];

// Check if properties are exactly the same
const propsEqual = areRoiPropertiesEqual(feature1.properties, feature2.properties);
logger.debug({ msg: 'Checking properties', propsEqual, feature1Properties: feature1.properties, feature2Properties: feature2.properties });
if (!propsEqual) {
continue;
logger.info({
msg: 'Properties are different, therefore not similar',
propsEqual,
feature1Properties: feature1.properties,
feature2Properties: feature2.properties,
});
return false; // If properties differ, they are not similar
}

// Check geometric similarity
Expand Down
Loading