diff --git a/.run/basemap-dev.run.xml b/.run/basemap-dev.run.xml
index 50c0b11aa..2eb7063de 100644
--- a/.run/basemap-dev.run.xml
+++ b/.run/basemap-dev.run.xml
@@ -4,10 +4,10 @@
-
-
-
-
+
+
+
+
diff --git a/.run/basemap-mbtiles.run.xml b/.run/basemap-mbtiles.run.xml
index be5178b3f..39faf39e5 100644
--- a/.run/basemap-mbtiles.run.xml
+++ b/.run/basemap-mbtiles.run.xml
@@ -4,11 +4,6 @@
-
-
-
-
-
diff --git a/.run/basemap-serve.run.xml b/.run/basemap-serve.run.xml
index 84970ba82..9459903ae 100644
--- a/.run/basemap-serve.run.xml
+++ b/.run/basemap-serve.run.xml
@@ -4,10 +4,10 @@
-
-
-
-
+
+
+
+
diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml
index 7b2b0ee7b..261d98e34 100644
--- a/.run/basemap-workflow.run.xml
+++ b/.run/basemap-workflow.run.xml
@@ -4,10 +4,10 @@
-
-
-
-
+
+
+
+
diff --git a/.run/naturalearth-create.run.xml b/.run/naturalearth-create.run.xml
index fb309124a..4d2c8a8e3 100644
--- a/.run/naturalearth-create.run.xml
+++ b/.run/naturalearth-create.run.xml
@@ -4,6 +4,11 @@
+
+
+
+
+
diff --git a/.run/naturalearth-dev.run.xml b/.run/naturalearth-dev.run.xml
index 2ac6bf9b2..7be7b8d6c 100644
--- a/.run/naturalearth-dev.run.xml
+++ b/.run/naturalearth-dev.run.xml
@@ -4,6 +4,11 @@
+
+
+
+
+
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
index 7c36a536c..60dd9efd8 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java
@@ -23,7 +23,7 @@
import picocli.CommandLine.Command;
@Command(name = "database", description = "Database commands.",
- subcommands = {ExecuteSql.class, ImportOpenStreetMap.class, UpdateOpenStreetMap.class},
+ subcommands = {ExecuteSql.class, ImportOsmPbf.class, UpdateOpenStreetMap.class},
sortOptions = false)
public class Database implements Runnable {
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
similarity index 87%
rename from baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java
rename to baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
index f512a1f12..195ce110c 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java
@@ -28,7 +28,7 @@
import picocli.CommandLine.Option;
@Command(name = "import-osm", description = "Import OpenStreetMap data in Postgres.")
-public class ImportOpenStreetMap implements Callable {
+public class ImportOsmPbf implements Callable {
@Mixin
private Options options;
@@ -47,8 +47,13 @@ public class ImportOpenStreetMap implements Callable {
@Override
public Integer call() throws Exception {
- new org.apache.baremaps.workflow.tasks.ImportOpenStreetMap(file.toAbsolutePath(),
- database, srid).execute(new WorkflowContext());
+ new org.apache.baremaps.workflow.tasks.ImportOsmPbf(
+ file.toAbsolutePath(),
+ null,
+ true,
+ database,
+ srid,
+ true).execute(new WorkflowContext());
return 0;
}
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOpenStreetMap.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOpenStreetMap.java
index 3585353d9..6007be7f8 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOpenStreetMap.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/UpdateOpenStreetMap.java
@@ -22,6 +22,7 @@
import java.util.concurrent.Callable;
import org.apache.baremaps.cli.Options;
import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.baremaps.workflow.tasks.UpdateOsmDatabase;
import picocli.CommandLine.Command;
import picocli.CommandLine.Mixin;
import picocli.CommandLine.Option;
@@ -42,7 +43,7 @@ public class UpdateOpenStreetMap implements Callable {
@Override
public Integer call() throws Exception {
- new org.apache.baremaps.workflow.tasks.UpdateOpenStreetMap(database, srid)
+ new UpdateOsmDatabase(database, srid)
.execute(new WorkflowContext());
return 0;
}
diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
index 12823272d..dc85e334b 100644
--- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
+++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java
@@ -79,8 +79,9 @@ public Integer call() throws Exception {
var tileStoreType = new TypeLiteral>() {};
var tileStoreSupplier = (Supplier) () -> {
try {
+ var config = configReader.read(this.tilesetPath);
var tilesetObject =
- objectMapper.readValue(configReader.read(this.tilesetPath), Tileset.class);
+ objectMapper.readValue(config, Tileset.class);
return new PostgresTileStore(datasource, tilesetObject);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java
index 649340e8d..60e318ecc 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/geocoderosm/GeocoderOsmDocumentMapper.java
@@ -41,7 +41,7 @@ public class GeocoderOsmDocumentMapper implements Function {
@Override
public Document apply(Element element) {
var document = new Document();
- document.add(new StoredField("osm_id", element.id()));
+ document.add(new StoredField("osm_id", element.getId()));
document.add(new StoredField("osm_type", element.getClass().getSimpleName()));
if (element.getTags().containsKey(OsmTags.NAME.key())) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DiffService.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DiffService.java
index ec87cbed1..6857a82ad 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DiffService.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/DiffService.java
@@ -121,13 +121,13 @@ private Stream geometriesForPreviousVersion(Change change) {
private Optional geometriesForPreviousVersion(Entity entity) {
try {
if (entity instanceof Node node) {
- var previousNode = nodeRepository.get(node.id());
+ var previousNode = nodeRepository.get(node.getId());
return Optional.ofNullable(previousNode).map(Node::getGeometry);
} else if (entity instanceof Way way) {
- var previousWay = wayRepository.get(way.id());
+ var previousWay = wayRepository.get(way.getId());
return Optional.ofNullable(previousWay).map(Way::getGeometry);
} else if (entity instanceof Relation relation) {
- var previousRelation = relationRepository.get(relation.id());
+ var previousRelation = relationRepository.get(relation.getId());
return Optional.ofNullable(previousRelation).map(Relation::getGeometry);
} else {
return Optional.empty();
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/CoordinateMapBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/CoordinateMapBuilder.java
index d15647c5f..34a4731be 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/CoordinateMapBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/CoordinateMapBuilder.java
@@ -43,7 +43,7 @@ public CoordinateMapBuilder(DataMap coordinateMap) {
@Override
public void accept(Entity entity) {
if (entity instanceof Node node) {
- coordinateMap.put(node.id(), new Coordinate(node.getLon(), node.getLat()));
+ coordinateMap.put(node.getId(), new Coordinate(node.getLon(), node.getLat()));
}
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/GeometryMapBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/GeometryMapBuilder.java
index 522264eb2..3bac8678e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/GeometryMapBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/GeometryMapBuilder.java
@@ -50,7 +50,7 @@ public GeometryMapBuilder(DataMap geometryMap, Predicate
@Override
public void accept(Entity entity) {
if (filter.test(entity) && entity instanceof Element element) {
- geometryMap.put(element.id(), element.getGeometry());
+ geometryMap.put(element.getId(), element.getGeometry());
}
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/OpenstreetmapDocumentMapper.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/OpenstreetmapDocumentMapper.java
index 43ca7202e..b66597200 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/OpenstreetmapDocumentMapper.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/OpenstreetmapDocumentMapper.java
@@ -39,7 +39,7 @@ public class OpenstreetmapDocumentMapper implements Function
public Document apply(Element element) {
var document = new Document();
- document.add(new StoredField("id", element.id()));
+ document.add(new StoredField("id", element.getId()));
element.getTags().forEach((key, value) -> {
document.add(new StoredField(key, value.toString()));
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/ReferenceMapBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/ReferenceMapBuilder.java
index baf50855d..90368cc89 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/ReferenceMapBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/ReferenceMapBuilder.java
@@ -43,7 +43,7 @@ public ReferenceMapBuilder(DataMap> referenceMap) {
@Override
public void accept(Entity entity) {
if (entity instanceof Way way) {
- referenceMap.put(way.id(), way.getNodes());
+ referenceMap.put(way.getId(), way.getNodes());
}
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationGeometryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationGeometryBuilder.java
index 29c91f855..bf4661251 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationGeometryBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/RelationGeometryBuilder.java
@@ -99,10 +99,10 @@ public void accept(Relation relation) {
var end = System.currentTimeMillis();
var duration = end - start;
if (duration > 60 * 1000) {
- logger.debug("Relation #{} processed in {} ms", relation.id(), duration);
+ logger.debug("Relation #{} processed in {} ms", relation.getId(), duration);
}
} catch (Exception e) {
- logger.debug("Unable to build the geometry for relation #" + relation.id(), e);
+ logger.debug("Unable to build the geometry for relation #" + relation.getId(), e);
relation.setGeometry(GEOMETRY_FACTORY_WGS84.createEmpty(0));
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/WayGeometryBuilder.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/WayGeometryBuilder.java
index e7608aded..9432eed7e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/WayGeometryBuilder.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/function/WayGeometryBuilder.java
@@ -66,7 +66,7 @@ public void accept(Way way) {
}
}
} catch (Exception e) {
- logger.debug("Unable to build the geometry for way #" + way.id(), e);
+ logger.debug("Unable to build the geometry for way #" + way.getId(), e);
way.setGeometry(GEOMETRY_FACTORY_WGS84.createEmpty(0));
}
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Element.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Element.java
index 67ee3272b..e10cfa67a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Element.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Element.java
@@ -34,14 +34,18 @@ abstract class Element implements Entity
permits Node, Way, Relation
{
- protected final long id;
+ protected Long id;
- protected final Info info;
+ protected Info info;
- protected final Map tags;
+ protected Map tags;
protected Geometry geometry;
+ protected Element() {
+
+ }
+
protected Element(long id, Info info, Map tags) {
this(id, info, tags, null);
}
@@ -66,10 +70,17 @@ protected Element(long id, Info info, Map tags, Geometry geometr
*
* @return the id
*/
- public long id() {
+ public long getId() {
return id;
}
+ /**
+ * Sets the id.
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+
/**
* Returns the info.
*
@@ -79,6 +90,15 @@ public Info getInfo() {
return info;
}
+ /**
+ * Sets the info.
+ *
+ * @param info the info
+ */
+ public void setInfo(Info info) {
+ this.info = info;
+ }
+
/**
* Returns the tags.
*
@@ -88,6 +108,15 @@ public Map getTags() {
return tags;
}
+ /**
+ * Sets the tags.
+ *
+ * @param tags the tags
+ */
+ public void setTags(Map tags) {
+ this.tags = tags;
+ }
+
/**
* Returns the geometry.
*
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Node.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Node.java
index e7a902319..4b340cca6 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Node.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Node.java
@@ -27,9 +27,16 @@
/** Represents a node element in an OpenStreetMap dataset. */
public final class Node extends Element {
- private final double lon;
+ private Double lon;
- private final double lat;
+ private Double lat;
+
+ /**
+ * Constructs an OpenStreetMap {@code Node}.
+ */
+ public Node() {
+ super();
+ }
/**
* Constructs an OpenStreetMap {@code Node} with the specified parameters.
@@ -40,7 +47,7 @@ public final class Node extends Element {
* @param lon the longitude
* @param lat the latitude
*/
- public Node(long id, Info info, Map tags, double lon, double lat) {
+ public Node(Long id, Info info, Map tags, Double lon, Double lat) {
super(id, info, tags);
this.lon = lon;
this.lat = lat;
@@ -56,7 +63,7 @@ public Node(long id, Info info, Map tags, double lon, double lat
* @param lat the latitude
* @param geometry the geometry
*/
- public Node(long id, Info info, Map tags, double lon, double lat,
+ public Node(Long id, Info info, Map tags, Double lon, Double lat,
Geometry geometry) {
super(id, info, tags, geometry);
this.lon = lon;
@@ -68,19 +75,37 @@ public Node(long id, Info info, Map tags, double lon, double lat
*
* @return the longitude
*/
- public double getLon() {
+ public Double getLon() {
return lon;
}
+ /**
+ * Sets the longitude.
+ *
+ * @param lon the longitude
+ */
+ public void setLon(Double lon) {
+ this.lon = lon;
+ }
+
/**
* Returns the latitude.
*
* @return the latitude
*/
- public double getLat() {
+ public Double getLat() {
return lat;
}
+ /**
+ * Sets the latitude.
+ *
+ * @param lat the latitude
+ */
+ public void setLat(Double lat) {
+ this.lat = lat;
+ }
+
/** {@inheritDoc} */
@Override
public boolean equals(Object o) {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Relation.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Relation.java
index 1626cea03..03496cc31 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Relation.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Relation.java
@@ -28,7 +28,14 @@
/** Represents a relation element in an OpenStreetMap dataset. */
public final class Relation extends Element {
- private final List members;
+ private List members;
+
+ /**
+ * Constructs an OpenStreetMap {@code Relation}.
+ */
+ public Relation() {
+ super();
+ }
/**
* Constructs an OpenStreetMap {@code Relation} with the specified parameters.
@@ -38,7 +45,7 @@ public final class Relation extends Element {
* @param tags the tags
* @param members the members
*/
- public Relation(long id, Info info, Map tags, List members) {
+ public Relation(Long id, Info info, Map tags, List members) {
super(id, info, tags);
this.members = members;
}
@@ -52,7 +59,7 @@ public Relation(long id, Info info, Map tags, List membe
* @param members the members
* @param geometry the geometry
*/
- public Relation(long id, Info info, Map tags, List members,
+ public Relation(Long id, Info info, Map tags, List members,
Geometry geometry) {
super(id, info, tags, geometry);
this.members = members;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Way.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Way.java
index 1308c9f3d..c9bb9b558 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Way.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/model/Way.java
@@ -28,7 +28,14 @@
/** Represents a way element in an OpenStreetMap dataset. */
public final class Way extends Element {
- private final List nodes;
+ private List nodes;
+
+ /**
+ * Constructs an OpenStreetMap {@code Way}.
+ */
+ public Way() {
+ super();
+ }
/**
* Constructs an OpenStreetMap {@code Node} with the specified parameters.
@@ -38,7 +45,7 @@ public final class Way extends Element {
* @param tags the tags
* @param nodes the nodes
*/
- public Way(long id, Info info, Map tags, List nodes) {
+ public Way(Long id, Info info, Map tags, List nodes) {
super(id, info, tags);
this.nodes = nodes;
}
@@ -52,7 +59,7 @@ public Way(long id, Info info, Map tags, List nodes) {
* @param nodes the nodes
* @param geometry the geometry
*/
- public Way(long id, Info info, Map tags, List nodes, Geometry geometry) {
+ public Way(Long id, Info info, Map tags, List nodes, Geometry geometry) {
super(id, info, tags, geometry);
this.nodes = nodes;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java
index 02f1b8194..e54f22e4a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java
@@ -59,6 +59,8 @@ public class PostgresNodeRepository implements NodeRepository {
private final String delete;
+ private final String deleteIn;
+
private final String copy;
/**
@@ -91,13 +93,13 @@ public PostgresNodeRepository(DataSource dataSource, String tableName, String id
String tagsColumn, String longitudeColumn, String latitudeColumn, String geometryColumn) {
this.dataSource = dataSource;
this.createTable = String.format("""
- CREATE TABLE %1$s
+ CREATE TABLE IF NOT EXISTS %1$s
(
- %2$s bigint PRIMARY KEY,
+ %2$s int8 PRIMARY KEY,
%3$s int,
%4$s int,
%5$s timestamp without time zone,
- %6$s bigint,
+ %6$s int8,
%7$s jsonb,
%8$s float,
%9$s float,
@@ -124,6 +126,7 @@ public PostgresNodeRepository(DataSource dataSource, String tableName, String id
tableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn,
longitudeColumn, latitudeColumn, geometryColumn);
this.delete = String.format("DELETE FROM %1$s WHERE %2$s = ?", tableName, idColumn);
+ this.deleteIn = String.format("DELETE FROM %1$s WHERE %2$s = ANY (?)", tableName, idColumn);
this.copy = String.format(
"COPY %1$s (%2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s) FROM STDIN BINARY",
tableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn,
@@ -194,7 +197,7 @@ public List get(List keys) throws RepositoryException {
Map values = new HashMap<>();
while (result.next()) {
Node value = getValue(result);
- values.put(value.id(), value);
+ values.put(value.getId(), value);
}
return keys.stream().map(values::get).toList();
}
@@ -253,13 +256,9 @@ public void delete(List keys) throws RepositoryException {
return;
}
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement(delete)) {
- for (Long key : keys) {
- statement.clearParameters();
- statement.setObject(1, key);
- statement.addBatch();
- }
- statement.executeBatch();
+ PreparedStatement statement = connection.prepareStatement(deleteIn)) {
+ statement.setArray(1, connection.createArrayOf("int8", keys.toArray()));
+ statement.execute();
} catch (SQLException e) {
throw new RepositoryException(e);
}
@@ -277,7 +276,7 @@ public void copy(List values) throws RepositoryException {
writer.writeHeader();
for (Node value : values) {
writer.startRow(9);
- writer.writeLong(value.id());
+ writer.writeLong(value.getId());
writer.writeInteger(value.getInfo().getVersion());
writer.writeInteger(value.getInfo().getUid());
writer.writeLocalDateTime(value.getInfo().getTimestamp());
@@ -309,7 +308,7 @@ private Node getValue(ResultSet resultSet) throws SQLException, JsonProcessingEx
private void setValue(PreparedStatement statement, Node value)
throws SQLException, JsonProcessingException {
- statement.setObject(1, value.id());
+ statement.setObject(1, value.getId());
statement.setObject(2, value.getInfo().getVersion());
statement.setObject(3, value.getInfo().getUid());
statement.setObject(4, value.getInfo().getTimestamp());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java
index 39e7d0600..3e48679e2 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java
@@ -58,6 +58,8 @@ public class PostgresRelationRepository implements RelationRepository {
private final String delete;
+ private final String deleteIn;
+
private final String copy;
/**
@@ -92,7 +94,7 @@ public PostgresRelationRepository(DataSource dataSource, String tableName, Strin
String geometryColumn) {
this.dataSource = dataSource;
this.createTable = String.format("""
- CREATE TABLE %1$s (
+ CREATE TABLE IF NOT EXISTS %1$s (
%2$s bigint PRIMARY KEY,
%3$s int,
%4$s int,
@@ -130,6 +132,7 @@ ON CONFLICT (%2$s) DO UPDATE SET
%11$s = excluded.%11$s""", tableName, idColumn, versionColumn, uidColumn, timestampColumn,
changesetColumn, tagsColumn, memberRefs, memberTypes, memberRoles, geometryColumn);
this.delete = String.format("DELETE FROM %1$s WHERE %2$s = ?", tableName, idColumn);
+ this.deleteIn = String.format("DELETE FROM %1$s WHERE %2$s = ANY (?)", tableName, idColumn);
this.copy = String.format(
"COPY %1$s (%2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s, %10$s, %11$s) FROM STDIN BINARY",
tableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn,
@@ -200,7 +203,7 @@ public List get(List keys) throws RepositoryException {
Map values = new HashMap<>();
while (result.next()) {
Relation value = getValue(result);
- values.put(value.id(), value);
+ values.put(value.getId(), value);
}
return keys.stream().map(values::get).toList();
}
@@ -259,13 +262,9 @@ public void delete(List keys) throws RepositoryException {
return;
}
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement(delete)) {
- for (Long key : keys) {
- statement.clearParameters();
- statement.setObject(1, key);
- statement.addBatch();
- }
- statement.executeBatch();
+ PreparedStatement statement = connection.prepareStatement(deleteIn)) {
+ statement.setArray(1, connection.createArrayOf("int8", keys.toArray()));
+ statement.execute();
} catch (SQLException e) {
throw new RepositoryException(e);
}
@@ -283,7 +282,7 @@ public void copy(List values) throws RepositoryException {
writer.writeHeader();
for (Relation value : values) {
writer.startRow(10);
- writer.writeLong(value.id());
+ writer.writeLong(value.getId());
writer.writeInteger(value.getInfo().getVersion());
writer.writeInteger(value.getInfo().getUid());
writer.writeLocalDateTime(value.getInfo().getTimestamp());
@@ -324,7 +323,7 @@ private Relation getValue(ResultSet resultSet) throws SQLException, JsonProcessi
private void setValue(PreparedStatement statement, Relation value)
throws SQLException, JsonProcessingException {
- statement.setObject(1, value.id());
+ statement.setObject(1, value.getId());
statement.setObject(2, value.getInfo().getVersion());
statement.setObject(3, value.getInfo().getUid());
statement.setObject(4, value.getInfo().getTimestamp());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java
index bac693821..0e8d6124a 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java
@@ -62,6 +62,8 @@ public class PostgresWayRepository implements WayRepository {
private final String delete;
+ private final String deleteIn;
+
private final String copy;
/**
@@ -93,14 +95,14 @@ public PostgresWayRepository(DataSource dataSource, String tableName, String idC
String tagsColumn, String nodesColumn, String geometryColumn) {
this.dataSource = dataSource;
this.createTable = String.format("""
- CREATE TABLE %1$s (
- %2$s bigint PRIMARY KEY,
+ CREATE TABLE IF NOT EXISTS %1$s (
+ %2$s int8 PRIMARY KEY,
%3$s int,
%4$s int,
%5$s timestamp without time zone,
- %6$s bigint,
+ %6$s int8,
%7$s jsonb,
- %8$s bigint[],
+ %8$s int8[],
%9$s geometry
)""", tableName, idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn,
tagsColumn, nodesColumn, geometryColumn);
@@ -127,6 +129,7 @@ ON CONFLICT (%2$s) DO UPDATE SET
%9$s = excluded.%9$s""", tableName, idColumn, versionColumn, uidColumn, timestampColumn,
changesetColumn, tagsColumn, nodesColumn, geometryColumn);
this.delete = String.format("DELETE FROM %1$s WHERE %2$s = ?", tableName, idColumn);
+ this.deleteIn = String.format("DELETE FROM %1$s WHERE %2$s = ANY (?)", tableName, idColumn);
this.copy = String.format(
"COPY %1$s (%2$s, %3$s, %4$s, %5$s, %6$s, %7$s, %8$s, %9$s) FROM STDIN BINARY", tableName,
idColumn, versionColumn, uidColumn, timestampColumn, changesetColumn, tagsColumn,
@@ -197,7 +200,7 @@ public List get(List keys) throws RepositoryException {
Map values = new HashMap<>();
while (result.next()) {
Way value = getValue(result);
- values.put(value.id(), value);
+ values.put(value.getId(), value);
}
return keys.stream().map(values::get).toList();
}
@@ -256,13 +259,9 @@ public void delete(List keys) throws RepositoryException {
return;
}
try (Connection connection = dataSource.getConnection();
- PreparedStatement statement = connection.prepareStatement(delete)) {
- for (Long key : keys) {
- statement.clearParameters();
- statement.setObject(1, key);
- statement.execute();
- }
- statement.executeBatch();
+ PreparedStatement statement = connection.prepareStatement(deleteIn)) {
+ statement.setArray(1, connection.createArrayOf("int8", keys.toArray()));
+ statement.execute();
} catch (SQLException e) {
throw new RepositoryException(e);
}
@@ -279,7 +278,7 @@ public void copy(List values) throws RepositoryException {
writer.writeHeader();
for (Way value : values) {
writer.startRow(8);
- writer.writeLong(value.id());
+ writer.writeLong(value.getId());
writer.writeInteger(value.getInfo().getVersion());
writer.writeInteger(value.getInfo().getUid());
writer.writeLocalDateTime(value.getInfo().getTimestamp());
@@ -313,7 +312,7 @@ private Way getValue(ResultSet resultSet) throws SQLException, JsonProcessingExc
private void setValue(PreparedStatement statement, Way value)
throws SQLException, JsonProcessingException {
- statement.setObject(1, value.id());
+ statement.setObject(1, value.getId());
statement.setObject(2, value.getInfo().getVersion());
statement.setObject(3, value.getInfo().getUid());
statement.setObject(4, value.getInfo().getTimestamp());
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/repository/ChangeImporter.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/repository/ChangeImporter.java
index e52342d97..afe160782 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/repository/ChangeImporter.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/repository/ChangeImporter.java
@@ -24,11 +24,14 @@
import org.apache.baremaps.openstreetmap.model.Node;
import org.apache.baremaps.openstreetmap.model.Relation;
import org.apache.baremaps.openstreetmap.model.Way;
-import org.apache.baremaps.stream.StreamException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** A consumer for importing OpenStreetMap changes in a database. */
public class ChangeImporter implements Consumer {
+ private static final Logger logger = LoggerFactory.getLogger(ChangeImporter.class);
+
private final Repository nodeRepository;
private final Repository wayRepository;
private final Repository relationRepository;
@@ -52,34 +55,57 @@ public ChangeImporter(
/** {@inheritDoc} */
@Override
public void accept(Change change) {
+ var nodes = change.getEntities().stream()
+ .filter(entity -> entity instanceof Node)
+ .map(entity -> (Node) entity)
+ .toList();
+ var nodeIds = nodes.stream().map(Node::getId).toList();
+ var ways = change.getEntities().stream()
+ .filter(entity -> entity instanceof Way)
+ .map(entity -> (Way) entity)
+ .toList();
+ var wayIds = ways.stream().map(Way::getId).toList();
+ var relations = change.getEntities().stream()
+ .filter(entity -> entity instanceof Relation)
+ .map(entity -> (Relation) entity)
+ .toList();
+ var relationIds = relations.stream().map(Relation::getId).toList();
try {
- for (var entity : change.getEntities()) {
- switch (change.getType()) {
- case CREATE:
- case MODIFY:
- if (entity instanceof Node node) {
- nodeRepository.put(node);
- } else if (entity instanceof Way way) {
- wayRepository.put(way);
- } else if (entity instanceof Relation relation) {
- relationRepository.put(relation);
- }
- break;
- case DELETE:
- if (entity instanceof Node node) {
- nodeRepository.delete(node.id());
- } else if (entity instanceof Way way) {
- wayRepository.delete(way.id());
- } else if (entity instanceof Relation relation) {
- relationRepository.delete(relation.id());
- }
- break;
+ switch (change.getType()) {
+ case CREATE, MODIFY -> {
+ if (!nodes.isEmpty()) {
+ logger.trace("Creating {} nodes", nodes.size());
+ nodeRepository.delete(nodeIds);
+ nodeRepository.copy(nodes);
+ }
+ if (!ways.isEmpty()) {
+ logger.trace("Creating {} ways", ways.size());
+ wayRepository.delete(wayIds);
+ wayRepository.copy(ways);
+ }
+ if (!relations.isEmpty()) {
+ logger.trace("Creating {} relations", relations.size());
+ relationRepository.delete(relationIds);
+ relationRepository.copy(relations);
+ }
+ }
+ case DELETE -> {
+ if (!nodes.isEmpty()) {
+ logger.trace("Deleting {} nodes", nodes.size());
+ nodeRepository.delete(nodeIds);
+ }
+ if (!ways.isEmpty()) {
+ logger.trace("Deleting {} ways", ways.size());
+ wayRepository.delete(wayIds);
+ }
+ if (!relations.isEmpty()) {
+ logger.trace("Deleting {} relations", relations.size());
+ relationRepository.delete(relationIds);
+ }
}
}
} catch (RepositoryException e) {
- throw new StreamException(e);
+ logger.error("Error while saving changes", e);
}
}
-
-
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeSpliterator.java b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeSpliterator.java
index dbb6b9b73..9830d5baf 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeSpliterator.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/xml/XmlChangeSpliterator.java
@@ -155,8 +155,8 @@ private Element readElement() throws XMLStreamException {
private Node readNode() throws XMLStreamException {
long id = Long.parseLong(reader.getAttributeValue(null, ATTRIBUTE_NAME_ID));
Info info = readInfo();
- double latitude = Double.parseDouble(reader.getAttributeValue(null, ATTRIBUTE_NAME_LATITUDE));
- double longitude = Double.parseDouble(reader.getAttributeValue(null, ATTRIBUTE_NAME_LONGITUDE));
+ double lat = Double.parseDouble(reader.getAttributeValue(null, ATTRIBUTE_NAME_LATITUDE));
+ double lon = Double.parseDouble(reader.getAttributeValue(null, ATTRIBUTE_NAME_LONGITUDE));
// read the content of the node
Map tags = new HashMap<>();
@@ -172,7 +172,7 @@ private Node readNode() throws XMLStreamException {
}
}
- return new Node(id, info, tags, latitude, longitude);
+ return new Node(id, info, tags, lon, lat);
}
private Way readWay() throws XMLStreamException {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataTable.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataTable.java
index af2845af4..412b19ca5 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataTable.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/ShapefileDataTable.java
@@ -28,12 +28,16 @@
import org.apache.baremaps.database.schema.DataTableException;
import org.apache.baremaps.storage.shapefile.internal.ShapefileInputStream;
import org.apache.baremaps.storage.shapefile.internal.ShapefileReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A table that stores rows in a shapefile.
*/
public class ShapefileDataTable extends AbstractDataTable {
+ private static final Logger logger = LoggerFactory.getLogger(ShapefileDataTable.class);
+
private final ShapefileReader shapeFile;
/**
@@ -106,7 +110,8 @@ public boolean hasNext() {
next = shapefileInputStream.readRow();
}
return next != null;
- } catch (IOException exception) {
+ } catch (IOException e) {
+ logger.error("Malformed shapefile", e);
shapefileInputStream.close();
return false;
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java
index 24f9351f8..133eb9681 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/DbaseByteReader.java
@@ -168,6 +168,7 @@ public boolean nextRowAvailable() {
// 2) Check that the immediate next byte read isn't the EOF signal.
byte eofCheck = getByteBuffer().get();
+
if (eofCheck == 0x1A) {
return false;
} else {
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java
index 9c858fb9e..62b3cc266 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/storage/shapefile/internal/ShapefileByteReader.java
@@ -277,6 +277,10 @@ public void completeRow(DataRow row) throws ShapefileException {
}
switch (shapefileGeometryType) {
+ case NullShape:
+ loadNullRow(row);
+ break;
+
case Point:
loadPointRow(row);
break;
@@ -296,6 +300,15 @@ public void completeRow(DataRow row) throws ShapefileException {
getByteBuffer().order(ByteOrder.BIG_ENDIAN);
}
+ /**
+ * Load null row.
+ *
+ * @param row the row to fill.
+ */
+ private void loadNullRow(DataRow row) {
+ row.set(GEOMETRY_NAME, null);
+ }
+
/**
* Load point row.
*
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresQueryGenerator.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresQueryGenerator.java
index 5774bfbaf..3a89424bb 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresQueryGenerator.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresQueryGenerator.java
@@ -54,7 +54,7 @@ public class PostgresQueryGenerator {
* @param dataSource the data source
*/
public PostgresQueryGenerator(DataSource dataSource) {
- this(dataSource, null, null, null, null, null);
+ this(dataSource, null, null, null, null);
}
/**
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
index a11250333..add70a06e 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -90,6 +90,9 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
statement.setInt(i + 3, tileCoord.y());
}
+ // Log the sql query
+ logger.debug("Executing sql for tile {}: {}", tileCoord, statement);
+
try (ResultSet resultSet = statement.executeQuery();
OutputStream gzip = new GZIPOutputStream(data)) {
while (resultSet.next()) {
@@ -192,9 +195,8 @@ protected static Query prepareQuery(Tileset tileset, int zoom) {
var tileQueryTail = ") mvtTile";
tileSql.append(tileQueryTail);
- // Log the resulting sql
+ // Format the sql query
var sql = tileSql.toString().replace("\n", " ");
- logger.debug("sql: {}", sql);
return new Query(sql, paramCount);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/Compression.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/Compression.java
new file mode 100644
index 000000000..46a2569a3
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/Compression.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.utils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+
+public enum Compression {
+ none,
+ gzip,
+ bzip2;
+
+ public static Compression detect(Path file) {
+ if (file.toString().endsWith(".gz")) {
+ return gzip;
+ } else if (file.toString().endsWith(".bz2")) {
+ return bzip2;
+ } else {
+ return none;
+ }
+ }
+
+ public InputStream decompress(InputStream inputStream) throws IOException {
+ return switch (this) {
+ case gzip -> new GZIPInputStream(inputStream);
+ case bzip2 -> new BZip2CompressorInputStream(inputStream);
+ default -> inputStream;
+ };
+ }
+
+ public OutputStream compress(OutputStream outputStream) throws IOException {
+ return switch (this) {
+ case gzip -> new GZIPOutputStream(outputStream);
+ case bzip2 -> new BZip2CompressorOutputStream(outputStream);
+ default -> outputStream;
+ };
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java b/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java
index ec42a13a3..ca8328258 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/utils/ProjectionTransformer.java
@@ -19,6 +19,7 @@
+import java.util.Objects;
import java.util.stream.Stream;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.CoordinateSequence;
@@ -76,9 +77,11 @@ public ProjectionTransformer(int sourceSrid, int targetSrid) {
}
@Override
- protected CoordinateSequence transformCoordinates(CoordinateSequence coordinateSequence,
+ protected CoordinateSequence transformCoordinates(
+ CoordinateSequence coordinateSequence,
Geometry parent) {
Coordinate[] coordinateArray = Stream.of(coordinateSequence.toCoordinateArray())
+ .filter(Objects::nonNull)
.map(this::transformCoordinate).toArray(Coordinate[]::new);
return new CoordinateArraySequence(coordinateArray);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
index c070d7c7d..f1dbfbccc 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/vectortile/tileset/Tileset.java
@@ -76,7 +76,6 @@ public class Tileset {
@JsonProperty("center")
private List center = new ArrayList<>();
- @Deprecated
@JsonProperty("database")
private Object database;
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
index c6862761d..20cf880e2 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java
@@ -38,14 +38,22 @@
@JsonSubTypes.Type(value = ExecuteSqlScript.class, name = "ExecuteSqlScript"),
@JsonSubTypes.Type(value = ExportVectorTiles.class, name = "ExportVectorTiles"),
@JsonSubTypes.Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"),
- @JsonSubTypes.Type(value = ImportOpenStreetMap.class, name = "ImportOpenStreetMap"),
+ @JsonSubTypes.Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"),
+ @JsonSubTypes.Type(value = ImportOsmOsc.class, name = "ImportOsmOsc"),
@JsonSubTypes.Type(value = ImportShapefile.class, name = "ImportShapefile"),
@JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"),
@JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"),
@JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"),
- @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"),
+ @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"),
+ @JsonSubTypes.Type(value = DecompressFile.class, name = "DecompressFile"),
+ @JsonSubTypes.Type(value = UpdateOsmDatabase.class, name = "UpdateOsmDatabase"),
@JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"),
- @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")})
+ @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex"),
+ @JsonSubTypes.Type(value = ImportDaylightTranslations.class,
+ name = "ImportDaylightTranslations"),
+ @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures"),
+ @JsonSubTypes.Type(value = ImportWikidata.class, name = "ImportWikidata")
+})
public interface Task {
/**
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java
index 9ec177ed3..43de015c8 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java
@@ -210,28 +210,39 @@ private CompletableFuture getPreviousFutureStep(String stepId) {
*/
private void logStepMeasures() {
logger.info("----------------------------------------");
+
var workflowStart = stepMeasures.stream()
- .mapToLong(measures -> measures.stepMeasures.stream()
- .mapToLong(measure -> measure.start)
- .min().getAsLong())
- .min().getAsLong();
+ .flatMapToLong(measures -> measures.stepMeasures.stream()
+ .mapToLong(measure -> measure.start))
+ .min();
+
var workflowEnd = stepMeasures.stream()
- .mapToLong(measures -> measures.stepMeasures.stream()
- .mapToLong(measure -> measure.end)
- .max().getAsLong())
- .max().getAsLong();
- var workflowDuration = Duration.ofMillis(workflowEnd - workflowStart);
- logger.info("Workflow graph: {}", this.graph);
- logger.info(" Duration: {}", formatDuration(workflowDuration));
+ .flatMapToLong(measures -> measures.stepMeasures.stream()
+ .mapToLong(measure -> measure.end))
+ .max();
+
+ if (workflowStart.isPresent() && workflowEnd.isPresent()) {
+ var workflowDuration = Duration.ofMillis(workflowEnd.getAsLong() - workflowStart.getAsLong());
+ logger.info("Workflow graph: {}", this.graph);
+ logger.info(" Duration: {}", formatDuration(workflowDuration));
+ } else {
+ logger.info("Workflow graph: {}", this.graph);
+ logger.info(" Duration: unknown");
+ }
for (var stepMeasure : this.stepMeasures) {
- var stepStart =
- stepMeasure.stepMeasures.stream().mapToLong(measure -> measure.start).min().getAsLong();
- var stepEnd =
- stepMeasure.stepMeasures.stream().mapToLong(measure -> measure.end).max().getAsLong();
- var stepDuration = Duration.ofMillis(stepEnd - stepStart);
- logger.info("Step: {}, Duration: {} ms", stepMeasure.step.getId(),
- formatDuration(stepDuration));
+ var stepStart = stepMeasure.stepMeasures.stream()
+ .mapToLong(measure -> measure.start).min();
+ var stepEnd = stepMeasure.stepMeasures.stream()
+ .mapToLong(measure -> measure.end).max();
+
+ if (stepStart.isPresent() && stepEnd.isPresent()) {
+ var stepDuration = Duration.ofMillis(stepEnd.getAsLong() - stepStart.getAsLong());
+ logger.info("Step: {}, Duration: {} ms", stepMeasure.step.getId(),
+ formatDuration(stepDuration));
+ } else {
+ logger.info("Step: {}, Duration: unknown", stepMeasure.step.getId());
+ }
for (var taskMeasure : stepMeasure.stepMeasures) {
var taskDuration = Duration.ofMillis(taskMeasure.end - taskMeasure.start);
@@ -251,24 +262,28 @@ private void logStepMeasures() {
private static String formatDuration(Duration duration) {
var builder = new StringBuilder();
var days = duration.toDays();
- if (days > 0) {
- builder.append(days).append(" days ");
- }
- final long hrs = duration.toHours() - Duration.ofDays(duration.toDays()).toHours();
- if (hrs > 0) {
- builder.append(hrs).append(" hrs ");
- }
- final long min = duration.toMinutes() - Duration.ofHours(duration.toHours()).toMinutes();
- if (min > 0) {
- builder.append(min).append(" min ");
- }
- final long sec = duration.toSeconds() - Duration.ofMinutes(duration.toMinutes()).toSeconds();
- if (sec > 0) {
- builder.append(sec).append(" s ");
- }
- final long ms = duration.toMillis() - Duration.ofSeconds(duration.toSeconds()).toMillis();
- if (ms > 0) {
- builder.append(ms).append(" ms ");
+ if (duration.isZero()) {
+ builder.append("0 ms");
+ } else {
+ if (days > 0) {
+ builder.append(days).append(" days ");
+ }
+ final long hrs = duration.toHours() - Duration.ofDays(duration.toDays()).toHours();
+ if (hrs > 0) {
+ builder.append(hrs).append(" hrs ");
+ }
+ final long min = duration.toMinutes() - Duration.ofHours(duration.toHours()).toMinutes();
+ if (min > 0) {
+ builder.append(min).append(" min ");
+ }
+ final long sec = duration.toSeconds() - Duration.ofMinutes(duration.toMinutes()).toSeconds();
+ if (sec > 0) {
+ builder.append(sec).append(" s ");
+ }
+ final long ms = duration.toMillis() - Duration.ofSeconds(duration.toSeconds()).toMillis();
+ if (ms > 0) {
+ builder.append(ms).append(" ms ");
+ }
}
return builder.toString();
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java
new file mode 100644
index 000000000..29c6ec49d
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import java.io.BufferedInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.baremaps.workflow.WorkflowException;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+
+public record DecompressBZip2(Path source, Path target) implements Task {
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var sourcePath = source.toAbsolutePath();
+ try (var zis =
+ new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) {
+ var targetPath = target.toAbsolutePath();
+ if (!Files.exists(targetPath)) {
+ Files.createDirectories(targetPath.getParent());
+ Files.createFile(targetPath);
+ }
+ Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING);
+ } catch (Exception e) {
+ throw new WorkflowException(e);
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
new file mode 100644
index 000000000..904fc495b
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipFile;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public record DecompressFile(Path source, Path target, Compression compression) implements Task {
+
+ public enum Compression {
+ zip,
+ targz,
+ tarbz2,
+ gzip,
+ bzip2;
+ }
+
+ private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class);
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var sourcePath = source.toAbsolutePath();
+ var targetPath = target.toAbsolutePath();
+ switch (compression) {
+ case zip:
+ decompressZip(sourcePath, targetPath);
+ break;
+ case targz:
+ decompressTarGz(sourcePath, targetPath);
+ break;
+ case tarbz2:
+ decompressTarBz2(sourcePath, targetPath);
+ break;
+ case gzip:
+ decompressGzip(sourcePath, targetPath);
+ break;
+ case bzip2:
+ decompressBzip2(sourcePath, targetPath);
+ break;
+ }
+
+ }
+
+ public static void decompressBzip2(Path sourcePath, Path targetPath) throws IOException {
+ try (var zis =
+ new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) {
+ Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ public static void decompressGzip(Path sourcePath, Path targetPath) throws IOException {
+ try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) {
+ Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ public static void decompressTarGz(Path sourcePath, Path targetPath) throws IOException {
+ try (
+ GZIPInputStream gzipInputStream =
+ new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)));
+ TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) {
+ TarArchiveEntry entry;
+ while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) {
+ var path = targetPath.resolve(entry.getName());
+ if (entry.isDirectory()) {
+ Files.createDirectories(path);
+ } else {
+ Files.createDirectories(path.getParent());
+ try (BufferedOutputStream outputStream =
+ new BufferedOutputStream(Files.newOutputStream(path))) {
+ int bytesRead;
+ byte[] buffer = new byte[4096];
+ while ((bytesRead = tarInputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public static void decompressTarBz2(Path sourcePath, Path targetPath) throws IOException {
+ try (
+ BZip2CompressorInputStream gzipInputStream = new BZip2CompressorInputStream(
+ new BufferedInputStream(Files.newInputStream(sourcePath)));
+ TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) {
+ TarArchiveEntry entry;
+ while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) {
+ var path = targetPath.resolve(entry.getName());
+ if (entry.isDirectory()) {
+ Files.createDirectories(path);
+ } else {
+ Files.createDirectories(path.getParent());
+ try (BufferedOutputStream outputStream =
+ new BufferedOutputStream(Files.newOutputStream(path))) {
+ int bytesRead;
+ byte[] buffer = new byte[4096];
+ while ((bytesRead = tarInputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public static void decompressZip(Path sourcePath, Path targetPath) throws IOException {
+ try (var zipFile = new ZipFile(sourcePath.toFile())) {
+ var entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ var entry = entries.nextElement();
+ var path = targetPath.resolve(entry.getName());
+ Files.createDirectories(path.getParent());
+ Files.write(path, new byte[] {}, StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING);
+ try (var input = new BufferedInputStream(zipFile.getInputStream(entry));
+ var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) {
+ int nBytes = -1;
+ byte[] buffer = new byte[4096];
+ while ((nBytes = input.read(buffer)) > 0) {
+ output.write(buffer, 0, nBytes);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
index 667d263f3..25058c8d0 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExecuteSql.java
@@ -26,9 +26,13 @@
import org.apache.baremaps.workflow.Task;
import org.apache.baremaps.workflow.WorkflowContext;
import org.apache.baremaps.workflow.WorkflowException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public record ExecuteSql(Object database, Path file, boolean parallel) implements Task {
+ private static final Logger logger = LoggerFactory.getLogger(ExecuteSql.class);
+
@Override
public void execute(WorkflowContext context) throws Exception {
var script = clean(Files.readString(file));
@@ -39,9 +43,9 @@ public void execute(WorkflowContext context) throws Exception {
queries.forEach(
query -> {
var dataSource = context.getDataSource(database);
- try (var connection = dataSource.getConnection();
- var statement = connection.createStatement()) {
- statement.execute(query);
+ try (var connection = dataSource.getConnection()) {
+ logger.info("Execute SQL query: {}", query.replaceAll("\\s+", " "));
+ connection.createStatement().execute(query);
} catch (SQLException e) {
throw new WorkflowException(e);
}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java
new file mode 100644
index 000000000..ce0b3d903
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightFeatures.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository;
+import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository;
+import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository;
+import org.apache.baremaps.utils.ProjectionTransformer;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.locationtech.jts.geom.GeometryFactory;
+import org.locationtech.jts.io.WKTReader;
+
+public record ImportDaylightFeatures(Path file, Object database) implements Task {
+
+ record Feature(
+ @JsonProperty("osm_type") String type,
+ @JsonProperty("osm_id") Long id,
+ @JsonProperty("tags") Map tags,
+ @JsonProperty("wkt") String wkt,
+ @JsonProperty("category") String category) {
+
+ }
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var datasource = context.getDataSource(database);
+ var geometryFactory = new GeometryFactory();
+ var wktReader = new WKTReader(geometryFactory);
+ var projectionTransformer = new ProjectionTransformer(4326, 3857);
+
+ // Initialize the repositories
+ var nodeRepository = new PostgresNodeRepository(datasource);
+ var wayRepository = new PostgresWayRepository(datasource);
+ var relationRepository = new PostgresRelationRepository(datasource);
+ nodeRepository.create();
+ wayRepository.create();
+ relationRepository.create();
+
+ // Process the file
+ var objectMapper = new ObjectMapper();
+ var javaType = objectMapper.getTypeFactory().constructParametricType(List.class, Feature.class);
+ List features = objectMapper.readValue(file.toFile(), javaType);
+ for (var feature : features) {
+ switch (feature.type()) {
+ case "node" -> {
+ var node = nodeRepository.get(feature.id());
+ if (node != null) {
+ // Merge the tags
+ var tags = new HashMap<>(feature.tags());
+ if (node.getTags() != null) {
+ tags.putAll(node.getTags());
+ }
+ node.setTags(tags);
+
+ // Update the node
+ nodeRepository.put(node);
+ }
+ }
+ case "way" -> {
+ var way = wayRepository.get(feature.id());
+ if (way != null) {
+ // Merge the tags
+ var tags = new HashMap<>(feature.tags());
+ if (way.getTags() != null) {
+ tags.putAll(way.getTags());
+ }
+ way.setTags(tags);
+
+ // Update the way
+ wayRepository.put(way);
+ }
+ }
+ case "relation" -> {
+ var relation = relationRepository.get(feature.id());
+ if (relation != null) {
+ // Merge the tags
+ var tags = new HashMap<>(feature.tags());
+ if (relation.getTags() != null) {
+ tags.putAll(relation.getTags());
+ }
+ relation.setTags(tags);
+
+ // Update the relation
+ relationRepository.put(relation);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java
new file mode 100644
index 000000000..c5e9d86a3
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportDaylightTranslations.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.stream.Collectors;
+import org.apache.baremaps.openstreetmap.postgres.PostgresNodeRepository;
+import org.apache.baremaps.openstreetmap.postgres.PostgresRelationRepository;
+import org.apache.baremaps.openstreetmap.postgres.PostgresWayRepository;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+
+public record ImportDaylightTranslations(Path file, Object database) implements Task {
+
+ record Group(String type, Long id, String name) {
+
+ }
+
+ record Line(String type, Long id, String name, String attributeKey, String attributeValue) {
+
+ public Group group() {
+ return new Group(type, id, name);
+ }
+
+ public static Line parse(String line) {
+ var parts = line.split("\t");
+ var type = parts[0];
+ var id = Long.parseLong(parts[1]);
+ var name = parts[2];
+ var key = parts[3];
+ var val = parts[4];
+ return new Line(type, id, name, key, val);
+ }
+ }
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var datasource = context.getDataSource(database);
+
+ // Initialize the repositories
+ var nodeRepository = new PostgresNodeRepository(datasource);
+ var wayRepository = new PostgresWayRepository(datasource);
+ var relationRepository = new PostgresRelationRepository(datasource);
+ nodeRepository.create();
+ wayRepository.create();
+ relationRepository.create();
+
+ // Process the file
+ try (var lines = Files.lines(file)) {
+ var entries = lines.map(Line::parse).collect(Collectors.groupingBy(Line::group));
+ for (var entry : entries.entrySet()) {
+ var group = entry.getKey();
+ switch (group.type()) {
+ case "node" -> {
+ var node = nodeRepository.get(group.id());
+ if (node != null) {
+ var tags = new HashMap<>(node.getTags());
+ for (var line : entry.getValue()) {
+ tags.put(line.attributeKey(), line.attributeValue());
+ }
+ node.setTags(tags);
+ nodeRepository.put(node);
+ }
+ }
+ case "way" -> {
+ var way = wayRepository.get(group.id());
+ if (way != null) {
+ var tags = new HashMap<>(way.getTags());
+ for (var line : entry.getValue()) {
+ tags.put(line.attributeKey(), line.attributeValue());
+ }
+ way.setTags(tags);
+ wayRepository.put(way);
+ }
+ }
+ case "relation" -> {
+ var relation = relationRepository.get(group.id());
+ if (relation != null) {
+ var tags = new HashMap<>(relation.getTags());
+ for (var line : entry.getValue()) {
+ tags.put(line.attributeKey(), line.attributeValue());
+ }
+ relation.setTags(tags);
+ relationRepository.put(relation);
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java
new file mode 100644
index 000000000..9fa527864
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmOsc.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn;
+
+import java.io.BufferedInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.baremaps.database.collection.*;
+import org.apache.baremaps.database.memory.MemoryMappedDirectory;
+import org.apache.baremaps.database.type.LongDataType;
+import org.apache.baremaps.database.type.LongListDataType;
+import org.apache.baremaps.database.type.PairDataType;
+import org.apache.baremaps.database.type.geometry.LonLatDataType;
+import org.apache.baremaps.openstreetmap.function.*;
+import org.apache.baremaps.openstreetmap.postgres.*;
+import org.apache.baremaps.openstreetmap.repository.ChangeImporter;
+import org.apache.baremaps.openstreetmap.xml.XmlChangeReader;
+import org.apache.baremaps.utils.Compression;
+import org.apache.baremaps.utils.FileUtils;
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public record ImportOsmOsc(
+ Path file,
+ Path cache,
+ Object database,
+ Integer srid,
+ Compression compression) implements Task {
+
+ private static final Logger logger = LoggerFactory.getLogger(ImportOsmOsc.class);
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ var datasource = context.getDataSource(database);
+ var path = file.toAbsolutePath();
+
+ var cacheDir = cache != null ? cache : Files.createTempDirectory(Paths.get("."), "cache_");
+
+ var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinates"));
+ var coordinateMap = new MemoryAlignedDataMap<>(
+ new LonLatDataType(),
+ new MemoryMappedDirectory(coordinateDir));;
+
+ var referenceKeysDir = Files.createDirectories(cacheDir.resolve("reference_keys"));
+ var referenceValuesDir = Files.createDirectories(cacheDir.resolve("reference_vals"));
+ var referenceMap =
+ new MonotonicDataMap<>(
+ new MemoryAlignedDataList<>(
+ new PairDataType<>(new LongDataType(), new LongDataType()),
+ new MemoryMappedDirectory(referenceKeysDir)),
+ new AppendOnlyBuffer<>(
+ new LongListDataType(),
+ new MemoryMappedDirectory(referenceValuesDir)));
+
+ var nodeRepository = new PostgresNodeRepository(datasource);
+ var wayRepository = new PostgresWayRepository(datasource);
+ var relationRepository = new PostgresRelationRepository(datasource);
+
+ var coordinateMapBuilder = new CoordinateMapBuilder(coordinateMap);
+ var referenceMapBuilder = new ReferenceMapBuilder(referenceMap);
+ var buildGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap);
+ var reprojectGeometry = new EntityProjectionTransformer(4326, srid);
+ var prepareGeometries = coordinateMapBuilder
+ .andThen(referenceMapBuilder)
+ .andThen(buildGeometry)
+ .andThen(reprojectGeometry);
+ var prepareChange = consumeThenReturn(new ChangeEntitiesHandler(prepareGeometries));
+ var importChange = new ChangeImporter(nodeRepository, wayRepository, relationRepository);
+
+ try (var changeInputStream =
+ new BufferedInputStream(compression.decompress(Files.newInputStream(path)))) {
+ new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(importChange);
+ }
+
+ FileUtils.deleteRecursively(cacheDir);
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
similarity index 73%
rename from baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
rename to baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
index d95195be6..d8a589255 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java
@@ -46,11 +46,15 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record ImportOpenStreetMap(Path file, Object database, Integer databaseSrid)
- implements
- Task {
+public record ImportOsmPbf(
+ Path file,
+ Path cache,
+ Boolean cleanCache,
+ Object database,
+ Integer databaseSrid,
+ Boolean replaceExisting) implements Task {
- private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class);
+ private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class);
@Override
public void execute(WorkflowContext context) throws Exception {
@@ -62,39 +66,26 @@ public void execute(WorkflowContext context) throws Exception {
var wayRepository = new PostgresWayRepository(dataSource);
var relationRepository = new PostgresRelationRepository(dataSource);
- headerRepository.drop();
- nodeRepository.drop();
- wayRepository.drop();
- relationRepository.drop();
-
- headerRepository.create();
- nodeRepository.create();
- wayRepository.create();
- relationRepository.create();
-
- var cacheDir = Files.createTempDirectory(Paths.get("."), "cache_");
-
- DataMap coordinateMap;
- if (Files.size(path) > 1 << 30) {
- var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinate_keys"));
- coordinateMap = new MemoryAlignedDataMap<>(
- new LonLatDataType(),
- new MemoryMappedDirectory(coordinateDir));
- } else {
- var coordinateKeysDir = Files.createDirectories(cacheDir.resolve("coordinate_keys"));
- var coordinateValuesDir = Files.createDirectories(cacheDir.resolve("coordinate_vals"));
- coordinateMap =
- new MonotonicDataMap<>(
- new MemoryAlignedDataList<>(
- new PairDataType<>(new LongDataType(), new LongDataType()),
- new MemoryMappedDirectory(coordinateKeysDir)),
- new AppendOnlyBuffer<>(
- new LonLatDataType(),
- new MemoryMappedDirectory(coordinateValuesDir)));
+ if (replaceExisting) {
+ headerRepository.drop();
+ nodeRepository.drop();
+ wayRepository.drop();
+ relationRepository.drop();
+ headerRepository.create();
+ nodeRepository.create();
+ wayRepository.create();
+ relationRepository.create();
}
- var referenceKeysDir = Files.createDirectory(cacheDir.resolve("reference_keys"));
- var referenceValuesDir = Files.createDirectory(cacheDir.resolve("reference_vals"));
+ var cacheDir = cache != null ? cache : Files.createTempDirectory(Paths.get("."), "cache_");
+
+ var coordinateDir = Files.createDirectories(cacheDir.resolve("coordinates"));
+ var coordinateMap = new MemoryAlignedDataMap<>(
+ new LonLatDataType(),
+ new MemoryMappedDirectory(coordinateDir));
+
+ var referenceKeysDir = Files.createDirectories(cacheDir.resolve("reference_keys"));
+ var referenceValuesDir = Files.createDirectories(cacheDir.resolve("reference_vals"));
var referenceMap =
new MonotonicDataMap<>(
new MemoryAlignedDataList<>(
@@ -114,7 +105,9 @@ public void execute(WorkflowContext context) throws Exception {
relationRepository,
databaseSrid);
- FileUtils.deleteRecursively(cacheDir);
+ if (cleanCache) {
+ FileUtils.deleteRecursively(cacheDir);
+ }
}
public static void execute(
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java
new file mode 100644
index 000000000..4b218b3e9
--- /dev/null
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import org.apache.baremaps.workflow.Task;
+import org.apache.baremaps.workflow.WorkflowContext;
+import org.wikidata.wdtk.datamodel.interfaces.*;
+import org.wikidata.wdtk.dumpfiles.DumpProcessingController;
+
+public record ImportWikidata(Object database) implements Task {
+
+ @Override
+ public void execute(WorkflowContext context) throws Exception {
+ DumpProcessingController controller = new DumpProcessingController("wikidatawiki");
+ controller.setOfflineMode(false);
+ controller.registerEntityDocumentProcessor(new CustomProcessor(), null, true);
+ controller.processMostRecentJsonDump();
+ }
+
+ public class CustomProcessor implements EntityDocumentDumpProcessor {
+
+ @Override
+ public void processItemDocument(ItemDocument itemDocument) {}
+
+ @Override
+ public void processPropertyDocument(PropertyDocument propertyDocument) {}
+
+ @Override
+ public void processLexemeDocument(LexemeDocument lexemeDocument) {}
+
+ @Override
+ public void processMediaInfoDocument(MediaInfoDocument mediaInfoDocument) {}
+
+ @Override
+ public void open() {
+
+ }
+
+ @Override
+ public void close() {
+
+ }
+ }
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java
similarity index 88%
rename from baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java
rename to baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java
index 7c6a7e5c6..0b195a277 100644
--- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOpenStreetMap.java
+++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/UpdateOsmDatabase.java
@@ -49,19 +49,19 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public record UpdateOpenStreetMap(Object database, Integer databaseSrid) implements Task {
+public record UpdateOsmDatabase(Object database, Integer databaseSrid) implements Task {
- private static final Logger logger = LoggerFactory.getLogger(UpdateOpenStreetMap.class);
+ private static final Logger logger = LoggerFactory.getLogger(UpdateOsmDatabase.class);
@Override
public void execute(WorkflowContext context) throws Exception {
var datasource = context.getDataSource(database);
- DataMap coordinateMap = new PostgresCoordinateMap(datasource);
- DataMap> referenceMap = new PostgresReferenceMap(datasource);
- HeaderRepository headerRepository = new PostgresHeaderRepository(datasource);
- Repository nodeRepository = new PostgresNodeRepository(datasource);
- Repository wayRepository = new PostgresWayRepository(datasource);
- Repository relationRepository = new PostgresRelationRepository(datasource);
+ var coordinateMap = new PostgresCoordinateMap(datasource);
+ var referenceMap = new PostgresReferenceMap(datasource);
+ var headerRepository = new PostgresHeaderRepository(datasource);
+ var nodeRepository = new PostgresNodeRepository(datasource);
+ var wayRepository = new PostgresWayRepository(datasource);
+ var relationRepository = new PostgresRelationRepository(datasource);
execute(
coordinateMap,
referenceMap,
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java b/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java
index 3943456a1..394aac0e0 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/database/calcite/CalciteTest.java
@@ -108,6 +108,7 @@ SELECT ST_AsText(ST_AsMVTGeom(
ST_MakeEnvelope(0, 0, 4096, 4096),
4096, 0, true))
""";
+
try (Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(sql)) {
assertTrue(resultSet.next());
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
index 6439937db..1d707c3a9 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/EntityDataTypeGeometryBuilderTest.java
@@ -80,81 +80,81 @@ public ProjCoordinate transform(ProjCoordinate src, ProjCoordinate tgt) throws P
static final Info INFO = new Info(0, TIMESTAMP, 0, 0);
- static final Node NODE_0 = new Node(0, INFO, ImmutableMap.of(), 0, 0);
+ static final Node NODE_0 = new Node(0L, INFO, ImmutableMap.of(), 0d, 0d);
- static final Node NODE_1 = new Node(1, INFO, ImmutableMap.of(), 0, 4);
+ static final Node NODE_1 = new Node(1L, INFO, ImmutableMap.of(), 0d, 4d);
- static final Node NODE_2 = new Node(2, INFO, ImmutableMap.of(), 4, 4);
+ static final Node NODE_2 = new Node(2L, INFO, ImmutableMap.of(), 4d, 4d);
- static final Node NODE_3 = new Node(3, INFO, ImmutableMap.of(), 4, 0);
+ static final Node NODE_3 = new Node(3L, INFO, ImmutableMap.of(), 4d, 0d);
- static final Node NODE_4 = new Node(4, INFO, ImmutableMap.of(), 1, 1);
+ static final Node NODE_4 = new Node(4L, INFO, ImmutableMap.of(), 1d, 1d);
- static final Node NODE_5 = new Node(5, INFO, ImmutableMap.of(), 1, 2);
+ static final Node NODE_5 = new Node(5L, INFO, ImmutableMap.of(), 1d, 2d);
- static final Node NODE_6 = new Node(6, INFO, ImmutableMap.of(), 2, 2);
+ static final Node NODE_6 = new Node(6L, INFO, ImmutableMap.of(), 2d, 2d);
- static final Node NODE_7 = new Node(7, INFO, ImmutableMap.of(), 2, 1);
+ static final Node NODE_7 = new Node(7L, INFO, ImmutableMap.of(), 2d, 1d);
- static final Node NODE_8 = new Node(8, INFO, ImmutableMap.of(), 4, 1);
+ static final Node NODE_8 = new Node(8L, INFO, ImmutableMap.of(), 4d, 1d);
- static final Node NODE_9 = new Node(9, INFO, ImmutableMap.of(), 4, 1);
+ static final Node NODE_9 = new Node(9L, INFO, ImmutableMap.of(), 4d, 1d);
- static final Node NODE_10 = new Node(10, INFO, ImmutableMap.of(), 5, 2);
+ static final Node NODE_10 = new Node(10L, INFO, ImmutableMap.of(), 5d, 2d);
- static final Node NODE_11 = new Node(11, INFO, ImmutableMap.of(), 5, 1);
+ static final Node NODE_11 = new Node(11L, INFO, ImmutableMap.of(), 5d, 1d);
- static final Node NODE_12 = new Node(12, INFO, ImmutableMap.of(), 2, 1);
+ static final Node NODE_12 = new Node(12L, INFO, ImmutableMap.of(), 2d, 1d);
- static final Node NODE_13 = new Node(13, INFO, ImmutableMap.of(), 2, 2);
+ static final Node NODE_13 = new Node(13L, INFO, ImmutableMap.of(), 2d, 2d);
- static final Node NODE_14 = new Node(14, INFO, ImmutableMap.of(), 3, 3);
+ static final Node NODE_14 = new Node(14L, INFO, ImmutableMap.of(), 3d, 3d);
- static final Node NODE_15 = new Node(15, INFO, ImmutableMap.of(), 3, 1);
+ static final Node NODE_15 = new Node(15L, INFO, ImmutableMap.of(), 3d, 1d);
static final DataMap COORDINATE_CACHE = new MockDataMap(Arrays
.asList(NODE_0, NODE_1, NODE_2, NODE_3, NODE_4, NODE_5, NODE_6, NODE_7, NODE_8, NODE_9,
NODE_10, NODE_11, NODE_12, NODE_13, NODE_14, NODE_15)
.stream()
- .collect(Collectors.toMap(n -> n.id(), n -> new Coordinate(n.getLon(), n.getLat()))));
+ .collect(Collectors.toMap(n -> n.getId(), n -> new Coordinate(n.getLon(), n.getLat()))));
- static final Way WAY_0 = new Way(0, INFO, ImmutableMap.of(), ImmutableList.of());
+ static final Way WAY_0 = new Way(0L, INFO, ImmutableMap.of(), ImmutableList.of());
- static final Way WAY_1 = new Way(1, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l));
+ static final Way WAY_1 = new Way(1L, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l));
static final Way WAY_2 =
- new Way(2, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l, 0l));
+ new Way(2L, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l, 0l));
static final Way WAY_3 =
- new Way(3, INFO, ImmutableMap.of(), ImmutableList.of(8l, 9l, 10l, 11l, 8l));
+ new Way(3L, INFO, ImmutableMap.of(), ImmutableList.of(8l, 9l, 10l, 11l, 8l));
static final Way WAY_4 =
- new Way(4, INFO, ImmutableMap.of(), ImmutableList.of(4l, 5l, 6l, 7l, 4l));
+ new Way(4L, INFO, ImmutableMap.of(), ImmutableList.of(4l, 5l, 6l, 7l, 4l));
static final Way WAY_5 =
- new Way(5, INFO, ImmutableMap.of(), ImmutableList.of(12l, 13l, 14l, 15l, 12l));
+ new Way(5L, INFO, ImmutableMap.of(), ImmutableList.of(12l, 13l, 14l, 15l, 12l));
static final DataMap> REFERENCE_CACHE =
new MockDataMap(Arrays.asList(WAY_0, WAY_1, WAY_2, WAY_3, WAY_4, WAY_5).stream()
- .collect(Collectors.toMap(w -> w.id(), w -> w.getNodes())));
+ .collect(Collectors.toMap(w -> w.getId(), w -> w.getNodes())));
- static final Relation RELATION_0 = new Relation(0, INFO, ImmutableMap.of(), Arrays.asList());
+ static final Relation RELATION_0 = new Relation(0L, INFO, ImmutableMap.of(), Arrays.asList());
static final Relation RELATION_1 =
- new Relation(1, INFO, ImmutableMap.of("type", "multipolygon"), Arrays.asList());
+ new Relation(1L, INFO, ImmutableMap.of("type", "multipolygon"), Arrays.asList());
- static final Relation RELATION_2 = new Relation(2, INFO, ImmutableMap.of("type", "multipolygon"),
+ static final Relation RELATION_2 = new Relation(2L, INFO, ImmutableMap.of("type", "multipolygon"),
Arrays.asList(new Member(2l, MemberType.WAY, "outer")));
static final Relation RELATION_3 =
- new Relation(3, INFO, ImmutableMap.of("type", "multipolygon"), Arrays.asList(
+ new Relation(3L, INFO, ImmutableMap.of("type", "multipolygon"), Arrays.asList(
new Member(2l, MemberType.WAY, "outer"), new Member(3l, MemberType.WAY, "inner")));
- static final Relation RELATION_4 = new Relation(4, INFO, ImmutableMap.of("type", "multipolygon"),
+ static final Relation RELATION_4 = new Relation(4L, INFO, ImmutableMap.of("type", "multipolygon"),
Arrays.asList(new Member(2l, MemberType.WAY, "outer"),
new Member(3l, MemberType.WAY, "inner"), new Member(4l, MemberType.WAY, "outer")));
- static final Relation RELATION_5 = new Relation(5, INFO, ImmutableMap.of("type", "multipolygon"),
+ static final Relation RELATION_5 = new Relation(5L, INFO, ImmutableMap.of("type", "multipolygon"),
Arrays.asList(new Member(2l, MemberType.WAY, "outer"),
new Member(4l, MemberType.WAY, "inner"), new Member(5l, MemberType.WAY, "inner")));
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
index 92602a46e..6d9e29f8f 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/geometry/RelationGeometryBuilderTest.java
@@ -44,10 +44,10 @@ Geometry handleRelation(String file) throws IOException {
List entities = new XmlEntityReader().stream(input).toList();
DataMap coordinateMap = new MockDataMap<>(
entities.stream().filter(e -> e instanceof Node).map(e -> (Node) e).collect(
- Collectors.toMap(n -> n.id(), n -> new Coordinate(n.getLon(), n.getLat()))));
+ Collectors.toMap(n -> n.getId(), n -> new Coordinate(n.getLon(), n.getLat()))));
DataMap> referenceMap =
new MockDataMap<>(entities.stream().filter(e -> e instanceof Way).map(e -> (Way) e)
- .collect(Collectors.toMap(w -> w.id(), w -> w.getNodes())));
+ .collect(Collectors.toMap(w -> w.getId(), w -> w.getNodes())));
Relation relation = entities.stream().filter(e -> e instanceof Relation).map(e -> (Relation) e)
.findFirst().get();
new RelationGeometryBuilder(coordinateMap, referenceMap).accept(relation);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepositoryTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepositoryTest.java
index 4bde161e9..4e7eb31ee 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepositoryTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepositoryTest.java
@@ -46,7 +46,7 @@ void beforeEach() throws SQLException, IOException {
@Tag("integration")
void insert() throws RepositoryException {
nodeRepository.put(Constants.NODE_0);
- assertEquals(Constants.NODE_0, nodeRepository.get(Constants.NODE_0.id()));
+ assertEquals(Constants.NODE_0, nodeRepository.get(Constants.NODE_0.getId()));
}
@Test
@@ -55,15 +55,15 @@ void insertAll() throws RepositoryException {
List nodes = Arrays.asList(Constants.NODE_0, Constants.NODE_1, Constants.NODE_2);
nodeRepository.put(nodes);
assertIterableEquals(nodes,
- nodeRepository.get(nodes.stream().map(e -> e.id()).collect(Collectors.toList())));
+ nodeRepository.get(nodes.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@Tag("integration")
void delete() throws RepositoryException {
nodeRepository.put(Constants.NODE_0);
- nodeRepository.delete(Constants.NODE_0.id());
- assertNull(nodeRepository.get(Constants.NODE_0.id()));
+ nodeRepository.delete(Constants.NODE_0.getId());
+ assertNull(nodeRepository.get(Constants.NODE_0.getId()));
}
@Test
@@ -71,9 +71,9 @@ void delete() throws RepositoryException {
void deleteAll() throws RepositoryException {
List nodes = Arrays.asList(Constants.NODE_0, Constants.NODE_1, Constants.NODE_2);
nodeRepository.put(nodes);
- nodeRepository.delete(nodes.stream().map(e -> e.id()).collect(Collectors.toList()));
+ nodeRepository.delete(nodes.stream().map(e -> e.getId()).collect(Collectors.toList()));
assertIterableEquals(Arrays.asList(null, null, null),
- nodeRepository.get(nodes.stream().map(e -> e.id()).collect(Collectors.toList())));
+ nodeRepository.get(nodes.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@@ -82,6 +82,6 @@ void copy() throws RepositoryException {
List nodes = Arrays.asList(Constants.NODE_0, Constants.NODE_1, Constants.NODE_2);
nodeRepository.copy(nodes);
assertIterableEquals(nodes,
- nodeRepository.get(nodes.stream().map(e -> e.id()).collect(Collectors.toList())));
+ nodeRepository.get(nodes.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepositoryTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepositoryTest.java
index 0256c3042..1f3ed0293 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepositoryTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepositoryTest.java
@@ -46,7 +46,7 @@ void init() throws SQLException, IOException {
@Tag("integration")
void insert() throws RepositoryException {
relationRepository.put(Constants.RELATION_2);
- assertEquals(Constants.RELATION_2, relationRepository.get(Constants.RELATION_2.id()));
+ assertEquals(Constants.RELATION_2, relationRepository.get(Constants.RELATION_2.getId()));
}
@Test
@@ -56,15 +56,15 @@ void insertAll() throws RepositoryException {
Arrays.asList(Constants.RELATION_2, Constants.RELATION_3, Constants.RELATION_4);
relationRepository.put(relations);
assertIterableEquals(relations, relationRepository
- .get(relations.stream().map(e -> e.id()).collect(Collectors.toList())));
+ .get(relations.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@Tag("integration")
void delete() throws RepositoryException {
relationRepository.put(Constants.RELATION_2);
- relationRepository.delete(Constants.RELATION_2.id());
- assertNull(relationRepository.get(Constants.RELATION_2.id()));
+ relationRepository.delete(Constants.RELATION_2.getId());
+ assertNull(relationRepository.get(Constants.RELATION_2.getId()));
}
@Test
@@ -73,9 +73,9 @@ void deleteAll() throws RepositoryException {
List relations =
Arrays.asList(Constants.RELATION_2, Constants.RELATION_3, Constants.RELATION_4);
relationRepository.put(relations);
- relationRepository.delete(relations.stream().map(e -> e.id()).collect(Collectors.toList()));
+ relationRepository.delete(relations.stream().map(e -> e.getId()).collect(Collectors.toList()));
assertIterableEquals(Arrays.asList(null, null, null), relationRepository
- .get(relations.stream().map(e -> e.id()).collect(Collectors.toList())));
+ .get(relations.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@@ -85,6 +85,6 @@ void copy() throws RepositoryException {
Arrays.asList(Constants.RELATION_2, Constants.RELATION_3, Constants.RELATION_4);
relationRepository.copy(relations);
assertIterableEquals(relations, relationRepository
- .get(relations.stream().map(e -> e.id()).collect(Collectors.toList())));
+ .get(relations.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepositoryTest.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepositoryTest.java
index 010327ab9..5bcb3a42a 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepositoryTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepositoryTest.java
@@ -46,7 +46,7 @@ void init() throws SQLException, IOException {
@Tag("integration")
void insert() throws RepositoryException {
wayRepository.put(Constants.WAY_0);
- assertEquals(Constants.WAY_0, wayRepository.get(Constants.WAY_0.id()));
+ assertEquals(Constants.WAY_0, wayRepository.get(Constants.WAY_0.getId()));
}
@Test
@@ -55,15 +55,15 @@ void insertAll() throws RepositoryException {
List ways = Arrays.asList(Constants.WAY_0, Constants.WAY_1, Constants.WAY_2);
wayRepository.put(ways);
assertIterableEquals(ways,
- wayRepository.get(ways.stream().map(e -> e.id()).collect(Collectors.toList())));
+ wayRepository.get(ways.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@Tag("integration")
void delete() throws RepositoryException {
wayRepository.put(Constants.WAY_0);
- wayRepository.delete(Constants.WAY_0.id());
- assertNull(wayRepository.get(Constants.WAY_0.id()));
+ wayRepository.delete(Constants.WAY_0.getId());
+ assertNull(wayRepository.get(Constants.WAY_0.getId()));
}
@Test
@@ -71,9 +71,9 @@ void delete() throws RepositoryException {
void deleteAll() throws RepositoryException {
List ways = Arrays.asList(Constants.WAY_0, Constants.WAY_1, Constants.WAY_2);
wayRepository.put(ways);
- wayRepository.delete(ways.stream().map(e -> e.id()).collect(Collectors.toList()));
+ wayRepository.delete(ways.stream().map(e -> e.getId()).collect(Collectors.toList()));
assertIterableEquals(Arrays.asList(null, null, null),
- wayRepository.get(ways.stream().map(e -> e.id()).collect(Collectors.toList())));
+ wayRepository.get(ways.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
@Test
@@ -82,6 +82,6 @@ void copy() throws RepositoryException {
List ways = Arrays.asList(Constants.WAY_0, Constants.WAY_1, Constants.WAY_2);
wayRepository.copy(ways);
assertIterableEquals(ways,
- wayRepository.get(ways.stream().map(e -> e.id()).collect(Collectors.toList())));
+ wayRepository.get(ways.stream().map(e -> e.getId()).collect(Collectors.toList())));
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/Constants.java b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/Constants.java
index de8cd50da..c61600436 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/Constants.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/openstreetmap/repository/Constants.java
@@ -42,80 +42,80 @@ public class Constants {
public static final Info INFO = new Info(0, TIMESTAMP, 0, 0);
- public static final Header HEADER_0 = new Header(1l, TIMESTAMP, "", "", "");
+ public static final Header HEADER_0 = new Header(1L, TIMESTAMP, "", "", "");
- public static final Header HEADER_1 = new Header(2l, TIMESTAMP, "", "", "");
+ public static final Header HEADER_1 = new Header(2L, TIMESTAMP, "", "", "");
- public static final Header HEADER_2 = new Header(3l, TIMESTAMP, "", "", "");
+ public static final Header HEADER_2 = new Header(3L, TIMESTAMP, "", "", "");
- public static final Node NODE_0 = new Node(0, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_0 = new Node(0L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(0, 0)));
- public static final Node NODE_1 = new Node(1, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_1 = new Node(1L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(0, 3)));
- public static final Node NODE_2 = new Node(2, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_2 = new Node(2L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(3, 3)));
- public static final Node NODE_3 = new Node(3, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_3 = new Node(3L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(3, 0)));
- public static final Node NODE_4 = new Node(4, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_4 = new Node(4L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)));
- public static final Node NODE_5 = new Node(5, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_5 = new Node(5L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(1, 2)));
- public static final Node NODE_6 = new Node(6, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_6 = new Node(6L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(2, 2)));
- public static final Node NODE_7 = new Node(7, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_7 = new Node(7L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(2, 1)));
- public static final Node NODE_8 = new Node(8, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_8 = new Node(8L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(4, 1)));
- public static final Node NODE_9 = new Node(9, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_9 = new Node(9L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(4, 2)));
- public static final Node NODE_10 = new Node(10, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_10 = new Node(10L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(5, 2)));
- public static final Node NODE_11 = new Node(11, INFO, ImmutableMap.of(), 0, 0,
+ public static final Node NODE_11 = new Node(11L, INFO, ImmutableMap.of(), 0d, 0d,
GEOMETRY_FACTORY.createPoint(new Coordinate(5, 1)));
- public static final Way WAY_0 = new Way(0, INFO, ImmutableMap.of(), ImmutableList.of(), null);
+ public static final Way WAY_0 = new Way(0L, INFO, ImmutableMap.of(), ImmutableList.of(), null);
public static final Way WAY_1 =
- new Way(1, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l), null);
+ new Way(1L, INFO, ImmutableMap.of(), ImmutableList.of(0L, 1L, 2L, 3L), null);
public static final Way WAY_2 =
- new Way(2, INFO, ImmutableMap.of(), ImmutableList.of(0l, 1l, 2l, 3l, 0l), null);
+ new Way(2L, INFO, ImmutableMap.of(), ImmutableList.of(0L, 1L, 2L, 3L, 0L), null);
public static final Way WAY_3 =
- new Way(3, INFO, ImmutableMap.of(), ImmutableList.of(8l, 9l, 10l, 11l, 8l), null);
+ new Way(3L, INFO, ImmutableMap.of(), ImmutableList.of(8L, 9L, 10L, 11L, 8L), null);
public static final Way WAY_4 =
- new Way(4, INFO, ImmutableMap.of(), ImmutableList.of(4l, 5l, 6l, 7l, 4l), null);
+ new Way(4L, INFO, ImmutableMap.of(), ImmutableList.of(4L, 5L, 6L, 7L, 4L), null);
public static final Relation RELATION_0 =
- new Relation(0, INFO, ImmutableMap.of(), ImmutableList.of(), null);
+ new Relation(0L, INFO, ImmutableMap.of(), ImmutableList.of(), null);
public static final Relation RELATION_1 =
- new Relation(1, INFO, ImmutableMap.of("type", "multipolygon"), ImmutableList.of(), null);
+ new Relation(1L, INFO, ImmutableMap.of("type", "multipolygon"), ImmutableList.of(), null);
public static final Relation RELATION_2 =
- new Relation(2, INFO, ImmutableMap.of("type", "multipolygon"),
+ new Relation(2L, INFO, ImmutableMap.of("type", "multipolygon"),
ImmutableList.of(new Member(2, MemberType.WAY, "outer")), null);
public static final Relation RELATION_3 =
new Relation(
- 3, INFO, ImmutableMap.of("type", "multipolygon"), ImmutableList
+ 3L, INFO, ImmutableMap.of("type", "multipolygon"), ImmutableList
.of(new Member(2, MemberType.WAY, "outer"), new Member(3, MemberType.WAY, "inner")),
null);
public static final Relation RELATION_4 =
- new Relation(4, INFO, ImmutableMap.of("type", "multipolygon"),
+ new Relation(4L, INFO, ImmutableMap.of("type", "multipolygon"),
ImmutableList.of(new Member(2, MemberType.WAY, "outer"),
new Member(3, MemberType.WAY, "inner"), new Member(4, MemberType.WAY, "outer")),
null);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java
index 3446d5ee7..58094cb1b 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java
@@ -55,6 +55,16 @@ public class TestFiles {
public static final Path STYLE_JS = resolve("style.js");
+ public static final Path FILE_BZ2 = resolve("archives/file.bz2");
+
+ public static final Path FILE_GZ = resolve("archives/file.gz");
+
+ public static final Path FILE_TAR_BZ2 = resolve("archives/file.tar.bz2");
+
+ public static final Path FILE_TAR_GZ = resolve("archives/file.tar.gz");
+
+ public static final Path FILE_ZIP = resolve("archives/file.zip");
+
public static Path resolve(String resource) {
Path cwd = Path.of("").toAbsolutePath();
Path pathFromRoot = Path.of("baremaps-core", "src", "test", "resources", resource);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
index a41bab115..0bfd1daba 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java
@@ -24,7 +24,7 @@
import java.nio.file.Paths;
import java.util.List;
import org.apache.baremaps.workflow.tasks.DownloadUrl;
-import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap;
+import org.apache.baremaps.workflow.tasks.ImportOsmPbf;
import org.junit.Test;
public class ObjectMapperTest {
@@ -41,16 +41,18 @@ public void test() throws IOException {
"https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf",
Paths.get("liechtenstein-latest.osm.pbf")))),
new Step("import", List.of("download"),
- List.of(new ImportOpenStreetMap(Paths.get("liechtenstein-latest.osm.pbf"),
+ List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"),
+ null,
+ true,
"jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
- 3857)))));
+ 3857, true)))));
var json = mapper.writeValueAsString(workflow1);
assertTrue(json.contains(DownloadUrl.class.getSimpleName()));
- assertTrue(json.contains(ImportOpenStreetMap.class.getSimpleName()));
+ assertTrue(json.contains(ImportOsmPbf.class.getSimpleName()));
// deserialize the workflow
var workflow2 = mapper.readValue(json, Workflow.class);
assertTrue(workflow2.getSteps().get(0).getTasks().get(0) instanceof DownloadUrl);
- assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOpenStreetMap);
+ assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOsmPbf);
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
index 888201670..4fb5b32de 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java
@@ -24,7 +24,7 @@
import org.apache.baremaps.testing.PostgresContainerTest;
import org.apache.baremaps.workflow.tasks.DownloadUrl;
import org.apache.baremaps.workflow.tasks.ImportGeoPackage;
-import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap;
+import org.apache.baremaps.workflow.tasks.ImportOsmPbf;
import org.apache.baremaps.workflow.tasks.ImportShapefile;
import org.apache.baremaps.workflow.tasks.UnzipFile;
import org.junit.jupiter.api.Disabled;
@@ -103,8 +103,9 @@ void execute() {
List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf",
Paths.get("downloads/liechtenstein.osm.pbf")))),
new Step("import-osmpbf", List.of("fetch-osmpbf"),
- List.of(new ImportOpenStreetMap(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(),
- 3857))),
+ List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), null, true,
+ jdbcUrl(),
+ 3857, true))),
new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl(
"https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip",
Paths.get("downloads/simplified-water-polygons-split-3857.zip")))),
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java
new file mode 100644
index 000000000..19f0f4ade
--- /dev/null
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.workflow.tasks;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import org.apache.baremaps.testing.TestFiles;
+import org.junit.jupiter.api.Test;
+
+class DecompressFileTest {
+
+ @Test
+ void decompressBzip2() throws IOException {
+ var source = TestFiles.FILE_BZ2;
+ var target = Files.createTempFile("baremaps", ".txt");
+ DecompressFile.decompressBzip2(source, target);
+ assertTrue(Files.readString(target).contains("test"));
+ }
+
+ @Test
+ void decompressGzip() throws IOException {
+ var source = TestFiles.FILE_GZ;
+ var target = Files.createTempFile("baremaps", ".txt");
+ DecompressFile.decompressGzip(source, target);
+ assertTrue(Files.readString(target).contains("test"));
+ }
+
+ @Test
+ void decompressTarGz() throws IOException {
+ var source = TestFiles.FILE_TAR_GZ;
+ var target = Files.createTempDirectory("baremaps");
+ DecompressFile.decompressTarGz(source, target);
+ assertTrue(Files.readString(target.resolve("file.txt")).contains("test"));
+ }
+
+ @Test
+ void decompressTarBz2() throws IOException {
+ var source = TestFiles.FILE_TAR_BZ2;
+ var target = Files.createTempDirectory("baremaps");
+ DecompressFile.decompressTarBz2(source, target);
+ assertTrue(Files.readString(target.resolve("file.txt")).contains("test"));
+ }
+
+ @Test
+ void decompressZip() throws IOException {
+ var source = TestFiles.FILE_ZIP;
+ var target = Files.createTempDirectory("baremaps");
+ DecompressFile.decompressZip(source, target);
+ assertTrue(Files.readString(target.resolve("file.txt")).contains("test"));
+ }
+}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java
index 3fc552aa8..780a4c016 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java
@@ -30,7 +30,7 @@ class ExecuteCommandTest {
@Test
@Disabled
void execute() throws Exception {
- var path = Paths.get("test.txt").toAbsolutePath();
+ var path = Paths.get("file.txt").toAbsolutePath();
new ExecuteCommand(String.format("echo test > %s", path)).execute(new WorkflowContext());
assertTrue(Files.exists(path));
assertTrue(Files.readString(path).contains("test"));
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
index 2ebbac710..857969896 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java
@@ -57,7 +57,7 @@ void monaco() throws Exception {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap,
+ ImportOsmPbf.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap,
referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857);
assertEquals(3047l, headerRepository.selectLatest().getReplicationSequenceNumber());
@@ -75,7 +75,7 @@ void monaco() throws Exception {
while (replicationSequenceNumber < 3075) {
new DiffService(coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository,
relationRepository, 3857, 14).call();
- UpdateOpenStreetMap.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
+ UpdateOsmDatabase.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository, relationRepository, 3857);
long nextReplicationSequenceNumber =
headerRepository.selectLatest().getReplicationSequenceNumber();
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
similarity index 91%
rename from baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java
rename to baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
index 6a864a59d..9684c6df1 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java
@@ -25,7 +25,7 @@
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
-class ImportOpenStreetMapTest extends PostgresContainerTest {
+class ImportOsmPbfTest extends PostgresContainerTest {
@Test
@Tag("integration")
@@ -33,7 +33,7 @@ void execute() throws Exception {
var file = TestFiles.resolve("data.osm.pbf");
var jdbcUrl = jdbcUrl();
var srid = 3857;
- var task = new ImportOpenStreetMap(file, jdbcUrl, srid);
+ var task = new ImportOsmPbf(file, null, true, jdbcUrl, srid, true);
task.execute(new WorkflowContext());
}
}
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
index f032fd19b..60e517ab2 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java
@@ -61,7 +61,7 @@ void data() throws Exception {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository,
+ ImportOsmPbf.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository,
nodeRepository, wayRepository, relationRepository, 3857);
headerRepository.put(new Header(0l, LocalDateTime.of(2020, 1, 1, 0, 0, 0, 0),
@@ -94,7 +94,7 @@ void data() throws Exception {
assertNotNull(way);
// Update the database
- UpdateOpenStreetMap.execute(new PostgresCoordinateMap(dataSource()),
+ UpdateOsmDatabase.execute(new PostgresCoordinateMap(dataSource()),
new PostgresReferenceMap(dataSource()), headerRepository, nodeRepository, wayRepository,
relationRepository, 3857);
diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
index e41d631bb..dfc8585c4 100644
--- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
+++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java
@@ -58,7 +58,7 @@ void liechtenstein() throws Exception {
new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory()));
// Import data
- ImportOpenStreetMap.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap,
+ ImportOsmPbf.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap,
headerRepository,
nodeRepository, wayRepository, relationRepository, 3857);
@@ -75,7 +75,7 @@ void liechtenstein() throws Exception {
wayRepository, relationRepository, 3857, 14).call().size());
// Update the database
- UpdateOpenStreetMap.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
+ UpdateOsmDatabase.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository,
relationRepository, 3857);
assertEquals(2435l, headerRepository.selectLatest().getReplicationSequenceNumber());
@@ -83,7 +83,7 @@ void liechtenstein() throws Exception {
assertEquals(2, new DiffService(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository, relationRepository, 3857, 14).call().size());
- UpdateOpenStreetMap.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
+ UpdateOsmDatabase.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository,
relationRepository, 3857);
assertEquals(2436l, headerRepository.selectLatest().getReplicationSequenceNumber());
@@ -91,7 +91,7 @@ void liechtenstein() throws Exception {
assertEquals(0, new DiffService(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository, relationRepository, 3857, 14).call().size());
- UpdateOpenStreetMap.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
+ UpdateOsmDatabase.execute(coordinateMap, referenceMap, headerRepository, nodeRepository,
wayRepository,
relationRepository, 3857);
assertEquals(2437l, headerRepository.selectLatest().getReplicationSequenceNumber());
diff --git a/baremaps-core/src/test/resources/archives/file.bz2 b/baremaps-core/src/test/resources/archives/file.bz2
new file mode 100644
index 000000000..4250272de
Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.bz2 differ
diff --git a/baremaps-core/src/test/resources/archives/file.gz b/baremaps-core/src/test/resources/archives/file.gz
new file mode 100644
index 000000000..55a70e352
Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.gz differ
diff --git a/baremaps-core/src/test/resources/archives/file.tar.bz2 b/baremaps-core/src/test/resources/archives/file.tar.bz2
new file mode 100644
index 000000000..f5d017cbe
Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.tar.bz2 differ
diff --git a/baremaps-core/src/test/resources/archives/file.tar.gz b/baremaps-core/src/test/resources/archives/file.tar.gz
new file mode 100644
index 000000000..77812f522
Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.tar.gz differ
diff --git a/baremaps-core/src/test/resources/archives/file.zip b/baremaps-core/src/test/resources/archives/file.zip
new file mode 100644
index 000000000..b1011c290
Binary files /dev/null and b/baremaps-core/src/test/resources/archives/file.zip differ
diff --git a/basemap/config.js b/basemap/config.js
index 9b69b9894..8d045a0d8 100644
--- a/basemap/config.js
+++ b/basemap/config.js
@@ -21,4 +21,4 @@ export default {
"center": [6.6323, 46.5197],
"bounds": [6.02260949059, 45.7769477403, 10.4427014502, 47.8308275417],
"zoom": 14,
-}
+}
\ No newline at end of file
diff --git a/basemap/layers/highway/prepare.sql b/basemap/layers/highway/prepare.sql
index ccc19b661..dff5eb1c6 100644
--- a/basemap/layers/highway/prepare.sql
+++ b/basemap/layers/highway/prepare.sql
@@ -23,7 +23,7 @@ WITH
tags -> 'construction' AS construction,
geom AS geom
FROM osm_linestring
- WHERE tags ->> 'highway' IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'unclassified', 'residential', 'construction')
+ WHERE tags ->> 'highway' IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'unclassified', 'residential', 'construction')
),
-- Cluster the linestrings by highway type
clustered AS (
diff --git a/basemap/layers/linestring/prepare.sql b/basemap/layers/linestring/prepare.sql
index b9bc48a99..e86180d04 100644
--- a/basemap/layers/linestring/prepare.sql
+++ b/basemap/layers/linestring/prepare.sql
@@ -12,10 +12,10 @@
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
-DROP TABLE IF EXISTS osm_linestring CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS osm_linestring CASCADE;
CREATE MATERIALIZED VIEW osm_linestring AS
-SELECT id, tags, geom
+SELECT id, tags, geom, changeset
FROM osm_ways
LEFT JOIN osm_member ON id = member_ref
WHERE ST_GeometryType(osm_ways.geom) = 'ST_LineString'
diff --git a/basemap/tileset.js b/basemap/tileset.js
index 4ae8e48f3..f680b1832 100644
--- a/basemap/tileset.js
+++ b/basemap/tileset.js
@@ -44,7 +44,7 @@ export default {
"tiles": [
`${config.host}/tiles/{z}/{x}/{y}.mvt`
],
- attribution: '© OpenStreetMap © geoBoundaries',
+ attribution: '© OpenStreetMap contributors, Microsoft, Esri Community Maps contributors',
database: config.database,
"vector_layers": [
aerialway,
@@ -59,11 +59,11 @@ export default {
leisure,
man_made,
natural,
- ocean,
+ //ocean,
point,
power,
railway,
route,
- waterway,
+ //waterway,
]
}
diff --git a/basemap/workflow.js b/basemap/workflow.js
index 2b407c652..47ae09a7b 100644
--- a/basemap/workflow.js
+++ b/basemap/workflow.js
@@ -48,7 +48,7 @@ export default {
]
},
{
- "id": "water-polygons",
+ "id": "openstreetmap-water-polygons",
"needs": [],
"tasks": [
{
@@ -71,7 +71,7 @@ export default {
]
},
{
- "id": "simplified-water-polygons",
+ "id": "openstreetmap-simplified-water-polygons",
"needs": [],
"tasks": [
{
@@ -96,8 +96,8 @@ export default {
{
"id": "openstreetmap-ocean",
"needs": [
- "water-polygons",
- "simplified-water-polygons",
+ "openstreetmap-water-polygons",
+ "openstreetmap-simplified-water-polygons",
],
"tasks": [
{
@@ -127,10 +127,11 @@ export default {
"path": "data/data.osm.pbf"
},
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "data/data.osm.pbf",
"database": config.database,
- "databaseSrid": 3857
+ "databaseSrid": 3857,
+ "replaceExisting": true,
},
]
},
diff --git a/daylight/config.js b/daylight/config.js
new file mode 100644
index 000000000..124861f55
--- /dev/null
+++ b/daylight/config.js
@@ -0,0 +1,23 @@
+/**
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ **/
+export default {
+ "host": "http://localhost:9000",
+ "database": "jdbc:postgresql://localhost:5432/daylight?&user=daylight&password=daylight",
+ "center": [0, 0],
+ "bounds": [-180, -85.0511, 180, 85.0511],
+ "zoom": 3,
+}
diff --git a/daylight/layers/coastline/prepare.sql b/daylight/layers/coastline/prepare.sql
new file mode 100644
index 000000000..c0f2c0eeb
--- /dev/null
+++ b/daylight/layers/coastline/prepare.sql
@@ -0,0 +1,23 @@
+-- Licensed under the Apache License, Version 2.0 (the License); you may not use this file except
+-- in compliance with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software distributed under the License
+-- is distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+-- or implied. See the License for the specific language governing permissions and limitations under
+-- the License.
+
+DROP MATERIALIZED VIEW IF EXISTS osm_coastline CASCADE;
+
+DROP MATERIALIZED VIEW IF EXISTS osm_coastline_simplified CASCADE;
+
+CREATE MATERIALIZED VIEW osm_coastline AS
+SELECT row_number() OVER () as id, '{"ocean":"water"}'::jsonb as tags, st_setsrid(geometry, 3857) AS geom FROM water_polygons_shp;
+
+CREATE MATERIALIZED VIEW osm_coastline_simplified AS
+SELECT id, tags, st_simplifypreservetopology(geom, 300) as geom FROM osm_coastline WHERE ST_Area(geom) > 300000;
+
+CREATE INDEX IF NOT EXISTS osm_coastline_index ON osm_coastline USING SPGIST (geom);
+
+CREATE INDEX IF NOT EXISTS osm_coastline_simplified_index ON osm_coastline_simplified USING SPGIST (geom);
diff --git a/daylight/layers/coastline/style.js b/daylight/layers/coastline/style.js
new file mode 100644
index 000000000..00ebdd4b6
--- /dev/null
+++ b/daylight/layers/coastline/style.js
@@ -0,0 +1,34 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+import {withSortKeys, asLayerObject} from "../../../basemap/utils/utils.js";
+import theme from "../../../basemap/theme.js";
+
+let directives = [
+ {
+ filter: ['==', ['get', 'ocean'], 'water'],
+ 'fill-color': theme.oceanWaterFillColor,
+ 'fill-sort-key': 10,
+ },
+];
+
+export default asLayerObject(withSortKeys(directives), {
+ id: 'ocean_overlay',
+ type: 'fill',
+ source: 'baremaps',
+ 'source-layer': 'ocean',
+ layout: {
+ visibility: 'visible',
+ },
+ paint: {
+ 'fill-antialias': true,
+ },
+});
diff --git a/daylight/layers/coastline/tileset.js b/daylight/layers/coastline/tileset.js
new file mode 100644
index 000000000..cfc5da40e
--- /dev/null
+++ b/daylight/layers/coastline/tileset.js
@@ -0,0 +1,26 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+export default {
+ "id": "ocean",
+ "queries": [
+ {
+ "minzoom": 0,
+ "maxzoom": 10,
+ "sql": "SELECT id, tags, geom FROM osm_coastline_simplified"
+ },
+ {
+ "minzoom": 10,
+ "maxzoom": 20,
+ "sql": "SELECT id, tags, geom FROM osm_coastline"
+ }
+ ]
+}
diff --git a/daylight/layers/landcover/prepare.sql b/daylight/layers/landcover/prepare.sql
new file mode 100644
index 000000000..927e30400
--- /dev/null
+++ b/daylight/layers/landcover/prepare.sql
@@ -0,0 +1,17 @@
+-- Licensed under the Apache License, Version 2.0 (the License); you may not use this file except
+-- in compliance with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software distributed under the License
+-- is distributed on an AS IS BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+-- or implied. See the License for the specific language governing permissions and limitations under
+-- the License.
+
+DROP MATERIALIZED VIEW IF EXISTS esa_landcover CASCADE;
+
+CREATE MATERIALIZED VIEW esa_landcover AS
+SELECT id as id, jsonb_build_object('landcover', class) as tags, geometry as geom
+FROM low_shp;
+
+CREATE INDEX IF NOT EXISTS esa_landcover_index ON esa_landcover USING SPGIST (geom);
diff --git a/daylight/layers/landcover/style.js b/daylight/layers/landcover/style.js
new file mode 100644
index 000000000..4e7147dee
--- /dev/null
+++ b/daylight/layers/landcover/style.js
@@ -0,0 +1,56 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+import {withSortKeys, asLayerObject} from "../../../basemap/utils/utils.js";
+
+let directives = [
+ {
+ filter: ['==', ['get', 'landcover'], 'snow'],
+ 'fill-color': '#ffffff',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'crop'],
+ 'fill-color': '#f0e68c',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'urban'],
+ 'fill-color': '#808080',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'trees'],
+ 'fill-color': '#228b22',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'grass'],
+ 'fill-color': '#7cfc00',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'barren'],
+ 'fill-color': '#d2b48c',
+ },
+ {
+ filter: ['==', ['get', 'landcover'], 'shrub'],
+ 'fill-color': '#8b4513',
+ }
+];
+
+export default asLayerObject(withSortKeys(directives), {
+ id: 'landcover',
+ type: 'fill',
+ source: 'baremaps',
+ 'source-layer': 'landcover',
+ layout: {
+ visibility: 'visible',
+ },
+ paint: {
+ 'fill-antialias': false,
+ },
+});
\ No newline at end of file
diff --git a/daylight/layers/landcover/tileset.js b/daylight/layers/landcover/tileset.js
new file mode 100644
index 000000000..323f0e73e
--- /dev/null
+++ b/daylight/layers/landcover/tileset.js
@@ -0,0 +1,21 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+export default {
+ "id": "landcover",
+ "queries": [
+ {
+ "minzoom": 0,
+ "maxzoom": 10,
+ "sql": "SELECT id, tags, geom FROM esa_landcover"
+ },
+ ]
+}
diff --git a/daylight/style.js b/daylight/style.js
new file mode 100644
index 000000000..b0e1b0ccc
--- /dev/null
+++ b/daylight/style.js
@@ -0,0 +1,130 @@
+/**
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ **/
+import config from "./config.js";
+
+import background from "../basemap/layers/background/style.js";
+import aeroway_line from "../basemap/layers/aeroway/line.js";
+import aeroway_polygon from "../basemap/layers/aeroway/polygon.js";
+import amenity_background from "../basemap/layers/amenity/background.js";
+import amenity_fountain from "../basemap/layers/amenity/fountain.js";
+import amenity_overlay from "../basemap/layers/amenity/overlay.js";
+import boundary_line from "../basemap/layers/boundary/line.js";
+import landuse_background from "../basemap/layers/landuse/background.js";
+import landuse_overlay from "../basemap/layers/landuse/overlay.js";
+import natural_background from "../basemap/layers/natural/background.js";
+import natural_overlay from "../basemap/layers/natural/overlay.js";
+import natural_tree from "../basemap/layers/natural/tree.js";
+import natural_trunk from "../basemap/layers/natural/trunk.js";
+import power_background from "../basemap/layers/power/background.js";
+import power_tower from "../basemap/layers/power/tower.js";
+import power_cable from "../basemap/layers/power/cable.js";
+import leisure_background from "../basemap/layers/leisure/background.js";
+import leisure_overlay from "../basemap/layers/leisure/overlay.js";
+import railway_tunnel from "../basemap/layers/railway/tunnel.js";
+import railway_line from "../basemap/layers/railway/line.js";
+
+import highway_line from '../basemap/layers/highway/highway_line.js';
+import highway_outline from '../basemap/layers/highway/highway_outline.js';
+import highway_dash from '../basemap/layers/highway/highway_dash.js';
+import highway_tunnel_line from '../basemap/layers/highway/tunnel_line.js';
+import highway_tunnel_outline from '../basemap/layers/highway/tunnel_outline.js';
+import highway_pedestrian_area from '../basemap/layers/highway/pedestrian_area.js';
+import highway_bridge_line from '../basemap/layers/highway/bridge_line.js';
+import highway_bridge_outline from '../basemap/layers/highway/bridge_outline.js';
+import highway_construction_line from "../basemap/layers/highway/construction_line.js";
+import highway_construction_dash from "../basemap/layers/highway/construction_dash.js";
+import highway_label from '../basemap/layers/highway/highway_label.js';
+
+import route_line from "../basemap/layers/route/style.js"
+import building_shape from "../basemap/layers/building/shape.js";
+import building_number from "../basemap/layers/building/number.js";
+import man_made_bridge from "../basemap/layers/man_made/bridge.js";
+import man_made_pier_line from "../basemap/layers/man_made/pier_line.js";
+import man_made_pier_label from "../basemap/layers/man_made/pier_label.js";
+import waterway_line from "../basemap/layers/waterway/line.js"
+import waterway_label from "../basemap/layers/waterway/label.js"
+import waterway_tunnel_line from "../basemap/layers/waterway/tunnel_line.js"
+import waterway_tunnel_casing from "../basemap/layers/waterway/tunnel_casing.js"
+import icon from "../basemap/layers/point/icon.js";
+import country_label from '../basemap/layers/point/country_label.js';
+import point_label from '../basemap/layers/point/point_label.js';
+
+import coastline from './layers/coastline/style.js';
+import landcover from './layers/landcover/style.js';
+
+export default {
+ "version": 8,
+ "name": "OpenStreetMapVecto",
+ "center": config.center,
+ "zoom": config.zoom,
+ "sources": {
+ "baremaps": {
+ "type": "vector",
+ "url": `${config.host}/tiles.json`
+ }
+ },
+ "sprite": `https://baremaps.apache.org/sprites/osm/sprite`,
+ "glyphs": "https://baremaps.apache.org/fonts/{fontstack}/{range}.pbf",
+ "layers": [
+ background,
+ power_background,
+ aeroway_polygon,
+ landuse_background,
+ leisure_background,
+ amenity_background,
+ natural_background,
+ landuse_overlay,
+ natural_overlay,
+ amenity_overlay,
+ leisure_overlay,
+ coastline,
+ // landcover,
+ waterway_line,
+ waterway_tunnel_casing,
+ waterway_tunnel_line,
+ man_made_bridge,
+ amenity_fountain,
+ highway_tunnel_outline,
+ highway_tunnel_line,
+ railway_tunnel,
+ building_shape,
+ building_number,
+ highway_construction_dash,
+ highway_construction_line,
+ highway_outline,
+ highway_line,
+ highway_dash,
+ highway_pedestrian_area,
+ railway_line,
+ highway_bridge_outline,
+ highway_bridge_line,
+ highway_label,
+ aeroway_line,
+ route_line,
+ power_cable,
+ power_tower,
+ man_made_pier_line,
+ man_made_pier_label,
+ natural_tree,
+ natural_trunk,
+ boundary_line,
+ waterway_label,
+ icon,
+ point_label,
+ country_label,
+ ],
+};
\ No newline at end of file
diff --git a/daylight/tileset.js b/daylight/tileset.js
new file mode 100644
index 000000000..13a2fe358
--- /dev/null
+++ b/daylight/tileset.js
@@ -0,0 +1,67 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+import config from "./config.js";
+
+import aerialway from "../basemap/layers/aerialway/tileset.js";
+import aeroway from "../basemap/layers/aeroway/tileset.js";
+import amenity from "../basemap/layers/amenity/tileset.js";
+import attraction from "../basemap/layers/attraction/tileset.js";
+import barrier from "../basemap/layers/barrier/tileset.js";
+import boundary from "../basemap/layers/boundary/tileset.js";
+import building from "../basemap/layers/building/tileset.js";
+import highway from "../basemap/layers/highway/tileset.js";
+import natural from "../basemap/layers/natural/tileset.js";
+import leisure from "../basemap/layers/leisure/tileset.js";
+import landuse from "../basemap/layers/landuse/tileset.js";
+import railway from "../basemap/layers/railway/tileset.js";
+import route from "../basemap/layers/route/tileset.js";
+import man_made from "../basemap/layers/man_made/tileset.js";
+import power from "../basemap/layers/power/tileset.js";
+import point from "../basemap/layers/point/tileset.js";
+import waterway from "../basemap/layers/waterway/tileset.js";
+import coastline from "./layers/coastline/tileset.js";
+import landcover from "./layers/landcover/tileset.js";
+
+
+export default {
+ "tilejson": "2.2.0",
+ "center": [...config.center, config.zoom],
+ "bounds": [...config.bounds],
+ "minzoom": 0.0,
+ "maxzoom": 14.0,
+ "tiles": [
+ `${config.host}/tiles/{z}/{x}/{y}.mvt`
+ ],
+ attribution: '© OpenStreetMap © geoBoundaries',
+ database: config.database,
+ "vector_layers": [
+ aerialway,
+ aeroway,
+ amenity,
+ attraction,
+ barrier,
+ boundary,
+ building,
+ highway,
+ landuse,
+ leisure,
+ man_made,
+ natural,
+ coastline,
+ point,
+ power,
+ railway,
+ route,
+ waterway,
+ // landcover,
+ ]
+}
diff --git a/daylight/workflow.js b/daylight/workflow.js
new file mode 100644
index 000000000..3c387b642
--- /dev/null
+++ b/daylight/workflow.js
@@ -0,0 +1,430 @@
+/**
+ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ in compliance with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software distributed under the License
+ is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ or implied. See the License for the specific language governing permissions and limitations under
+ the License.
+ **/
+
+import config from "./config.js";
+
+export default {
+ "steps": [
+ {
+ "id": "daylight-data",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/planet-v1.33.osm.pbf",
+ "path": "data/data.osm.pbf"
+ },
+ {
+ "type": "ImportOsmPbf",
+ "file": "data/data.osm.pbf",
+ "cache": "cache/",
+ "cleanCache": false,
+ "database": config.database,
+ "databaseSrid": 3857,
+ "replaceExisting": true,
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/ml-buildings-v1.33.osm.pbf",
+ "path": "data/buildings.osm.pbf"
+ },
+ {
+ "type": "ImportOsmPbf",
+ "file": "data/buildings.osm.pbf",
+ "cache": "building_cache/",
+ "cleanCache": true,
+ "database": config.database,
+ "databaseSrid": 3857,
+ "replaceExisting": false,
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/fb-ml-roads-v1.33.osc.gz",
+ "path": "data/roads.osc.gz"
+ },
+ {
+ "type": "ImportOsmOsc",
+ "file": "data/roads.osc.gz",
+ "cache": "cache/",
+ "cleanCache": false,
+ "compression": "gzip",
+ "database": config.database,
+ "srid": 3857
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/admin-v1.33.osc.gz",
+ "path": "data/admin.osc.gz"
+ },
+ {
+ "type": "ImportOsmOsc",
+ "file": "data/admin.osc.gz",
+ "cache": "cache/",
+ "cleanCache": false,
+ "compression": "gzip",
+ "database": config.database,
+ "srid": 3857
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/coastlines-v1.33.tgz",
+ "path": "data/coastlines.tgz"
+ },
+ {
+ "type": "DecompressFile",
+ "compression": "targz",
+ "source": "data/coastlines.tgz",
+ "target": "data/coastlines"
+ },
+ {
+ "type": "ImportShapefile",
+ "file": "data/coastlines/water_polygons.shp",
+ "database": config.database,
+ "sourceSRID": 4326,
+ "targetSRID": 3857
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "./layers/coastline/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/preferred-localization-v1.33.tsv",
+ "path": "data/preferred-localization.tsv"
+ },
+ {
+ "type": "ImportDaylightTranslations",
+ "file": "data/preferred-localization.tsv",
+ "database": config.database,
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.33/important-features-v1.33.json",
+ "path": "data/important-features.json"
+ },
+ {
+ "type": "ImportDaylightFeatures",
+ "file": "data/important-features.json",
+ "database": config.database,
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shp",
+ "path": "data/landcover/low.shp"
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.dbf",
+ "path": "data/landcover/low.dbf"
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.prj",
+ "path": "data/landcover/low.prj"
+ },
+ {
+ "type": "DownloadUrl",
+ "url": "https://daylight-openstreetmap.s3.us-west-2.amazonaws.com/landcover/low.shx",
+ "path": "data/landcover/low.shx"
+ },
+ {
+ "type": "ImportShapefile",
+ "file": "data/landcover/low.shp",
+ "database": config.database,
+ "sourceSRID": 4326,
+ "targetSRID": 3857
+ },
+ ]
+ },
+ {
+ "id": "daylight-nodes",
+ "needs": ["daylight-data"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/queries/osm_nodes.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-ways",
+ "needs": ["daylight-data"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/queries/osm_ways.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-relations",
+ "needs": ["daylight-data"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/queries/osm_relations.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-member",
+ "needs": ["daylight-data"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/member/prepare.sql",
+ "database": config.database,
+ },
+ ]
+ },
+ {
+ "id": "daylight-point",
+ "needs": ["daylight-nodes"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/point/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/point/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/point/index.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-linestring",
+ "needs": ["daylight-member"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/linestring/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/linestring/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/linestring/index.sql",
+ "database": config.database,
+ },
+ ]
+ },
+ {
+ "id": "daylight-polygon",
+ "needs": ["daylight-member"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/polygon/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/polygon/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/polygon/index.sql",
+ "database": config.database,
+ },
+ ]
+ },
+ {
+ "id": "daylight-highway",
+ "needs": ["daylight-linestring"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/highway/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/highway/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/highway/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/highway/index.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-railway",
+ "needs": ["daylight-linestring"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/railway/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/railway/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/railway/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/railway/index.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-route",
+ "needs": ["daylight-linestring"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/route/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/route/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/route/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/route/index.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ ]
+ },
+ {
+ "id": "daylight-natural",
+ "needs": ["daylight-polygon"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/natural/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/natural/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/natural/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/natural/index.sql",
+ "database": config.database,
+ "parallel": true
+ },
+ ]
+ },
+ {
+ "id": "daylight-landuse",
+ "needs": ["daylight-polygon"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/landuse/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/landuse/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/landuse/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/landuse/index.sql",
+ "database": config.database,
+ "parallel": true
+ },
+ ]
+ },
+ {
+ "id": "daylight-waterway",
+ "needs": ["daylight-linestring"],
+ "tasks": [
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/waterway/clean.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/waterway/prepare.sql",
+ "database": config.database,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/waterway/simplify.sql",
+ "database": config.database,
+ "parallel": true,
+ },
+ {
+ "type": "ExecuteSql",
+ "file": "../basemap/layers/waterway/index.sql",
+ "database": config.database,
+ "parallel": true
+ },
+ ]
+ },
+ ]
+}
diff --git a/examples/extrusion/workflow.json b/examples/extrusion/workflow.json
index de11af58f..039ad0d1a 100644
--- a/examples/extrusion/workflow.json
+++ b/examples/extrusion/workflow.json
@@ -18,7 +18,7 @@
],
"tasks": [
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "greater-london-latest.osm.pbf",
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
"databaseSrid": 3857
diff --git a/examples/openstreetmap/workflow.json b/examples/openstreetmap/workflow.json
index 9ebd677a5..d488cb4bf 100644
--- a/examples/openstreetmap/workflow.json
+++ b/examples/openstreetmap/workflow.json
@@ -18,7 +18,7 @@
],
"tasks": [
{
- "type": "ImportOpenStreetMap",
+ "type": "ImportOsmPbf",
"file": "liechtenstein-latest.osm.pbf",
"database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps",
"databaseSrid": 3857
diff --git a/examples/wikidata/workflow.json b/examples/wikidata/workflow.json
new file mode 100644
index 000000000..77ed204cb
--- /dev/null
+++ b/examples/wikidata/workflow.json
@@ -0,0 +1,14 @@
+{
+ "steps": [
+ {
+ "id": "natural_earth_vector",
+ "needs": [],
+ "tasks": [
+ {
+ "type": "ImportWikidata",
+ "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps"
+ }
+ ]
+ }
+ ]
+}
diff --git a/pom.xml b/pom.xml
index 7bb72aff0..d87a6a756 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@ limitations under the License.
2.0.9
3.43.2.1
1.19.1
+ 0.14.6
9.4.2
9.4.2
0.8.11
@@ -530,6 +531,31 @@ limitations under the License.
org.testcontainers
testcontainers
+
+ org.wikidata.wdtk
+ wdtk-datamodel
+ ${version.lib.wdtk}
+
+
+ org.wikidata.wdtk
+ wdtk-dumpfiles
+ ${version.lib.wdtk}
+
+
+ org.wikidata.wdtk
+ wdtk-rdf
+ ${version.lib.wdtk}
+
+
+ org.wikidata.wdtk
+ wdtk-util
+ ${version.lib.wdtk}
+
+
+ org.wikidata.wdtk
+ wdtk-wikibaseapi
+ ${version.lib.wdtk}
+
org.wololo
flatgeobuf