From 26df3a07ad3255c8dcd9448c10245638cf9c3277 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sat, 26 Aug 2023 14:20:42 +0200 Subject: [PATCH 01/66] Create inital workflow for daylight --- .../org/apache/baremaps/workflow/Task.java | 1 + .../workflow/tasks/DecompressBZip2.java | 41 ++ basemap/daylight.js | 408 ++++++++++++++++++ 3 files changed, 450 insertions(+) create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java create mode 100644 basemap/daylight.js diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index c6862761d..3a252c7f6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -43,6 +43,7 @@ @JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"), @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), + @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"), @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"), @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")}) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java new file mode 100644 index 000000000..3eb95012f --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressBZip2.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import java.io.BufferedInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.apache.baremaps.workflow.WorkflowException; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + +public record DecompressBZip2(Path source, Path target) implements Task { + + @Override + public void execute(WorkflowContext context) throws Exception { + var sourcePath = source.toAbsolutePath(); + try (var zis = + new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + var targetPath = target.toAbsolutePath(); + if (!Files.exists(targetPath)) { + Files.createDirectories(targetPath.getParent()); + Files.createFile(targetPath); + } + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + throw new WorkflowException(e); + } + } +} diff --git a/basemap/daylight.js b/basemap/daylight.js new file mode 100644 index 000000000..008336f98 --- /dev/null +++ b/basemap/daylight.js @@ -0,0 +1,408 @@ +/** + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. + **/ + +let config = { + "database": "jdbc:postgresql://localhost:5432/daylight?user=daylight&password=daylight" +}; + +export default { + "steps": [ + { + "id": "openstreetmap-data", + "needs": [], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", + "path": "data/data.osm.pbf" + }, + { + "type": "ImportOpenStreetMap", + "file": "data/data.osm.pbf", + "database": config.database, + "databaseSrid": 3857 + }, + ] + }, + { + "id": "openstreetmap-roads", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2", + "path": "data/roads.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/roads.osc.bz2", + "directory": "data/roads.osc" + }, + ] + }, + { + "id": "openstreetmap-admin", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", + "path": "data/admin.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/admin.osc.bz2", + "directory": "data/admin.osc" + }, + ] + }, + { + "id": "openstreetmap-coastlines", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", + "path": "data/coastlines.osc.bz2" + }, + { + "type": "DecompressBZip2", + "file": "data/coastlines.osc.bz2", + "directory": "data/coastlines.osc" + }, + ] + }, + { + "id": "openstreetmap-preferred-localization", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", + "path": "data/preferred-localization.tsv" + }, + ] + }, + { + "id": "openstreetmap-important-features", + "needs": ["openstreetmap-data"], + "tasks": [ + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", + "path": "data/important-features.json" + }, + ] + }, + + // { + // "id": "openstreetmap-nodes", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_nodes.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-ways", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_ways.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-relations", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "queries/osm_relations.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-member", + // "needs": [ + // "openstreetmap-data" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/member/prepare.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-point", + // "needs": [ + // "openstreetmap-nodes" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/point/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/point/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/point/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-linestring", + // "needs": [ + // "openstreetmap-member" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/linestring/index.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-polygon", + // "needs": [ + // "openstreetmap-member", + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/polygon/index.sql", + // "database": config.database, + // }, + // ] + // }, + // { + // "id": "openstreetmap-highway", + // "needs": [ + // "openstreetmap-linestring" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/highway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/highway/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-railway", + // "needs": ["openstreetmap-linestring"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/railway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/railway/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-route", + // "needs": ["openstreetmap-linestring"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/route/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/route/index.sql", + // "database": config.database, + // "parallel": true, + // }, + // ] + // }, + // { + // "id": "openstreetmap-natural", + // "needs": ["openstreetmap-polygon"], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/natural/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/natural/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + // { + // "id": "openstreetmap-landuse", + // "needs": [ + // "openstreetmap-polygon" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/landuse/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + // { + // "id": "openstreetmap-waterway", + // "needs": [ + // "openstreetmap-linestring" + // ], + // "tasks": [ + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/clean.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/prepare.sql", + // "database": config.database, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/simplify.sql", + // "database": config.database, + // "parallel": true, + // }, + // { + // "type": "ExecuteSql", + // "file": "layers/waterway/index.sql", + // "database": config.database, + // "parallel": true + // }, + // ] + // }, + ] +} From 0efd709bd3c08723b9a3a064bac1bac16250f581 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sun, 27 Aug 2023 21:34:30 +0200 Subject: [PATCH 02/66] Add decompress task --- .../org/apache/baremaps/workflow/Task.java | 1 + .../workflow/tasks/DecompressFile.java | 149 ++++++++++++++++++ .../apache/baremaps/testing/TestFiles.java | 10 ++ .../workflow/tasks/DecompressFileTest.java | 63 ++++++++ .../workflow/tasks/ExecuteCommandTest.java | 2 +- .../src/test/resources/archives/file.bz2 | Bin 0 -> 40 bytes .../src/test/resources/archives/file.gz | Bin 0 -> 33 bytes .../src/test/resources/archives/file.tar.bz2 | Bin 0 -> 136 bytes .../src/test/resources/archives/file.tar.gz | Bin 0 -> 123 bytes .../src/test/resources/archives/file.zip | Bin 0 -> 170 bytes 10 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java create mode 100644 baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java create mode 100644 baremaps-core/src/test/resources/archives/file.bz2 create mode 100644 baremaps-core/src/test/resources/archives/file.gz create mode 100644 baremaps-core/src/test/resources/archives/file.tar.bz2 create mode 100644 baremaps-core/src/test/resources/archives/file.tar.gz create mode 100644 baremaps-core/src/test/resources/archives/file.zip diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 3a252c7f6..77f4336d7 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -44,6 +44,7 @@ @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), @JsonSubTypes.Type(value = UngzipFile.class, name = "UngzipFile"), @JsonSubTypes.Type(value = DecompressBZip2.class, name = "DecompressBZip2"), + @JsonSubTypes.Type(value = DecompressFile.class, name = "DecompressFile"), @JsonSubTypes.Type(value = UpdateOpenStreetMap.class, name = "UpdateOpenStreetMap"), @JsonSubTypes.Type(value = CreateGeonamesIndex.class, name = "CreateGeonamesIndex"), @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex")}) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java new file mode 100644 index 000000000..9cfb8b3bc --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/DecompressFile.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipFile; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record DecompressFile(Path source, Path target, Compression compression) implements Task { + + public enum Compression { + zip, + targz, + tarbz2, + gzip, + bzip2 + } + + private static final Logger logger = LoggerFactory.getLogger(UngzipFile.class); + + @Override + public void execute(WorkflowContext context) throws Exception { + var sourcePath = source.toAbsolutePath(); + var targetPath = target.toAbsolutePath(); + switch (compression) { + case zip: + decompressZip(sourcePath, targetPath); + break; + case targz: + decompressTarGz(sourcePath, targetPath); + break; + case tarbz2: + decompressTarBz2(sourcePath, targetPath); + break; + case gzip: + decompressGzip(sourcePath, targetPath); + break; + case bzip2: + decompressBzip2(sourcePath, targetPath); + break; + } + + } + + public static void decompressBzip2(Path sourcePath, Path targetPath) throws IOException { + try (var zis = + new BZip2CompressorInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + public static void decompressGzip(Path sourcePath, Path targetPath) throws IOException { + try (var zis = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath)))) { + Files.copy(zis, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + public static void decompressTarGz(Path sourcePath, Path targetPath) throws IOException { + try ( + GZIPInputStream gzipInputStream = + new GZIPInputStream(new BufferedInputStream(Files.newInputStream(sourcePath))); + TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + var path = targetPath.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(path); + } else { + Files.createDirectories(path.getParent()); + try (BufferedOutputStream outputStream = + new BufferedOutputStream(Files.newOutputStream(path))) { + int bytesRead; + byte[] buffer = new byte[4096]; + while ((bytesRead = tarInputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } + } + } + } + + public static void decompressTarBz2(Path sourcePath, Path targetPath) throws IOException { + try ( + BZip2CompressorInputStream gzipInputStream = new BZip2CompressorInputStream( + new BufferedInputStream(Files.newInputStream(sourcePath))); + TarArchiveInputStream tarInputStream = new TarArchiveInputStream(gzipInputStream)) { + TarArchiveEntry entry; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + var path = targetPath.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(path); + } else { + Files.createDirectories(path.getParent()); + try (BufferedOutputStream outputStream = + new BufferedOutputStream(Files.newOutputStream(path))) { + int bytesRead; + byte[] buffer = new byte[4096]; + while ((bytesRead = tarInputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } + } + } + } + } + + public static void decompressZip(Path sourcePath, Path targetPath) throws IOException { + try (var zipFile = new ZipFile(sourcePath.toFile())) { + var entries = zipFile.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + var path = targetPath.resolve(entry.getName()); + Files.createDirectories(path.getParent()); + Files.write(path, new byte[] {}, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + try (var input = new BufferedInputStream(zipFile.getInputStream(entry)); + var output = new BufferedOutputStream(new FileOutputStream(path.toFile()))) { + int nBytes = -1; + byte[] buffer = new byte[4096]; + while ((nBytes = input.read(buffer)) > 0) { + output.write(buffer, 0, nBytes); + } + } + } + } + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java index 3446d5ee7..58094cb1b 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/testing/TestFiles.java @@ -55,6 +55,16 @@ public class TestFiles { public static final Path STYLE_JS = resolve("style.js"); + public static final Path FILE_BZ2 = resolve("archives/file.bz2"); + + public static final Path FILE_GZ = resolve("archives/file.gz"); + + public static final Path FILE_TAR_BZ2 = resolve("archives/file.tar.bz2"); + + public static final Path FILE_TAR_GZ = resolve("archives/file.tar.gz"); + + public static final Path FILE_ZIP = resolve("archives/file.zip"); + public static Path resolve(String resource) { Path cwd = Path.of("").toAbsolutePath(); Path pathFromRoot = Path.of("baremaps-core", "src", "test", "resources", resource); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java new file mode 100644 index 000000000..de10032a8 --- /dev/null +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/DecompressFileTest.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import org.apache.baremaps.testing.TestFiles; +import org.junit.jupiter.api.Test; + +class DecompressFileTest { + + @Test + void decompressBzip2() throws IOException { + var source = TestFiles.FILE_BZ2; + var target = Files.createTempFile("baremaps", ".txt"); + DecompressFile.decompressBzip2(source, target); + assertTrue(Files.readString(target).contains("test")); + } + + @Test + void decompressGzip() throws IOException { + var source = TestFiles.FILE_GZ; + var target = Files.createTempFile("baremaps", ".txt"); + DecompressFile.decompressGzip(source, target); + assertTrue(Files.readString(target).contains("test")); + } + + @Test + void decompressTarGz() throws IOException { + var source = TestFiles.FILE_TAR_GZ; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressTarGz(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } + + @Test + void decompressTarBz2() throws IOException { + var source = TestFiles.FILE_TAR_BZ2; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressTarBz2(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } + + @Test + void decompressZip() throws IOException { + var source = TestFiles.FILE_ZIP; + var target = Files.createTempDirectory("baremaps"); + DecompressFile.decompressZip(source, target); + assertTrue(Files.readString(target.resolve("file.txt")).contains("test")); + } +} diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java index 3fc552aa8..780a4c016 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ExecuteCommandTest.java @@ -30,7 +30,7 @@ class ExecuteCommandTest { @Test @Disabled void execute() throws Exception { - var path = Paths.get("test.txt").toAbsolutePath(); + var path = Paths.get("file.txt").toAbsolutePath(); new ExecuteCommand(String.format("echo test > %s", path)).execute(new WorkflowContext()); assertTrue(Files.exists(path)); assertTrue(Files.readString(path).contains("test")); diff --git a/baremaps-core/src/test/resources/archives/file.bz2 b/baremaps-core/src/test/resources/archives/file.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..4250272ded4dd670bbf1fe2ba4eceb79c8d3622d GIT binary patch literal 40 wcmZ>Y%CIzaj8qGbH10mXhJk^Rv4M$!he3fsafW1zMC?hChfWUX^zAkP0Nzpyx&QzG literal 0 HcmV?d00001 diff --git a/baremaps-core/src/test/resources/archives/file.gz b/baremaps-core/src/test/resources/archives/file.gz new file mode 100644 index 0000000000000000000000000000000000000000..55a70e3528ca4edeb85ff25cccd0552c404bc6de GIT binary patch literal 33 ocmb2|=HOT{^K}XXb6RFjs$NM&34^w$t{x);PhI^D76t|e0I^~TF#rGn literal 0 HcmV?d00001 diff --git a/baremaps-core/src/test/resources/archives/file.tar.bz2 b/baremaps-core/src/test/resources/archives/file.tar.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..f5d017cbe515e4373d3f5bf852799250cfe38c9b GIT binary patch literal 136 zcmV;30C)dFT4*^jL0KkKSvJhz@c;lo`;gH91b{#Re}Di$dt_cf1ONaCAOLbR)M=rE zQKlmR&`N4#DUqh0q3Rew+s63eFU(Pyl3?cn?vOB<7+F|&mX&a#0%mII#W}c2@SnGQ qdfo?-ohubjz_zLM-S4e-d)WGG^*6;pu0$dMk#{6hg$V;}&JPg%sXO2R literal 0 HcmV?d00001 diff --git a/baremaps-core/src/test/resources/archives/file.tar.gz b/baremaps-core/src/test/resources/archives/file.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..77812f522ba4673dcc6f050573cf23b1768c5865 GIT binary patch literal 123 zcmV->0EGV^iwFR-n(Jf$1MSgY3VTj?B*&xipAj6QBnUktlQc)5b!pXqAV&?0VZ9rUF!Og(P z@|6*&Oa!Q|B(=CCz?+dtjv1FZ5 Date: Sun, 27 Aug 2023 21:39:54 +0200 Subject: [PATCH 03/66] Fix workflow --- basemap/{daylight.js => daylight/workflow.js} | 82 ++++++++++--------- 1 file changed, 42 insertions(+), 40 deletions(-) rename basemap/{daylight.js => daylight/workflow.js} (84%) diff --git a/basemap/daylight.js b/basemap/daylight/workflow.js similarity index 84% rename from basemap/daylight.js rename to basemap/daylight/workflow.js index 008336f98..2c23d5d84 100644 --- a/basemap/daylight.js +++ b/basemap/daylight/workflow.js @@ -20,30 +20,31 @@ export default { "id": "openstreetmap-data", "needs": [], "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", - "path": "data/data.osm.pbf" - }, - { - "type": "ImportOpenStreetMap", - "file": "data/data.osm.pbf", - "database": config.database, - "databaseSrid": 3857 - }, + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", + // "path": "data/data.osm.pbf" + // }, + // { + // "type": "ImportOpenStreetMap", + // "file": "data/data.osm.pbf", + // "database": config.database, + // "databaseSrid": 3857 + // }, ] }, { "id": "openstreetmap-roads", "needs": ["openstreetmap-data"], "tasks": [ + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2", + // "path": "data/roads.osc.bz2" + // }, { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2", - "path": "data/roads.osc.bz2" - }, - { - "type": "DecompressBZip2", + "type": "DecompressFile", + "compression": "bzip2", "file": "data/roads.osc.bz2", "directory": "data/roads.osc" }, @@ -53,13 +54,14 @@ export default { "id": "openstreetmap-admin", "needs": ["openstreetmap-data"], "tasks": [ + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", + // "path": "data/admin.osc.bz2" + // }, { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", - "path": "data/admin.osc.bz2" - }, - { - "type": "DecompressBZip2", + "type": "DecompressFile", + "compression": "bzip2", "file": "data/admin.osc.bz2", "directory": "data/admin.osc" }, @@ -69,13 +71,14 @@ export default { "id": "openstreetmap-coastlines", "needs": ["openstreetmap-data"], "tasks": [ + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", + // "path": "data/coastlines.osc.bz2" + // }, { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", - "path": "data/coastlines.osc.bz2" - }, - { - "type": "DecompressBZip2", + "type": "DecompressFile", + "compression": "bzip2", "file": "data/coastlines.osc.bz2", "directory": "data/coastlines.osc" }, @@ -85,25 +88,24 @@ export default { "id": "openstreetmap-preferred-localization", "needs": ["openstreetmap-data"], "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", - "path": "data/preferred-localization.tsv" - }, + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", + // "path": "data/preferred-localization.tsv" + // }, ] }, { "id": "openstreetmap-important-features", "needs": ["openstreetmap-data"], "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", - "path": "data/important-features.json" - }, + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", + // "path": "data/important-features.json" + // }, ] }, - // { // "id": "openstreetmap-nodes", // "needs": [ From 7ab0af3006ce090bc4da2c0b81bb2e70a3442734 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Sun, 27 Aug 2023 21:41:55 +0200 Subject: [PATCH 04/66] Fix workflow --- basemap/daylight/workflow.js | 72 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 2c23d5d84..4fe28296b 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -20,28 +20,28 @@ export default { "id": "openstreetmap-data", "needs": [], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", - // "path": "data/data.osm.pbf" - // }, - // { - // "type": "ImportOpenStreetMap", - // "file": "data/data.osm.pbf", - // "database": config.database, - // "databaseSrid": 3857 - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", + "path": "data/data.osm.pbf" + }, + { + "type": "ImportOpenStreetMap", + "file": "data/data.osm.pbf", + "database": config.database, + "databaseSrid": 3857 + }, ] }, { "id": "openstreetmap-roads", "needs": ["openstreetmap-data"], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2", - // "path": "data/roads.osc.bz2" - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/fb-ml-roads-v1.29.osc.bz2", + "path": "data/roads.osc.bz2" + }, { "type": "DecompressFile", "compression": "bzip2", @@ -54,11 +54,11 @@ export default { "id": "openstreetmap-admin", "needs": ["openstreetmap-data"], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", - // "path": "data/admin.osc.bz2" - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", + "path": "data/admin.osc.bz2" + }, { "type": "DecompressFile", "compression": "bzip2", @@ -71,11 +71,11 @@ export default { "id": "openstreetmap-coastlines", "needs": ["openstreetmap-data"], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", - // "path": "data/coastlines.osc.bz2" - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", + "path": "data/coastlines.osc.bz2" + }, { "type": "DecompressFile", "compression": "bzip2", @@ -88,22 +88,22 @@ export default { "id": "openstreetmap-preferred-localization", "needs": ["openstreetmap-data"], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", - // "path": "data/preferred-localization.tsv" - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", + "path": "data/preferred-localization.tsv" + }, ] }, { "id": "openstreetmap-important-features", "needs": ["openstreetmap-data"], "tasks": [ - // { - // "type": "DownloadUrl", - // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", - // "path": "data/important-features.json" - // }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", + "path": "data/important-features.json" + }, ] }, // { From 5896ddc9b7bab3dc847b7ef0ef2ec3ea33b5a162 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 08:41:37 +0200 Subject: [PATCH 05/66] Fix workflow --- basemap/daylight/workflow.js | 98 +++++++++++++++--------------------- 1 file changed, 40 insertions(+), 58 deletions(-) diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 4fe28296b..2d59fb66f 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -20,21 +20,21 @@ export default { "id": "openstreetmap-data", "needs": [], "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", - "path": "data/data.osm.pbf" - }, - { - "type": "ImportOpenStreetMap", - "file": "data/data.osm.pbf", - "database": config.database, - "databaseSrid": 3857 - }, + // { + // "type": "DownloadUrl", + // "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/planet-v1.29.osm.pbf", + // "path": "data/data.osm.pbf" + // }, + // { + // "type": "ImportOpenStreetMap", + // "file": "data/data.osm.pbf", + // "database": config.database, + // "databaseSrid": 3857 + // }, ] }, { - "id": "openstreetmap-roads", + "id": "openstreetmap-download", "needs": ["openstreetmap-data"], "tasks": [ { @@ -43,66 +43,48 @@ export default { "path": "data/roads.osc.bz2" }, { - "type": "DecompressFile", - "compression": "bzip2", - "file": "data/roads.osc.bz2", - "directory": "data/roads.osc" + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", + "path": "data/admin.osc.bz2" + }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", + "path": "data/coastlines.osc.bz2" + }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", + "path": "data/preferred-localization.tsv" + }, + { + "type": "DownloadUrl", + "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", + "path": "data/important-features.json" }, ] }, { - "id": "openstreetmap-admin", - "needs": ["openstreetmap-data"], + "id": "openstreetmap-decompress", + "needs": ["openstreetmap-download"], "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/admin-v1.29.osc.bz2", - "path": "data/admin.osc.bz2" - }, { "type": "DecompressFile", "compression": "bzip2", - "file": "data/admin.osc.bz2", - "directory": "data/admin.osc" - }, - ] - }, - { - "id": "openstreetmap-coastlines", - "needs": ["openstreetmap-data"], - "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", - "path": "data/coastlines.osc.bz2" + "source": "data/roads.osc.bz2", + "target": "data/roads.osc" }, { "type": "DecompressFile", "compression": "bzip2", - "file": "data/coastlines.osc.bz2", - "directory": "data/coastlines.osc" + "source": "data/admin.osc.bz2", + "target": "data/admin.osc" }, - ] - }, - { - "id": "openstreetmap-preferred-localization", - "needs": ["openstreetmap-data"], - "tasks": [ { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/preferred-localization-v1.29.tsv", - "path": "data/preferred-localization.tsv" - }, - ] - }, - { - "id": "openstreetmap-important-features", - "needs": ["openstreetmap-data"], - "tasks": [ - { - "type": "DownloadUrl", - "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/important-features-v1.29.json", - "path": "data/important-features.json" + "type": "DecompressFile", + "compression": "bzip2", + "source": "data/coastlines.osc.bz2", + "target": "data/coastlines.osc" }, ] }, From 8d2d2aea6aa2976c3b1e74fe603c072b00e49099 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 09:12:42 +0200 Subject: [PATCH 06/66] Fix coastline in workflow --- basemap/daylight/workflow.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 2d59fb66f..1fe73e284 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -50,7 +50,7 @@ export default { { "type": "DownloadUrl", "url": "https://daylight-map-distribution.s3.us-west-1.amazonaws.com/release/v1.29/coastlines-v1.29.tgz", - "path": "data/coastlines.osc.bz2" + "path": "data/coastlines.tgz" }, { "type": "DownloadUrl", @@ -83,8 +83,8 @@ export default { { "type": "DecompressFile", "compression": "bzip2", - "source": "data/coastlines.osc.bz2", - "target": "data/coastlines.osc" + "source": "data/coastlines.tgz", + "target": "data/coastlines" }, ] }, From 4d70327521e47ec4a4ed0954a7192842ac379a96 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 09:17:35 +0200 Subject: [PATCH 07/66] Fix compression --- basemap/daylight/workflow.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 1fe73e284..258dd3f1f 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -82,7 +82,7 @@ export default { }, { "type": "DecompressFile", - "compression": "bzip2", + "compression": "targz", "source": "data/coastlines.tgz", "target": "data/coastlines" }, From 37c85db16b7e672df5b7bbbe387405dce799438c Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 13:51:34 +0200 Subject: [PATCH 08/66] Fix NoSuchElementException in workflow executor --- .../org/apache/baremaps/workflow/WorkflowExecutor.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index 9ec177ed3..53f3bf385 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -213,13 +213,13 @@ private void logStepMeasures() { var workflowStart = stepMeasures.stream() .mapToLong(measures -> measures.stepMeasures.stream() .mapToLong(measure -> measure.start) - .min().getAsLong()) - .min().getAsLong(); + .min().orElseGet(() -> 0L)) + .min().orElseGet(() -> 0L); var workflowEnd = stepMeasures.stream() .mapToLong(measures -> measures.stepMeasures.stream() .mapToLong(measure -> measure.end) - .max().getAsLong()) - .max().getAsLong(); + .max().orElseGet(() -> 0L)) + .max().orElseGet(() -> 0L); var workflowDuration = Duration.ofMillis(workflowEnd - workflowStart); logger.info("Workflow graph: {}", this.graph); logger.info(" Duration: {}", formatDuration(workflowDuration)); From ae9d215c84b821f4587e2bd9b2a0e20652216a84 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 14:43:20 +0200 Subject: [PATCH 09/66] Fix incorect durations in workflow --- .../baremaps/workflow/WorkflowExecutor.java | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index 53f3bf385..f112ab086 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -210,28 +210,39 @@ private CompletableFuture getPreviousFutureStep(String stepId) { */ private void logStepMeasures() { logger.info("----------------------------------------"); + var workflowStart = stepMeasures.stream() - .mapToLong(measures -> measures.stepMeasures.stream() - .mapToLong(measure -> measure.start) - .min().orElseGet(() -> 0L)) - .min().orElseGet(() -> 0L); + .flatMapToLong(measures -> measures.stepMeasures.stream() + .mapToLong(measure -> measure.start)) + .min(); + var workflowEnd = stepMeasures.stream() - .mapToLong(measures -> measures.stepMeasures.stream() - .mapToLong(measure -> measure.end) - .max().orElseGet(() -> 0L)) - .max().orElseGet(() -> 0L); - var workflowDuration = Duration.ofMillis(workflowEnd - workflowStart); - logger.info("Workflow graph: {}", this.graph); - logger.info(" Duration: {}", formatDuration(workflowDuration)); + .flatMapToLong(measures -> measures.stepMeasures.stream() + .mapToLong(measure -> measure.end)) + .max(); + + if (workflowStart.isPresent() && workflowEnd.isPresent()) { + var workflowDuration = Duration.ofMillis(workflowEnd.getAsLong() - workflowStart.getAsLong()); + logger.info("Workflow graph: {}", this.graph); + logger.info(" Duration: {}", formatDuration(workflowDuration)); + } else { + logger.info("Workflow graph: {}", this.graph); + logger.info(" Duration: unknown"); + } for (var stepMeasure : this.stepMeasures) { - var stepStart = - stepMeasure.stepMeasures.stream().mapToLong(measure -> measure.start).min().getAsLong(); - var stepEnd = - stepMeasure.stepMeasures.stream().mapToLong(measure -> measure.end).max().getAsLong(); - var stepDuration = Duration.ofMillis(stepEnd - stepStart); - logger.info("Step: {}, Duration: {} ms", stepMeasure.step.getId(), - formatDuration(stepDuration)); + var stepStart = stepMeasure.stepMeasures.stream() + .mapToLong(measure -> measure.start).min(); + var stepEnd = stepMeasure.stepMeasures.stream() + .mapToLong(measure -> measure.end).max(); + + if (stepStart.isPresent() && stepEnd.isPresent()) { + var stepDuration = Duration.ofMillis(stepEnd.getAsLong() - stepStart.getAsLong()); + logger.info("Step: {}, Duration: {} ms", stepMeasure.step.getId(), + formatDuration(stepDuration)); + } else { + logger.info("Step: {}, Duration: unknown", stepMeasure.step.getId()); + } for (var taskMeasure : stepMeasure.stepMeasures) { var taskDuration = Duration.ofMillis(taskMeasure.end - taskMeasure.start); From d107cd273f3dc91e6e6be1cfec542ee9ede3f94a Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 15:59:45 +0200 Subject: [PATCH 10/66] Display durations of 0 ms --- .../java/org/apache/baremaps/workflow/WorkflowExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index f112ab086..7af8b07d6 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -278,7 +278,7 @@ private static String formatDuration(Duration duration) { builder.append(sec).append(" s "); } final long ms = duration.toMillis() - Duration.ofSeconds(duration.toSeconds()).toMillis(); - if (ms > 0) { + if (ms >= 0) { builder.append(ms).append(" ms "); } return builder.toString(); From d0278cc424c3a7cdbcd70d760587f0fb84fb435f Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 16:00:04 +0200 Subject: [PATCH 11/66] Display durations of 0 ms --- .../baremaps/workflow/WorkflowExecutor.java | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java index 7af8b07d6..43de015c8 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/WorkflowExecutor.java @@ -212,14 +212,14 @@ private void logStepMeasures() { logger.info("----------------------------------------"); var workflowStart = stepMeasures.stream() - .flatMapToLong(measures -> measures.stepMeasures.stream() - .mapToLong(measure -> measure.start)) - .min(); + .flatMapToLong(measures -> measures.stepMeasures.stream() + .mapToLong(measure -> measure.start)) + .min(); var workflowEnd = stepMeasures.stream() - .flatMapToLong(measures -> measures.stepMeasures.stream() - .mapToLong(measure -> measure.end)) - .max(); + .flatMapToLong(measures -> measures.stepMeasures.stream() + .mapToLong(measure -> measure.end)) + .max(); if (workflowStart.isPresent() && workflowEnd.isPresent()) { var workflowDuration = Duration.ofMillis(workflowEnd.getAsLong() - workflowStart.getAsLong()); @@ -232,9 +232,9 @@ private void logStepMeasures() { for (var stepMeasure : this.stepMeasures) { var stepStart = stepMeasure.stepMeasures.stream() - .mapToLong(measure -> measure.start).min(); + .mapToLong(measure -> measure.start).min(); var stepEnd = stepMeasure.stepMeasures.stream() - .mapToLong(measure -> measure.end).max(); + .mapToLong(measure -> measure.end).max(); if (stepStart.isPresent() && stepEnd.isPresent()) { var stepDuration = Duration.ofMillis(stepEnd.getAsLong() - stepStart.getAsLong()); @@ -262,24 +262,28 @@ private void logStepMeasures() { private static String formatDuration(Duration duration) { var builder = new StringBuilder(); var days = duration.toDays(); - if (days > 0) { - builder.append(days).append(" days "); - } - final long hrs = duration.toHours() - Duration.ofDays(duration.toDays()).toHours(); - if (hrs > 0) { - builder.append(hrs).append(" hrs "); - } - final long min = duration.toMinutes() - Duration.ofHours(duration.toHours()).toMinutes(); - if (min > 0) { - builder.append(min).append(" min "); - } - final long sec = duration.toSeconds() - Duration.ofMinutes(duration.toMinutes()).toSeconds(); - if (sec > 0) { - builder.append(sec).append(" s "); - } - final long ms = duration.toMillis() - Duration.ofSeconds(duration.toSeconds()).toMillis(); - if (ms >= 0) { - builder.append(ms).append(" ms "); + if (duration.isZero()) { + builder.append("0 ms"); + } else { + if (days > 0) { + builder.append(days).append(" days "); + } + final long hrs = duration.toHours() - Duration.ofDays(duration.toDays()).toHours(); + if (hrs > 0) { + builder.append(hrs).append(" hrs "); + } + final long min = duration.toMinutes() - Duration.ofHours(duration.toHours()).toMinutes(); + if (min > 0) { + builder.append(min).append(" min "); + } + final long sec = duration.toSeconds() - Duration.ofMinutes(duration.toMinutes()).toSeconds(); + if (sec > 0) { + builder.append(sec).append(" s "); + } + final long ms = duration.toMillis() - Duration.ofSeconds(duration.toSeconds()).toMillis(); + if (ms > 0) { + builder.append(ms).append(" ms "); + } } return builder.toString(); } From c09471063dee32846ed4f9aab766174a3b128207 Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Mon, 28 Aug 2023 17:03:47 +0200 Subject: [PATCH 12/66] wqRename import osm task and add osc task --- .../baremaps/cli/database/Database.java | 2 +- ...rtOpenStreetMap.java => ImportOsmPbf.java} | 8 +- .../org/apache/baremaps/workflow/Task.java | 2 +- .../baremaps/workflow/tasks/ImportOsc.java | 91 +++++++++++++++++++ ...rtOpenStreetMap.java => ImportOsmPbf.java} | 2 +- .../baremaps/workflow/ObjectMapperTest.java | 8 +- .../baremaps/workflow/WorkflowTest.java | 4 +- .../workflow/tasks/ImportMonacoTest.java | 2 +- ...nStreetMapTest.java => ImportPbfTest.java} | 4 +- .../workflow/tasks/ImportUpdateDataTest.java | 2 +- .../tasks/ImportUpdateLiechtensteinTest.java | 2 +- basemap/daylight/workflow.js | 2 +- basemap/workflow.js | 2 +- examples/extrusion/workflow.json | 2 +- examples/openstreetmap/workflow.json | 2 +- 15 files changed, 114 insertions(+), 21 deletions(-) rename baremaps-cli/src/main/java/org/apache/baremaps/cli/database/{ImportOpenStreetMap.java => ImportOsmPbf.java} (89%) create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java rename baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/{ImportOpenStreetMap.java => ImportOsmPbf.java} (99%) rename baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/{ImportOpenStreetMapTest.java => ImportPbfTest.java} (91%) diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java index 7c36a536c..60dd9efd8 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/Database.java @@ -23,7 +23,7 @@ import picocli.CommandLine.Command; @Command(name = "database", description = "Database commands.", - subcommands = {ExecuteSql.class, ImportOpenStreetMap.class, UpdateOpenStreetMap.class}, + subcommands = {ExecuteSql.class, ImportOsmPbf.class, UpdateOpenStreetMap.class}, sortOptions = false) public class Database implements Runnable { diff --git a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java similarity index 89% rename from baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java rename to baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java index f512a1f12..71249f053 100644 --- a/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOpenStreetMap.java +++ b/baremaps-cli/src/main/java/org/apache/baremaps/cli/database/ImportOsmPbf.java @@ -28,7 +28,7 @@ import picocli.CommandLine.Option; @Command(name = "import-osm", description = "Import OpenStreetMap data in Postgres.") -public class ImportOpenStreetMap implements Callable { +public class ImportOsmPbf implements Callable { @Mixin private Options options; @@ -47,8 +47,10 @@ public class ImportOpenStreetMap implements Callable { @Override public Integer call() throws Exception { - new org.apache.baremaps.workflow.tasks.ImportOpenStreetMap(file.toAbsolutePath(), - database, srid).execute(new WorkflowContext()); + new org.apache.baremaps.workflow.tasks.ImportOsmPbf( + file.toAbsolutePath(), + database, + srid).execute(new WorkflowContext()); return 0; } } diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 77f4336d7..88331a33f 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -38,7 +38,7 @@ @JsonSubTypes.Type(value = ExecuteSqlScript.class, name = "ExecuteSqlScript"), @JsonSubTypes.Type(value = ExportVectorTiles.class, name = "ExportVectorTiles"), @JsonSubTypes.Type(value = ImportGeoPackage.class, name = "ImportGeoPackage"), - @JsonSubTypes.Type(value = ImportOpenStreetMap.class, name = "ImportOpenStreetMap"), + @JsonSubTypes.Type(value = ImportOsmPbf.class, name = "ImportOsmPbf"), @JsonSubTypes.Type(value = ImportShapefile.class, name = "ImportShapefile"), @JsonSubTypes.Type(value = LogMessage.class, name = "LogMessage"), @JsonSubTypes.Type(value = UnzipFile.class, name = "UnzipFile"), diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java new file mode 100644 index 000000000..f0aa82859 --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java @@ -0,0 +1,91 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import static org.apache.baremaps.stream.ConsumerUtils.consumeThenReturn; + +import java.io.BufferedInputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Path; +import java.util.zip.GZIPInputStream; +import org.apache.baremaps.openstreetmap.function.ChangeEntitiesHandler; +import org.apache.baremaps.openstreetmap.function.EntityGeometryBuilder; +import org.apache.baremaps.openstreetmap.function.EntityProjectionTransformer; +import org.apache.baremaps.openstreetmap.model.Header; +import org.apache.baremaps.openstreetmap.model.Node; +import org.apache.baremaps.openstreetmap.model.Relation; +import org.apache.baremaps.openstreetmap.model.Way; +import org.apache.baremaps.openstreetmap.postgres.*; +import org.apache.baremaps.openstreetmap.repository.ChangeImporter; +import org.apache.baremaps.openstreetmap.repository.Repository; +import org.apache.baremaps.openstreetmap.state.StateReader; +import org.apache.baremaps.openstreetmap.xml.XmlChangeReader; +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record ImportOsc(Path file, String database, Integer srid) implements Task { + + private static final Logger logger = LoggerFactory.getLogger(ImportOsc.class); + + @Override + public void execute(WorkflowContext context) throws Exception { + var datasource = context.getDataSource(database); + + Repository nodeRepository = new PostgresNodeRepository(datasource); + Repository wayRepository = new PostgresWayRepository(datasource); + Repository relationRepository = new PostgresRelationRepository(datasource); + execute( + nodeRepository, + wayRepository, + relationRepository, + srid); + } + + public static void execute( + Repository nodeRepository, + Repository wayRepository, + Repository relationRepository, + int srid) throws Exception { + + var createGeometry = new EntityGeometryBuilder(coordinateMap, referenceMap); + var reprojectGeometry = new EntityProjectionTransformer(4326, srid); + var prepareGeometries = new ChangeEntitiesHandler(createGeometry.andThen(reprojectGeometry)); + var prepareChange = consumeThenReturn(prepareGeometries); + var saveChange = new ChangeImporter(nodeRepository, wayRepository, relationRepository); + + var changeUrl = resolve(replicationUrl, sequenceNumber, "osc.gz"); + try (var changeInputStream = + new GZIPInputStream(new BufferedInputStream(changeUrl.openStream()))) { + new XmlChangeReader().stream(changeInputStream).map(prepareChange).forEach(saveChange); + } + + var stateUrl = resolve(replicationUrl, sequenceNumber, "state.txt"); + try (var stateInputStream = new BufferedInputStream(stateUrl.openStream())) { + var state = new StateReader().state(stateInputStream); + headerRepository.put(new Header(state.getSequenceNumber(), state.getTimestamp(), + header.getReplicationUrl(), header.getSource(), header.getWritingProgram())); + } + } + + public static URL resolve(String replicationUrl, Long sequenceNumber, String extension) + throws MalformedURLException { + var s = String.format("%09d", sequenceNumber); + var uri = String.format("%s/%s/%s/%s.%s", replicationUrl, s.substring(0, 3), s.substring(3, 6), + s.substring(6, 9), extension); + return URI.create(uri).toURL(); + } +} diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java similarity index 99% rename from baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java rename to baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java index d95195be6..1c02461c9 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMap.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmPbf.java @@ -50,7 +50,7 @@ public record ImportOpenStreetMap(Path file, Object database, Integer databaseSr implements Task { - private static final Logger logger = LoggerFactory.getLogger(ImportOpenStreetMap.class); + private static final Logger logger = LoggerFactory.getLogger(ImportOsmPbf.class); @Override public void execute(WorkflowContext context) throws Exception { diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java index a41bab115..041f024ac 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/ObjectMapperTest.java @@ -24,7 +24,7 @@ import java.nio.file.Paths; import java.util.List; import org.apache.baremaps.workflow.tasks.DownloadUrl; -import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap; +import org.apache.baremaps.workflow.tasks.ImportOsmPbf; import org.junit.Test; public class ObjectMapperTest { @@ -41,16 +41,16 @@ public void test() throws IOException { "https://download.geofabrik.de/europe/liechtenstein-latest.osm.pbf", Paths.get("liechtenstein-latest.osm.pbf")))), new Step("import", List.of("download"), - List.of(new ImportOpenStreetMap(Paths.get("liechtenstein-latest.osm.pbf"), + List.of(new ImportOsmPbf(Paths.get("liechtenstein-latest.osm.pbf"), "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", 3857))))); var json = mapper.writeValueAsString(workflow1); assertTrue(json.contains(DownloadUrl.class.getSimpleName())); - assertTrue(json.contains(ImportOpenStreetMap.class.getSimpleName())); + assertTrue(json.contains(ImportOsmPbf.class.getSimpleName())); // deserialize the workflow var workflow2 = mapper.readValue(json, Workflow.class); assertTrue(workflow2.getSteps().get(0).getTasks().get(0) instanceof DownloadUrl); - assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOpenStreetMap); + assertTrue(workflow2.getSteps().get(1).getTasks().get(0) instanceof ImportOsmPbf); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java index 888201670..408930b94 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/WorkflowTest.java @@ -24,7 +24,7 @@ import org.apache.baremaps.testing.PostgresContainerTest; import org.apache.baremaps.workflow.tasks.DownloadUrl; import org.apache.baremaps.workflow.tasks.ImportGeoPackage; -import org.apache.baremaps.workflow.tasks.ImportOpenStreetMap; +import org.apache.baremaps.workflow.tasks.ImportOsmPbf; import org.apache.baremaps.workflow.tasks.ImportShapefile; import org.apache.baremaps.workflow.tasks.UnzipFile; import org.junit.jupiter.api.Disabled; @@ -103,7 +103,7 @@ void execute() { List.of(new DownloadUrl("https://tiles.baremaps.com/samples/liechtenstein.osm.pbf", Paths.get("downloads/liechtenstein.osm.pbf")))), new Step("import-osmpbf", List.of("fetch-osmpbf"), - List.of(new ImportOpenStreetMap(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(), + List.of(new ImportOsmPbf(Paths.get("downloads/liechtenstein.osm.pbf"), jdbcUrl(), 3857))), new Step("fetch-shapefile", List.of(), List.of(new DownloadUrl( "https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip", diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java index 2ebbac710..705108c03 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportMonacoTest.java @@ -57,7 +57,7 @@ void monaco() throws Exception { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap, + ImportOsmPbf.execute(TestFiles.resolve("monaco/monaco-210801.osm.pbf"), coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); assertEquals(3047l, headerRepository.selectLatest().getReplicationSequenceNumber()); diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java similarity index 91% rename from baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java rename to baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java index 6a864a59d..8750c2efc 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportOpenStreetMapTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportPbfTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -class ImportOpenStreetMapTest extends PostgresContainerTest { +class ImportOsmPbfTest extends PostgresContainerTest { @Test @Tag("integration") @@ -33,7 +33,7 @@ void execute() throws Exception { var file = TestFiles.resolve("data.osm.pbf"); var jdbcUrl = jdbcUrl(); var srid = 3857; - var task = new ImportOpenStreetMap(file, jdbcUrl, srid); + var task = new ImportOsmPbf(file, jdbcUrl, srid); task.execute(new WorkflowContext()); } } diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java index f032fd19b..d462d05c9 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateDataTest.java @@ -61,7 +61,7 @@ void data() throws Exception { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository, + ImportOsmPbf.execute(SIMPLE_DATA_OSM_PBF, coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); headerRepository.put(new Header(0l, LocalDateTime.of(2020, 1, 1, 0, 0, 0, 0), diff --git a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java index e41d631bb..3df9452bc 100644 --- a/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java +++ b/baremaps-core/src/test/java/org/apache/baremaps/workflow/tasks/ImportUpdateLiechtensteinTest.java @@ -58,7 +58,7 @@ void liechtenstein() throws Exception { new IndexedDataMap<>(new AppendOnlyBuffer<>(new LongListDataType(), new OnHeapMemory())); // Import data - ImportOpenStreetMap.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap, + ImportOsmPbf.execute(LIECHTENSTEIN_OSM_PBF, coordinateMap, referenceMap, headerRepository, nodeRepository, wayRepository, relationRepository, 3857); diff --git a/basemap/daylight/workflow.js b/basemap/daylight/workflow.js index 258dd3f1f..abc125503 100644 --- a/basemap/daylight/workflow.js +++ b/basemap/daylight/workflow.js @@ -26,7 +26,7 @@ export default { // "path": "data/data.osm.pbf" // }, // { - // "type": "ImportOpenStreetMap", + // "type": "ImportOsmPbf", // "file": "data/data.osm.pbf", // "database": config.database, // "databaseSrid": 3857 diff --git a/basemap/workflow.js b/basemap/workflow.js index 2b407c652..7bab101d4 100644 --- a/basemap/workflow.js +++ b/basemap/workflow.js @@ -127,7 +127,7 @@ export default { "path": "data/data.osm.pbf" }, { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "data/data.osm.pbf", "database": config.database, "databaseSrid": 3857 diff --git a/examples/extrusion/workflow.json b/examples/extrusion/workflow.json index de11af58f..039ad0d1a 100644 --- a/examples/extrusion/workflow.json +++ b/examples/extrusion/workflow.json @@ -18,7 +18,7 @@ ], "tasks": [ { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "greater-london-latest.osm.pbf", "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", "databaseSrid": 3857 diff --git a/examples/openstreetmap/workflow.json b/examples/openstreetmap/workflow.json index 9ebd677a5..d488cb4bf 100644 --- a/examples/openstreetmap/workflow.json +++ b/examples/openstreetmap/workflow.json @@ -18,7 +18,7 @@ ], "tasks": [ { - "type": "ImportOpenStreetMap", + "type": "ImportOsmPbf", "file": "liechtenstein-latest.osm.pbf", "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps", "databaseSrid": 3857 From ded7b753fcf3d4e6aea7a85adeae3986e058ea4a Mon Sep 17 00:00:00 2001 From: Bertil Chapuis Date: Tue, 29 Aug 2023 00:19:36 +0200 Subject: [PATCH 13/66] Improve change importer with copy api --- .run/basemap-workflow.run.xml | 5 + .../repository/ChangeImporter.java | 79 +++++++--- .../org/apache/baremaps/workflow/Task.java | 1 + .../baremaps/workflow/tasks/ImportOsc.java | 91 ----------- .../workflow/tasks/ImportOsmChange.java | 56 +++++++ .../workflow/tasks/UpdateOpenStreetMap.java | 34 +---- basemap/daylight/workflow.js | 144 ++++++++++-------- 7 files changed, 208 insertions(+), 202 deletions(-) delete mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsc.java create mode 100644 baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportOsmChange.java diff --git a/.run/basemap-workflow.run.xml b/.run/basemap-workflow.run.xml index 7b2b0ee7b..6d24c3cc6 100644 --- a/.run/basemap-workflow.run.xml +++ b/.run/basemap-workflow.run.xml @@ -9,6 +9,11 @@