From 95c2b487d792b7b38d5659bcfb9fc4a6accdebc3 Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:08:08 +0100 Subject: [PATCH 01/13] Docker image for Overpass Based on NGINX image. Uses supervisord to run: * nginx * fastcgi wrapper * Overpass dispatcher * fetch_osc.sh * apply_osc_to_db.sh On first run initializes itself with the data. Details provided in README_Docker.md. --- Dockerfile | 56 ++++++++++++++++++++++++++++++++++ README_Docker.md | 60 +++++++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 28 +++++++++++++++++ etc/nginx-overpass.conf | 66 +++++++++++++++++++++++++++++++++++++++++ etc/supervisord.conf | 31 +++++++++++++++++++ 5 files changed, 241 insertions(+) create mode 100644 Dockerfile create mode 100644 README_Docker.md create mode 100755 docker-entrypoint.sh create mode 100644 etc/nginx-overpass.conf create mode 100644 etc/supervisord.conf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..09e0476ba --- /dev/null +++ b/Dockerfile @@ -0,0 +1,56 @@ +FROM nginx:stable + +RUN addgroup overpass && adduser --home /db --disabled-password --ingroup overpass overpass + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + expat \ + libexpat1-dev \ + g++ \ + libtool \ + m4 \ + make \ + zlib1g \ + zlib1g-dev + +COPY . /app/ + +RUN cd /app/src \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app \ + && make -j $(grep -c ^processor /proc/cpuinfo) install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y + +RUN apt-get install --no-install-recommends --no-install-suggests -y \ + supervisor \ + bash \ + lftp \ + wget \ + fcgiwrap \ + bzip2 + + +RUN mkdir /nginx && chown nginx:nginx /nginx && mkdir -p /db/db /db/diffs && chown -R overpass:overpass /db +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf +VOLUME /db + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/README_Docker.md b/README_Docker.md new file mode 100644 index 000000000..405557f78 --- /dev/null +++ b/README_Docker.md @@ -0,0 +1,60 @@ +# How to use this image + +This image on startup initializes with data on first start. +To do it properly it needs a bit of guidance in form of enviroment variables. +* `OVERPASS_MODE` - takes the value of either `init` or `clone` +* `OVERPASS_META` - `yes` or `no` - passed to Overpass as `--meta` +* `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/) +* `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode +* `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied + +Image works in two modes `clone` or `init`. This affects how the instance gets initialized. If the mode is set to `clone` +then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated from diffs. This will result in Overpass instance +covering whole world. This mode works only with minute diffs. + +In `init` mode you need to point `OVERPASS_PLANET_URL` to address with planet (partial) dump. This file will be downloaded, +indexed by Overpass and later - updated using `OVERPASS_PLANET_SEQUENCE_ID` and `OVERPASS_DIFF_URL`. You need to check which +sequence number is for your planet file. Take it from desctiption or just take a sequence number a day before your planet +file is dated. + +Start initalization mode with `-i` and `-t` options to `docker run` so you will have a chance to monitor the progress of +file downloads etc. + +After initialization is finished Docker container will stop. Once you start it again (with `docker start` command) it will start +downloading minute diffs, applying them to database and serving API requests. + +Container exposes port 80. + +All data resides within /db directory in container. + +# Examples +## Overpass instance covering part of the world +In this example Overpass instance will be initialized with planet file for Poland downloaded from Geofabrik. Data will be stored in folder +`/big/docker/overpass_db/` on the host machine. Overpass will be available on port 12345 on host machine. +``` +docker run \ + -e OVERPASS_META=yes \ + -e OVERPASS_MODE=init \ + -e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/poland-latest.osm.bz2 \ + -e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/poland/minute/ \ + -e OVERPASS_PLANET_SEQUENCE_ID=2346486 \ + -v /big/docker/overpass_db/:/db \ + -p 12345:80 \ + -i -t \ + --name overpass_poland overpass +``` + +## Overpass clone covering whole world +In this example Overpass instance will be initialized with data from main Overpass instance and updated with master planet diffs. +Data will be stored in /big/docker/overpass_clone_db/ directory on the host machine and API will be exposed on port 12346 on host machine. +``` +docker run \ + -e OVERPASS_META=yes + -e OVERPASS_MODE=clone + -e OVERPASS_DIFF_URL=https://planet.openstreetmap.org/replication/minute/ + -v /big/docker/overpass_clone_db/:/db + -p 12346:80 + -i -t + --name overpass_world + overpass +``` diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 000000000..96bb58ee6 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ "--meta=$OVERPASS_META" \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c $OVERPASS_PLANET_URL -o /db/planet; exit" \ + && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \ + && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ + && rm /db/planet \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/etc/nginx-overpass.conf b/etc/nginx-overpass.conf new file mode 100644 index 000000000..c01961579 --- /dev/null +++ b/etc/nginx-overpass.conf @@ -0,0 +1,66 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/etc/supervisord.conf b/etc/supervisord.conf new file mode 100644 index 000000000..ea63ea9f8 --- /dev/null +++ b/etc/supervisord.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/bin/bash -c "find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base --meta --db-dir=/db/db" +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:fetch_diff] +command=/app/bin/fetch_osc.sh auto %(OVERPASS_DIFF_URL)s /db/diffs +user=overpass +redirect_stderr=true +priority=5 + +[program:apply_diff] +command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s +redirect_stderr=true +user=overpass +priority=4 + From 0e6f3f7c78d85c6e47d240e929dc5c78ea4b867d Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:23:01 +0100 Subject: [PATCH 02/13] Add quotations around variable name --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 96bb58ee6..cb24f6695 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -15,7 +15,7 @@ if [ ! -d /db/db ] ; then fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c $OVERPASS_PLANET_URL -o /db/planet; exit" \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ From 423cacadb822bb4f8cd6a594fd527b260bfbe27f Mon Sep 17 00:00:00 2001 From: W Date: Tue, 14 Mar 2017 00:36:20 +0100 Subject: [PATCH 03/13] Add support for attic, areas and compression options. Add liblz4 to image so Overpass will support additional compression algorithm. Add two processes: * dispatcher_areas - for calculating areas * areas_rules - rules loop Added src/rules to databases directory Added OVERPASS_COMPRESSION environment variable Added `attic` value to OVERPASS_META variable --- Dockerfile | 6 +++++- README_Docker.md | 4 +++- docker-entrypoint.sh | 10 ++++++++-- etc/supervisord.conf | 13 ++++++++++++- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09e0476ba..052e7d663 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN apt-get update \ expat \ libexpat1-dev \ g++ \ + liblz4-1 \ + liblz4-dev \ libtool \ m4 \ make \ @@ -32,10 +34,11 @@ RUN cd /app/src \ libexpat1-dev \ g++ \ libtool \ + liblz4-dev \ m4 \ make \ zlib1g-dev \ - && apt-get autoremove -y + && apt-get autoremove -y RUN apt-get install --no-install-recommends --no-install-suggests -y \ supervisor \ @@ -47,6 +50,7 @@ RUN apt-get install --no-install-recommends --no-install-suggests -y \ RUN mkdir /nginx && chown nginx:nginx /nginx && mkdir -p /db/db /db/diffs && chown -R overpass:overpass /db +COPY src/rules /db/db/rules COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf VOLUME /db diff --git a/README_Docker.md b/README_Docker.md index 405557f78..eb9cb0267 100644 --- a/README_Docker.md +++ b/README_Docker.md @@ -3,10 +3,12 @@ This image on startup initializes with data on first start. To do it properly it needs a bit of guidance in form of enviroment variables. * `OVERPASS_MODE` - takes the value of either `init` or `clone` -* `OVERPASS_META` - `yes` or `no` - passed to Overpass as `--meta` +* `OVERPASS_META` - `yes`, `no` or `attic` - passed to Overpass as `--meta` or `--keep-attic` * `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/) * `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode * `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied +* `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. +Ony useful in `init` mode. Defaults to `gz` Image works in two modes `clone` or `init`. This affects how the instance gets initialized. If the mode is set to `clone` then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated from diffs. This will result in Overpass instance diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 96bb58ee6..5754015a3 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,18 +4,24 @@ set -eo pipefail shopt -s nullglob OVERPASS_META=${OVERPASS_META:-no} OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "clone" ]; then + if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" + else + META="--meta=$OVERPASS_META" + fi mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ "--meta=$OVERPASS_META" \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META "--compresion_method=$OVERPASS_COMPRESSION" \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" exit fi if [ "$OVERPASS_MODE" = "init" ]; then - lftp -c "get -c $OVERPASS_PLANET_URL -o /db/planet; exit" \ + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ diff --git a/etc/supervisord.conf b/etc/supervisord.conf index ea63ea9f8..fa8ed5df9 100644 --- a/etc/supervisord.conf +++ b/etc/supervisord.conf @@ -18,7 +18,7 @@ user=nginx priority=3 [program:fetch_diff] -command=/app/bin/fetch_osc.sh auto %(OVERPASS_DIFF_URL)s /db/diffs +command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs user=overpass redirect_stderr=true priority=5 @@ -29,3 +29,14 @@ redirect_stderr=true user=overpass priority=4 +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db +user=overpass +redirect_stderr=true +priority=7 From 0debd98755e9502ff985068cd1b3d57c604f669f Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Tue, 7 Mar 2017 22:39:53 +0100 Subject: [PATCH 04/13] Fixed bad resource management in out:popup --- src/overpass_api/data/regular_expression.h | 3 + .../output_formats/output_popup.cc | 298 ++++++++++++++++++ .../output_formats/output_popup.h | 29 +- .../output_formats/output_popup_factory.cc | 15 +- 4 files changed, 334 insertions(+), 11 deletions(-) create mode 100644 src/overpass_api/output_formats/output_popup.cc diff --git a/src/overpass_api/data/regular_expression.h b/src/overpass_api/data/regular_expression.h index 59bbf50a7..1a8c6f563 100644 --- a/src/overpass_api/data/regular_expression.h +++ b/src/overpass_api/data/regular_expression.h @@ -88,6 +88,9 @@ class Regular_Expression } private: + Regular_Expression(const Regular_Expression&); + const Regular_Expression& operator=(const Regular_Expression&); + regex_t preg; Strategy strategy; // mutable bool cache_available; diff --git a/src/overpass_api/output_formats/output_popup.cc b/src/overpass_api/output_formats/output_popup.cc new file mode 100644 index 000000000..258786755 --- /dev/null +++ b/src/overpass_api/output_formats/output_popup.cc @@ -0,0 +1,298 @@ + +#include "../../expat/escape_xml.h" +#include "../frontend/basic_formats.h" +#include "output_popup.h" + +#include +#include + + +bool Tag_Filter::matches(const std::vector< std::pair< std::string, std::string > >* tags) const +{ + if (!tags) + return false; + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags->begin(); it != tags->end(); ++it) + { + if (it->first == key && condition.matches(it->second)) + return straight; + } + return !straight; +} + + +std::string link_if_any(const std::vector< std::pair< std::string, std::string > >& tags) +{ + std::string link; + + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags.begin(); it != tags.end(); ++it) + { + if (it->second.find("www.") != std::string::npos) + link = "http://" + it->second; + } + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags.begin(); it != tags.end(); ++it) + { + if (it->second.find("http://") != std::string::npos) + link = it->second; + } + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags.begin(); it != tags.end(); ++it) + { + if (it->second.find("https://") != std::string::npos) + link = it->second; + } + + return link; +} + + +std::string title_if_any(const std::string& link, const std::string& title_key, + const std::vector< std::pair< std::string, std::string > >& tags) +{ + std::string title; + + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags.begin(); it != tags.end(); ++it) + { + if (it->first == title_key) + { + if (link != "") + title += ""; + title += "" + it->second + ""; + if (link != "") + title += ""; + title += "
\n"; + + break; + } + } + + return title; +} + + +void Category_Filter::set_title(const std::string& title) +{ + output = "\n

" + title + "

"; +} + + +void Category_Filter::set_title_key(const std::string& title_key_) +{ + title_key = title_key_; +} + + +template< typename TSkel > std::string elem_type() { return ""; } +template< > std::string elem_type< Node_Skeleton >() { return "Node"; } +template< > std::string elem_type< Way_Skeleton >() { return "Way"; } +template< > std::string elem_type< Relation_Skeleton >() { return "Relation"; } + + +template< typename TSkel > +std::string print(const std::string& title_key, + const TSkel& skel, const std::vector< std::pair< std::string, std::string > >* tags) +{ + std::string link = tags ? link_if_any(*tags) : ""; + std::string result = "\n

" + (tags ? title_if_any(link, title_key, *tags) : ""); + + if (result == "\n

") + { + if (link != "") + result += ""; + std::ostringstream out; + out<() + " " + out.str() + ""; + if (link != "") + result += ""; + result += "
\n"; + } + + if (tags) + { + for (std::vector< std::pair< std::string, std::string > >::const_iterator it = tags->begin(); + it != tags->end(); ++it) + result += it->first + ": " + it->second + "
\n"; + } + return result + "

\n"; +} + + +bool Category_Filter::consider(const Node_Skeleton& skel, + const std::vector< std::pair< std::string, std::string > >* tags) +{ + std::vector< std::vector< Tag_Filter* > >::const_iterator it_conj = filter_disjunction.begin(); + for (; it_conj != filter_disjunction.end(); ++it_conj) + { + std::vector< Tag_Filter* >::const_iterator it = it_conj->begin(); + for (; it != it_conj->end(); ++it) + { + if ((*it)->matches(tags)) + break; + } + if (it != it_conj->end()) + break; + } + + if (it_conj != filter_disjunction.end()) + output += print(title_key, skel, tags); + + return it_conj == filter_disjunction.end(); +} + + +bool Category_Filter::consider(const Way_Skeleton& skel, + const std::vector< std::pair< std::string, std::string > >* tags) +{ + std::vector< std::vector< Tag_Filter* > >::const_iterator it_conj = filter_disjunction.begin(); + for (; it_conj != filter_disjunction.end(); ++it_conj) + { + std::vector< Tag_Filter* >::const_iterator it = it_conj->begin(); + for (; it != it_conj->end(); ++it) + { + if ((*it)->matches(tags)) + break; + } + if (it != it_conj->end()) + break; + } + + if (it_conj != filter_disjunction.end()) + output += print(title_key, skel, tags); + + return it_conj == filter_disjunction.end(); +} + + +bool Category_Filter::consider(const Relation_Skeleton& skel, + const std::vector< std::pair< std::string, std::string > >* tags) +{ + std::vector< std::vector< Tag_Filter* > >::const_iterator it_conj = filter_disjunction.begin(); + for (; it_conj != filter_disjunction.end(); ++it_conj) + { + std::vector< Tag_Filter* >::const_iterator it = it_conj->begin(); + for (; it != it_conj->end(); ++it) + { + if ((*it)->matches(tags)) + break; + } + if (it != it_conj->end()) + break; + } + + if (it_conj != filter_disjunction.end()) + output += print(title_key, skel, tags); + + return it_conj == filter_disjunction.end(); +} + + +bool Output_Popup::write_http_headers() +{ + std::cout<<"Content-type: text/html; charset=utf-8\n"; + return true; +} + + +void Output_Popup::write_payload_header + (const std::string& db_dir_, const std::string& timestamp_, const std::string& area_timestamp_) +{ + std::cout<< + "\n" + "\n" + "\n" + "\n" + " \n" + " OSM3S Response\n" + "\n" + "\n"; +} + + +void Output_Popup::display_remark(const std::string& text) +{ + //TODO +} + + +void Output_Popup::display_error(const std::string& text) +{ + //TODO +} + + +void Output_Popup::write_footer() +{ + for (std::vector< Category_Filter* >::iterator it = categories.begin(); it != categories.end(); ++it) + std::cout<<(*it)->result(); + + std::cout<<"\n\n\n"; +} + + +void Output_Popup::print_item(const Node_Skeleton& skel, + const Opaque_Geometry& geometry, + const std::vector< std::pair< std::string, std::string > >* tags, + const OSM_Element_Metadata_Skeleton< Node::Id_Type >* meta, + const std::map< uint32, std::string >* users, + Output_Mode mode, + const Feature_Action& action, + const Node_Skeleton* new_skel, + const Opaque_Geometry* new_geometry, + const std::vector< std::pair< std::string, std::string > >* new_tags, + const OSM_Element_Metadata_Skeleton< Node::Id_Type >* new_meta) +{ + for (std::vector< Category_Filter* >::iterator it = categories.begin(); it != categories.end(); ++it) + { + if ((*it)->consider(skel, tags)) + return; + } +} + + +void Output_Popup::print_item(const Way_Skeleton& skel, + const Opaque_Geometry& geometry, + const std::vector< std::pair< std::string, std::string > >* tags, + const OSM_Element_Metadata_Skeleton< Way::Id_Type >* meta, + const std::map< uint32, std::string >* users, + Output_Mode mode, + const Feature_Action& action, + const Way_Skeleton* new_skel, + const Opaque_Geometry* new_geometry, + const std::vector< std::pair< std::string, std::string > >* new_tags, + const OSM_Element_Metadata_Skeleton< Way::Id_Type >* new_meta) +{ + for (std::vector< Category_Filter* >::iterator it = categories.begin(); it != categories.end(); ++it) + { + if ((*it)->consider(skel, tags)) + return; + } +} + + +void Output_Popup::print_item(const Relation_Skeleton& skel, + const Opaque_Geometry& geometry, + const std::vector< std::pair< std::string, std::string > >* tags, + const OSM_Element_Metadata_Skeleton< Relation::Id_Type >* meta, + const std::map< uint32, std::string >* roles, + const std::map< uint32, std::string >* users, + Output_Mode mode, + const Feature_Action& action, + const Relation_Skeleton* new_skel, + const Opaque_Geometry* new_geometry, + const std::vector< std::pair< std::string, std::string > >* new_tags, + const OSM_Element_Metadata_Skeleton< Relation::Id_Type >* new_meta) +{ + for (std::vector< Category_Filter* >::iterator it = categories.begin(); it != categories.end(); ++it) + { + if ((*it)->consider(skel, tags)) + return; + } +} + + +void Output_Popup::print_item(const Derived_Skeleton& skel, + const Opaque_Geometry& geometry, + const std::vector< std::pair< std::string, std::string > >* tags, + Output_Mode mode) +{ + // Intentionally empty +} diff --git a/src/overpass_api/output_formats/output_popup.h b/src/overpass_api/output_formats/output_popup.h index 2291693b9..eaa7e815f 100644 --- a/src/overpass_api/output_formats/output_popup.h +++ b/src/overpass_api/output_formats/output_popup.h @@ -28,9 +28,21 @@ class Tag_Filter struct Category_Filter { + Category_Filter() {} + + ~Category_Filter() + { + for (std::vector< std::vector< Tag_Filter* > >::iterator it_conj = filter_disjunction.begin(); + it_conj != filter_disjunction.end(); ++it_conj) + { + for (std::vector< Tag_Filter* >::iterator it = it_conj->begin(); it != it_conj->end(); ++it) + delete *it; + } + } + void set_title(const std::string& title); void set_title_key(const std::string& title_key); - void add_filter(const std::vector< Tag_Filter >& conjunction) { filter_disjunction.push_back(conjunction); } + void add_filter(const std::vector< Tag_Filter* >& conjunction) { filter_disjunction.push_back(conjunction); } bool consider(const Node_Skeleton& skel, const std::vector< std::pair< std::string, std::string > >* tags); bool consider(const Way_Skeleton& skel, const std::vector< std::pair< std::string, std::string > >* tags); @@ -39,16 +51,25 @@ struct Category_Filter std::string result() const { return output; } private: + Category_Filter(const Category_Filter&); + const Category_Filter& operator=(const Category_Filter&); + std::string output; std::string title_key; - std::vector< std::vector< Tag_Filter > > filter_disjunction; + std::vector< std::vector< Tag_Filter* > > filter_disjunction; }; class Output_Popup : public Output_Handler { public: - Output_Popup(std::vector< Category_Filter > categories_) : categories(categories_) {} + Output_Popup(std::vector< Category_Filter* > categories_) : categories(categories_) {} + + virtual ~Output_Popup() + { + for (std::vector< Category_Filter* >::iterator it = categories.begin(); it != categories.end(); ++it) + delete *it; + } virtual bool write_http_headers(); virtual void write_payload_header(const std::string& db_dir, @@ -102,7 +123,7 @@ class Output_Popup : public Output_Handler Output_Mode mode); private: - std::vector< Category_Filter > categories; + std::vector< Category_Filter* > categories; }; diff --git a/src/overpass_api/output_formats/output_popup_factory.cc b/src/overpass_api/output_formats/output_popup_factory.cc index 5f3ef6c7c..6cfef4737 100644 --- a/src/overpass_api/output_formats/output_popup_factory.cc +++ b/src/overpass_api/output_formats/output_popup_factory.cc @@ -25,21 +25,21 @@ Output_Handler* Output_Popup_Generator::new_output_handler(const std::map< std:: { if (token) { - std::vector< Category_Filter > categories; + std::vector< Category_Filter* > categories; clear_until_after(*token, error_output, "(", "]", false); while (token->good() && **token == "(") { ++(*token); - Category_Filter category; + Category_Filter* category = new Category_Filter(); - category.set_title(get_text_token(*token, error_output, "title")); + category->set_title(get_text_token(*token, error_output, "title")); clear_until_after(*token, error_output, ";", ")", true); while (token->good() && **token == "[") { - std::vector< Tag_Filter > filter_conjunction; + std::vector< Tag_Filter* > filter_conjunction; while (token->good() && **token == "[") { @@ -71,16 +71,17 @@ Output_Handler* Output_Popup_Generator::new_output_handler(const std::map< std:: } clear_until_after(*token, error_output, "]"); - filter_conjunction.push_back(Tag_Filter(key, value, straight)); + filter_conjunction.push_back(new Tag_Filter(key, value, straight)); } clear_until_after(*token, error_output, ";", true); - category.add_filter(filter_conjunction); + category->add_filter(filter_conjunction); } + if (**token != ")") { - category.set_title_key(get_text_token(*token, error_output, "title key")); + category->set_title_key(get_text_token(*token, error_output, "title key")); clear_until_after(*token, error_output, ";", true); } clear_until_after(*token, error_output, ")", true); From 1553a91ef0630c478a793822bd9752578bce6f0f Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Wed, 8 Mar 2017 20:40:25 +0100 Subject: [PATCH 05/13] Fixed further bug in out:popup --- src/overpass_api/output_formats/output_popup.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/overpass_api/output_formats/output_popup.cc b/src/overpass_api/output_formats/output_popup.cc index 258786755..60266713e 100644 --- a/src/overpass_api/output_formats/output_popup.cc +++ b/src/overpass_api/output_formats/output_popup.cc @@ -134,7 +134,7 @@ bool Category_Filter::consider(const Node_Skeleton& skel, if (it_conj != filter_disjunction.end()) output += print(title_key, skel, tags); - return it_conj == filter_disjunction.end(); + return it_conj != filter_disjunction.end(); } @@ -157,7 +157,7 @@ bool Category_Filter::consider(const Way_Skeleton& skel, if (it_conj != filter_disjunction.end()) output += print(title_key, skel, tags); - return it_conj == filter_disjunction.end(); + return it_conj != filter_disjunction.end(); } @@ -180,7 +180,7 @@ bool Category_Filter::consider(const Relation_Skeleton& skel, if (it_conj != filter_disjunction.end()) output += print(title_key, skel, tags); - return it_conj == filter_disjunction.end(); + return it_conj != filter_disjunction.end(); } From 124bac46c6e440dde2993f4a1a1c14eb113f3442 Mon Sep 17 00:00:00 2001 From: W Date: Tue, 14 Mar 2017 00:44:59 +0100 Subject: [PATCH 06/13] Add --enable-lz4 to configure script --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 052e7d663..2731d9ef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN cd /app/src \ && libtoolize \ && automake --add-missing \ && autoconf \ - && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ && make -j $(grep -c ^processor /proc/cpuinfo) install clean \ && apt-get remove -y \ autoconf \ From bc55c489e1f9507b26fb63194a18ac4459727795 Mon Sep 17 00:00:00 2001 From: W Date: Thu, 16 Mar 2017 22:59:28 +0100 Subject: [PATCH 07/13] Copy areas rules to database folder --- docker-entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 5754015a3..e1286361e 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -15,6 +15,7 @@ if [ ! -d /db/db ] ; then fi mkdir -p /db/db \ && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META "--compresion_method=$OVERPASS_COMPRESSION" \ + && cp -r /app/src/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" exit @@ -25,6 +26,7 @@ if [ ! -d /db/db ] ; then && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ + && cp -r /app/src/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" exit From 47f2063bfc6a9eb852aafa0c8ea405c706774a80 Mon Sep 17 00:00:00 2001 From: W Date: Thu, 16 Mar 2017 23:01:22 +0100 Subject: [PATCH 08/13] Add rules load to supervisord --- etc/supervisord.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/supervisord.conf b/etc/supervisord.conf index fa8ed5df9..1f0af3644 100644 --- a/etc/supervisord.conf +++ b/etc/supervisord.conf @@ -36,7 +36,7 @@ redirect_stderr=true priority=6 [program:areas_rules] -command=nice /app/bin/rules_loop.sh /db/db +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s user=overpass redirect_stderr=true priority=7 From 92c015a493e8e1bf516973dbc800a188784e18c7 Mon Sep 17 00:00:00 2001 From: W Date: Fri, 17 Mar 2017 15:50:52 +0100 Subject: [PATCH 09/13] Fix passing compression algorithm. `OVERPASS_COMPRESSION` makes sense only for init_osm3s not for download_clone. Reported by: @mmd-osm --- docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e1286361e..63888636f 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -14,7 +14,7 @@ if [ ! -d /db/db ] ; then META="--meta=$OVERPASS_META" fi mkdir -p /db/db \ - && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META "--compresion_method=$OVERPASS_COMPRESSION" \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ && cp -r /app/src/rules /db/db \ && chown -R overpass:overpass /db \ && echo "Overpass ready, you can start your container with docker start" @@ -23,7 +23,7 @@ if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "init" ]; then lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \ + && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compresion_method=$OVERPASS_COMPRESSION" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ && cp -r /app/src/rules /db/db \ From 9bebfc6585bcba79145f526ee55901424a7eeb6b Mon Sep 17 00:00:00 2001 From: W Date: Fri, 17 Mar 2017 15:59:03 +0100 Subject: [PATCH 10/13] Fix spelling of parameter --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 63888636f..682a1f58f 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -23,7 +23,7 @@ if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "init" ]; then lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compresion_method=$OVERPASS_COMPRESSION" \ + && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compresion-method=$OVERPASS_COMPRESSION" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ && cp -r /app/src/rules /db/db \ From ab7004a3cc4a55a2f158c09aee02d3e30bb37a7c Mon Sep 17 00:00:00 2001 From: Wiktor Niesiobedzki Date: Thu, 27 Jul 2017 10:13:20 +0200 Subject: [PATCH 11/13] Add gecos parameter to adduser --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2731d9ef0..f489616d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM nginx:stable -RUN addgroup overpass && adduser --home /db --disabled-password --ingroup overpass overpass +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ From 96842ddbb4fa85560a2798f9269a6c9984d4f5ad Mon Sep 17 00:00:00 2001 From: Wiktor Niesiobedzki Date: Wed, 30 Aug 2017 09:28:08 +0200 Subject: [PATCH 12/13] Add documentation about OVERPASS_RULES_LOAD --- README_Docker.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README_Docker.md b/README_Docker.md index eb9cb0267..bdc6ae330 100644 --- a/README_Docker.md +++ b/README_Docker.md @@ -9,6 +9,10 @@ To do it properly it needs a bit of guidance in form of enviroment variables. * `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied * `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. Ony useful in `init` mode. Defaults to `gz` +* `OVERPASS_RULES_LOAD` - desired load generated by areas generation. Controls how long the script will sleep before regenerating +areas. Set it to 1, and script will sleep 99x times longer than it works, set it to 50 - and script will sleep as long as last +execution, set it to 100, and script will sleep 3 seconds between each execution. + Image works in two modes `clone` or `init`. This affects how the instance gets initialized. If the mode is set to `clone` then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated from diffs. This will result in Overpass instance From 4736f0fac26dc1367ab704d32d9087fbf3d559e3 Mon Sep 17 00:00:00 2001 From: Wiktor Niesiobedzki Date: Sat, 2 Sep 2017 18:21:19 +0200 Subject: [PATCH 13/13] Fix typo --- docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 682a1f58f..2165974aa 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -23,7 +23,7 @@ if [ ! -d /db/db ] ; then if [ "$OVERPASS_MODE" = "init" ]; then lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ - && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compresion-method=$OVERPASS_COMPRESSION" \ + && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ && rm /db/planet \ && cp -r /app/src/rules /db/db \