From e1629d37e88d3c8eb3a7886d0fadafb2c6493820 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Thu, 5 Apr 2018 23:33:53 +0300 Subject: [PATCH] OMS loader/updater script --- Dockerfile | 9 ++- README.md | 5 ++ openstreetmap-carto.lua.diff | 14 ++++ osm-loader.pl | 125 +++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 openstreetmap-carto.lua.diff create mode 100755 osm-loader.pl diff --git a/Dockerfile b/Dockerfile index 3a313b5..429434a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,8 @@ ENV PARALLEL_BUILD 4 ADD etc /etc RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \ less wget sudo curl unzip gdal-bin mapnik-utils apache2 runit autoconf libtool libmapnik-dev apache2-dev libgdal-dev \ - fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 && \ + fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 \ + osm2pgsql libdbd-pg-perl && \ (curl -L https://deb.nodesource.com/setup_8.x | bash -) && \ DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs build-essential gyp && \ apt-get clean && rm -rf /var/lib/apt/lists/* @@ -54,6 +55,10 @@ RUN cd /tmp && \ RUN mkdir -p /var/lib/mod_tile && chown www-data:www-data /var/lib/mod_tile RUN mkdir -p /var/run/renderd && chown www-data:www-data /var/run/renderd +COPY ./openstreetmap-carto.lua.diff /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION/ +RUN cd /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION && \ + patch openstreetmap-carto.lua < openstreetmap-carto.lua.diff + RUN mkdir -p /etc/service/renderd && mkdir -p /etc/service/apache2 COPY ./apache2/run /etc/service/apache2/run COPY ./renderd/run /etc/service/renderd/run @@ -68,5 +73,7 @@ COPY ./renderd/renderd.conf /usr/local/etc/renderd.conf COPY runit_bootstrap /usr/sbin/runit_bootstrap RUN chmod 755 /usr/sbin/runit_bootstrap +COPY ./osm-loader.pl /osm-loader.pl + EXPOSE 80 ENTRYPOINT ["/usr/sbin/runit_bootstrap"] diff --git a/README.md b/README.md index 7c061df..77b602c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ A basic image for rendering/serving tiles using OpenStreetMap data from an external PostgreSQL instance. + ## Build instructions Build using @@ -37,6 +38,10 @@ and listen address/port with RENDERD_LISTEN_ADDR (default is 0.0.0.0:80). Once the container is up you should be able to see a small map of the world once you point your browser to [http://127.0.0.1:8096/osm/0/0/0.png](http://127.0.0.1:8096/osm/0/0/0.png) +## Loading and updating OSM data + +Use osm-loader.pl (Russia by now). + ## Available Styles * [openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto), diff --git a/openstreetmap-carto.lua.diff b/openstreetmap-carto.lua.diff new file mode 100644 index 0000000..871c2c6 --- /dev/null +++ b/openstreetmap-carto.lua.diff @@ -0,0 +1,14 @@ +--- openstreetmap-carto.lua.orig 2018-04-04 15:59:52.904383753 +0300 ++++ openstreetmap-carto.lua 2018-04-04 15:59:18.364384188 +0300 +@@ -273,6 +273,11 @@ function filter_tags_generic(tags) + return 1, {} + end + ++ -- Remove religion ++ if tags['religion'] ~= nil then ++ return 1, {} ++ end ++ + -- Delete tags listed in delete_tags + for _, d in ipairs(delete_tags) do + tags[d] = nil diff --git a/osm-loader.pl b/osm-loader.pl new file mode 100755 index 0000000..753574a --- /dev/null +++ b/osm-loader.pl @@ -0,0 +1,125 @@ +#!/usr/bin/perl +# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_DIR + same Docker env vars + +use strict; +use DBI; +use POSIX; + +my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads'; +my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION}; + +-e $dir || mkdir($dir); +chdir $dir or die "Failed to chdir $dir"; + +my $dbh = DBI->connect( + 'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.$ENV{PG_ENV_OSM_PORT}, + $ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1} +); +my ($version) = eval { $dbh->selectrow_array( + 'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version' +) }; +if (!$version) +{ + $dbh->rollback; + $dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)'); + if ($ENV{OSM_INIT}) + { + my $cur = time()-86400; + my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'.osm.pbf'; + my $url = 'http://be.gis-lab.info/data/osm_dump/dump/RU/'.$fn; + system("curl -s -C - -f '$url' -o $dir/$fn"); + if (-e "$dir/$fn") + { + $dbh->do('CREATE EXTENSION IF NOT EXISTS postgis'); + $dbh->do('CREATE EXTENSION IF NOT EXISTS hstore'); + $dbh->commit; + $dbh->do( + 'INSERT INTO replication_state (name, value) VALUES (?, ?)', + {}, 'osm_version', POSIX::strftime("%Y-%m-%d", localtime($cur)) + ); + my $cmd = + "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore". + " --style $carto_dir/openstreetmap-carto.style". + " --tag-transform-script $carto_dir/openstreetmap-carto.lua". + " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'". + " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$dir/$fn'"; + system($cmd); + if ($?) + { + print "$cmd failed\n"; + $dbh->rollback; + exit; + } + $dbh->commit; + { + local $/ = undef; + my $fd; + open $fd, "$carto_dir/indexes.sql"; + for my $index (split /;\s*/, <$fd>) + { + $dbh->do($index); + } + close $fd; + } + $dbh->commit; + } + else + { + print "Failed to download $url\n"; + } + } + else + { + print "Current OSM version missing, run with OSM_INIT=1 environment variable to initialize\n"; + } + exit; +} + +my $ymd = [ split /-/, $version ]; +my $cur = POSIX::mktime(0, 0, 0, $ymd->[2]-0, $ymd->[1]-1, $ymd->[0]-1900); +my $now = time(); +my $apply = []; +while ($cur+86400 < $now) +{ + my $next = $cur+86400; + my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'-'.POSIX::strftime("%y%m%d", localtime($next)).'.osc.gz'; + my $url = 'http://be.gis-lab.info/data/osm_dump/diff/RU/'.$fn; + system("curl -C - -s -f '$url' -o $dir/$fn"); + $cur = $next; + if (-e "$dir/$fn") + { + push @$apply, $fn; + } + else + { + last; + } +} +if (@$apply) +{ + my $cmd = + "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o $dir/expire.list -I -s --hstore". + " --style $carto_dir/openstreetmap-carto.style". + " --tag-transform-script $carto_dir/openstreetmap-carto.lua". + " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'". + " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'"; + system($cmd); + if ($?) + { + print "$cmd failed\n"; + $dbh->rollback; + exit; + } + $dbh->do( + 'UPDATE replication_state SET value=? WHERE name=?', + {}, POSIX::strftime("%Y-%m-%d", localtime($cur)), 'osm_version' + ); + $dbh->commit; + if ($ENV{OSM_EXPIRE}) + { + system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11"); + system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11"); + } +} +$dbh->commit; +exit;