vitalif
/
docker-renderd-osm
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

OMS loader/updater script

master
Vitaliy Filippov 2018-04-05 23:33:53 +03:00
parent 7893db017e
commit e1629d37e8
4 changed files with 152 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Dockerfile
View File

@ -12,7 +12,8 @@ ENV PARALLEL_BUILD 4
ADD etc /etc
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
less wget sudo curl unzip gdal-bin mapnik-utils apache2 runit autoconf libtool libmapnik-dev apache2-dev libgdal-dev \
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 && \
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 \
osm2pgsql libdbd-pg-perl && \
(curl -L https://deb.nodesource.com/setup_8.x | bash -) && \
DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs build-essential gyp && \
apt-get clean && rm -rf /var/lib/apt/lists/*
@ -54,6 +55,10 @@ RUN cd /tmp && \
RUN mkdir -p /var/lib/mod_tile && chown www-data:www-data /var/lib/mod_tile
RUN mkdir -p /var/run/renderd && chown www-data:www-data /var/run/renderd
COPY ./openstreetmap-carto.lua.diff /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION/
RUN cd /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION && \
patch openstreetmap-carto.lua < openstreetmap-carto.lua.diff
RUN mkdir -p /etc/service/renderd && mkdir -p /etc/service/apache2
COPY ./apache2/run /etc/service/apache2/run
COPY ./renderd/run /etc/service/renderd/run
@ -68,5 +73,7 @@ COPY ./renderd/renderd.conf /usr/local/etc/renderd.conf
COPY runit_bootstrap /usr/sbin/runit_bootstrap
RUN chmod 755 /usr/sbin/runit_bootstrap
COPY ./osm-loader.pl /osm-loader.pl
EXPOSE 80
ENTRYPOINT ["/usr/sbin/runit_bootstrap"]

5
README.md
View File

@ -2,6 +2,7 @@
A basic image for rendering/serving tiles using OpenStreetMap data from an external PostgreSQL instance.
## Build instructions
Build using
@ -37,6 +38,10 @@ and listen address/port with RENDERD_LISTEN_ADDR (default is 0.0.0.0:80).
Once the container is up you should be able to see a small map of the
world once you point your browser to [http://127.0.0.1:8096/osm/0/0/0.png](http://127.0.0.1:8096/osm/0/0/0.png)
## Loading and updating OSM data
Use osm-loader.pl (Russia by now).
## Available Styles
* [openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto),

14
openstreetmap-carto.lua.diff Normal file
View File

@ -0,0 +1,14 @@
--- openstreetmap-carto.lua.orig 2018-04-04 15:59:52.904383753 +0300
+++ openstreetmap-carto.lua 2018-04-04 15:59:18.364384188 +0300
@@ -273,6 +273,11 @@ function filter_tags_generic(tags)
return 1, {}
end
+ -- Remove religion
+ if tags['religion'] ~= nil then
+ return 1, {}
+ end
+
-- Delete tags listed in delete_tags
for _, d in ipairs(delete_tags) do
tags[d] = nil

125
osm-loader.pl Executable file
View File

@ -0,0 +1,125 @@
#!/usr/bin/perl
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_DIR + same Docker env vars
use strict;
use DBI;
use POSIX;
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
-e $dir || mkdir($dir);
chdir $dir or die "Failed to chdir $dir";
my $dbh = DBI->connect(
'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.$ENV{PG_ENV_OSM_PORT},
$ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
);
my ($version) = eval { $dbh->selectrow_array(
'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
) };
if (!$version)
{
$dbh->rollback;
$dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
if ($ENV{OSM_INIT})
{
my $cur = time()-86400;
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'.osm.pbf';
my $url = 'http://be.gis-lab.info/data/osm_dump/dump/RU/'.$fn;
system("curl -s -C - -f '$url' -o $dir/$fn");
if (-e "$dir/$fn")
{
$dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
$dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
$dbh->commit;
$dbh->do(
'INSERT INTO replication_state (name, value) VALUES (?, ?)',
{}, 'osm_version', POSIX::strftime("%Y-%m-%d", localtime($cur))
);
my $cmd =
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
" --style $carto_dir/openstreetmap-carto.style".
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$dir/$fn'";
system($cmd);
if ($?)
{
print "$cmd failed\n";
$dbh->rollback;
exit;
}
$dbh->commit;
{
local $/ = undef;
my $fd;
open $fd, "$carto_dir/indexes.sql";
for my $index (split /;\s*/, <$fd>)
{
$dbh->do($index);
}
close $fd;
}
$dbh->commit;
}
else
{
print "Failed to download $url\n";
}
}
else
{
print "Current OSM version missing, run with OSM_INIT=1 environment variable to initialize\n";
}
exit;
}
my $ymd = [ split /-/, $version ];
my $cur = POSIX::mktime(0, 0, 0, $ymd->[2]-0, $ymd->[1]-1, $ymd->[0]-1900);
my $now = time();
my $apply = [];
while ($cur+86400 < $now)
{
my $next = $cur+86400;
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'-'.POSIX::strftime("%y%m%d", localtime($next)).'.osc.gz';
my $url = 'http://be.gis-lab.info/data/osm_dump/diff/RU/'.$fn;
system("curl -C - -s -f '$url' -o $dir/$fn");
$cur = $next;
if (-e "$dir/$fn")
{
push @$apply, $fn;
}
else
{
last;
}
}
if (@$apply)
{
my $cmd =
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o $dir/expire.list -I -s --hstore".
" --style $carto_dir/openstreetmap-carto.style".
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
system($cmd);
if ($?)
{
print "$cmd failed\n";
$dbh->rollback;
exit;
}
$dbh->do(
'UPDATE replication_state SET value=? WHERE name=?',
{}, POSIX::strftime("%Y-%m-%d", localtime($cur)), 'osm_version'
);
$dbh->commit;
if ($ENV{OSM_EXPIRE})
{
system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
}
}
$dbh->commit;
exit;