OMS loader/updater script
parent
7893db017e
commit
e1629d37e8
|
@ -12,7 +12,8 @@ ENV PARALLEL_BUILD 4
|
|||
ADD etc /etc
|
||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
|
||||
less wget sudo curl unzip gdal-bin mapnik-utils apache2 runit autoconf libtool libmapnik-dev apache2-dev libgdal-dev \
|
||||
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 && \
|
||||
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 \
|
||||
osm2pgsql libdbd-pg-perl && \
|
||||
(curl -L https://deb.nodesource.com/setup_8.x | bash -) && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs build-essential gyp && \
|
||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
@ -54,6 +55,10 @@ RUN cd /tmp && \
|
|||
RUN mkdir -p /var/lib/mod_tile && chown www-data:www-data /var/lib/mod_tile
|
||||
RUN mkdir -p /var/run/renderd && chown www-data:www-data /var/run/renderd
|
||||
|
||||
COPY ./openstreetmap-carto.lua.diff /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION/
|
||||
RUN cd /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION && \
|
||||
patch openstreetmap-carto.lua < openstreetmap-carto.lua.diff
|
||||
|
||||
RUN mkdir -p /etc/service/renderd && mkdir -p /etc/service/apache2
|
||||
COPY ./apache2/run /etc/service/apache2/run
|
||||
COPY ./renderd/run /etc/service/renderd/run
|
||||
|
@ -68,5 +73,7 @@ COPY ./renderd/renderd.conf /usr/local/etc/renderd.conf
|
|||
COPY runit_bootstrap /usr/sbin/runit_bootstrap
|
||||
RUN chmod 755 /usr/sbin/runit_bootstrap
|
||||
|
||||
COPY ./osm-loader.pl /osm-loader.pl
|
||||
|
||||
EXPOSE 80
|
||||
ENTRYPOINT ["/usr/sbin/runit_bootstrap"]
|
||||
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
A basic image for rendering/serving tiles using OpenStreetMap data from an external PostgreSQL instance.
|
||||
|
||||
|
||||
## Build instructions
|
||||
|
||||
Build using
|
||||
|
@ -37,6 +38,10 @@ and listen address/port with RENDERD_LISTEN_ADDR (default is 0.0.0.0:80).
|
|||
Once the container is up you should be able to see a small map of the
|
||||
world once you point your browser to [http://127.0.0.1:8096/osm/0/0/0.png](http://127.0.0.1:8096/osm/0/0/0.png)
|
||||
|
||||
## Loading and updating OSM data
|
||||
|
||||
Use osm-loader.pl (Russia by now).
|
||||
|
||||
## Available Styles
|
||||
|
||||
* [openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto),
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
--- openstreetmap-carto.lua.orig 2018-04-04 15:59:52.904383753 +0300
|
||||
+++ openstreetmap-carto.lua 2018-04-04 15:59:18.364384188 +0300
|
||||
@@ -273,6 +273,11 @@ function filter_tags_generic(tags)
|
||||
return 1, {}
|
||||
end
|
||||
|
||||
+ -- Remove religion
|
||||
+ if tags['religion'] ~= nil then
|
||||
+ return 1, {}
|
||||
+ end
|
||||
+
|
||||
-- Delete tags listed in delete_tags
|
||||
for _, d in ipairs(delete_tags) do
|
||||
tags[d] = nil
|
|
@ -0,0 +1,125 @@
|
|||
#!/usr/bin/perl
|
||||
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_DIR + same Docker env vars
|
||||
|
||||
use strict;
|
||||
use DBI;
|
||||
use POSIX;
|
||||
|
||||
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
|
||||
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
|
||||
|
||||
-e $dir || mkdir($dir);
|
||||
chdir $dir or die "Failed to chdir $dir";
|
||||
|
||||
my $dbh = DBI->connect(
|
||||
'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.$ENV{PG_ENV_OSM_PORT},
|
||||
$ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
|
||||
);
|
||||
my ($version) = eval { $dbh->selectrow_array(
|
||||
'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
|
||||
) };
|
||||
if (!$version)
|
||||
{
|
||||
$dbh->rollback;
|
||||
$dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
|
||||
if ($ENV{OSM_INIT})
|
||||
{
|
||||
my $cur = time()-86400;
|
||||
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'.osm.pbf';
|
||||
my $url = 'http://be.gis-lab.info/data/osm_dump/dump/RU/'.$fn;
|
||||
system("curl -s -C - -f '$url' -o $dir/$fn");
|
||||
if (-e "$dir/$fn")
|
||||
{
|
||||
$dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
|
||||
$dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
|
||||
$dbh->commit;
|
||||
$dbh->do(
|
||||
'INSERT INTO replication_state (name, value) VALUES (?, ?)',
|
||||
{}, 'osm_version', POSIX::strftime("%Y-%m-%d", localtime($cur))
|
||||
);
|
||||
my $cmd =
|
||||
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
|
||||
" --style $carto_dir/openstreetmap-carto.style".
|
||||
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
||||
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
||||
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$dir/$fn'";
|
||||
system($cmd);
|
||||
if ($?)
|
||||
{
|
||||
print "$cmd failed\n";
|
||||
$dbh->rollback;
|
||||
exit;
|
||||
}
|
||||
$dbh->commit;
|
||||
{
|
||||
local $/ = undef;
|
||||
my $fd;
|
||||
open $fd, "$carto_dir/indexes.sql";
|
||||
for my $index (split /;\s*/, <$fd>)
|
||||
{
|
||||
$dbh->do($index);
|
||||
}
|
||||
close $fd;
|
||||
}
|
||||
$dbh->commit;
|
||||
}
|
||||
else
|
||||
{
|
||||
print "Failed to download $url\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
print "Current OSM version missing, run with OSM_INIT=1 environment variable to initialize\n";
|
||||
}
|
||||
exit;
|
||||
}
|
||||
|
||||
my $ymd = [ split /-/, $version ];
|
||||
my $cur = POSIX::mktime(0, 0, 0, $ymd->[2]-0, $ymd->[1]-1, $ymd->[0]-1900);
|
||||
my $now = time();
|
||||
my $apply = [];
|
||||
while ($cur+86400 < $now)
|
||||
{
|
||||
my $next = $cur+86400;
|
||||
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'-'.POSIX::strftime("%y%m%d", localtime($next)).'.osc.gz';
|
||||
my $url = 'http://be.gis-lab.info/data/osm_dump/diff/RU/'.$fn;
|
||||
system("curl -C - -s -f '$url' -o $dir/$fn");
|
||||
$cur = $next;
|
||||
if (-e "$dir/$fn")
|
||||
{
|
||||
push @$apply, $fn;
|
||||
}
|
||||
else
|
||||
{
|
||||
last;
|
||||
}
|
||||
}
|
||||
if (@$apply)
|
||||
{
|
||||
my $cmd =
|
||||
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o $dir/expire.list -I -s --hstore".
|
||||
" --style $carto_dir/openstreetmap-carto.style".
|
||||
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
||||
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
||||
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
|
||||
system($cmd);
|
||||
if ($?)
|
||||
{
|
||||
print "$cmd failed\n";
|
||||
$dbh->rollback;
|
||||
exit;
|
||||
}
|
||||
$dbh->do(
|
||||
'UPDATE replication_state SET value=? WHERE name=?',
|
||||
{}, POSIX::strftime("%Y-%m-%d", localtime($cur)), 'osm_version'
|
||||
);
|
||||
$dbh->commit;
|
||||
if ($ENV{OSM_EXPIRE})
|
||||
{
|
||||
system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
|
||||
system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
|
||||
}
|
||||
}
|
||||
$dbh->commit;
|
||||
exit;
|
Loading…
Reference in New Issue