OMS loader/updater script
parent
7893db017e
commit
e1629d37e8
|
@ -12,7 +12,8 @@ ENV PARALLEL_BUILD 4
|
||||||
ADD etc /etc
|
ADD etc /etc
|
||||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
|
||||||
less wget sudo curl unzip gdal-bin mapnik-utils apache2 runit autoconf libtool libmapnik-dev apache2-dev libgdal-dev \
|
less wget sudo curl unzip gdal-bin mapnik-utils apache2 runit autoconf libtool libmapnik-dev apache2-dev libgdal-dev \
|
||||||
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 && \
|
fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-hanazono ttf-unifont ca-certificates gnupg2 \
|
||||||
|
osm2pgsql libdbd-pg-perl && \
|
||||||
(curl -L https://deb.nodesource.com/setup_8.x | bash -) && \
|
(curl -L https://deb.nodesource.com/setup_8.x | bash -) && \
|
||||||
DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs build-essential gyp && \
|
DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs build-essential gyp && \
|
||||||
apt-get clean && rm -rf /var/lib/apt/lists/*
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||||
|
@ -54,6 +55,10 @@ RUN cd /tmp && \
|
||||||
RUN mkdir -p /var/lib/mod_tile && chown www-data:www-data /var/lib/mod_tile
|
RUN mkdir -p /var/lib/mod_tile && chown www-data:www-data /var/lib/mod_tile
|
||||||
RUN mkdir -p /var/run/renderd && chown www-data:www-data /var/run/renderd
|
RUN mkdir -p /var/run/renderd && chown www-data:www-data /var/run/renderd
|
||||||
|
|
||||||
|
COPY ./openstreetmap-carto.lua.diff /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION/
|
||||||
|
RUN cd /usr/share/mapnik/openstreetmap-carto-$OSM_CARTO_VERSION && \
|
||||||
|
patch openstreetmap-carto.lua < openstreetmap-carto.lua.diff
|
||||||
|
|
||||||
RUN mkdir -p /etc/service/renderd && mkdir -p /etc/service/apache2
|
RUN mkdir -p /etc/service/renderd && mkdir -p /etc/service/apache2
|
||||||
COPY ./apache2/run /etc/service/apache2/run
|
COPY ./apache2/run /etc/service/apache2/run
|
||||||
COPY ./renderd/run /etc/service/renderd/run
|
COPY ./renderd/run /etc/service/renderd/run
|
||||||
|
@ -68,5 +73,7 @@ COPY ./renderd/renderd.conf /usr/local/etc/renderd.conf
|
||||||
COPY runit_bootstrap /usr/sbin/runit_bootstrap
|
COPY runit_bootstrap /usr/sbin/runit_bootstrap
|
||||||
RUN chmod 755 /usr/sbin/runit_bootstrap
|
RUN chmod 755 /usr/sbin/runit_bootstrap
|
||||||
|
|
||||||
|
COPY ./osm-loader.pl /osm-loader.pl
|
||||||
|
|
||||||
EXPOSE 80
|
EXPOSE 80
|
||||||
ENTRYPOINT ["/usr/sbin/runit_bootstrap"]
|
ENTRYPOINT ["/usr/sbin/runit_bootstrap"]
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
A basic image for rendering/serving tiles using OpenStreetMap data from an external PostgreSQL instance.
|
A basic image for rendering/serving tiles using OpenStreetMap data from an external PostgreSQL instance.
|
||||||
|
|
||||||
|
|
||||||
## Build instructions
|
## Build instructions
|
||||||
|
|
||||||
Build using
|
Build using
|
||||||
|
@ -37,6 +38,10 @@ and listen address/port with RENDERD_LISTEN_ADDR (default is 0.0.0.0:80).
|
||||||
Once the container is up you should be able to see a small map of the
|
Once the container is up you should be able to see a small map of the
|
||||||
world once you point your browser to [http://127.0.0.1:8096/osm/0/0/0.png](http://127.0.0.1:8096/osm/0/0/0.png)
|
world once you point your browser to [http://127.0.0.1:8096/osm/0/0/0.png](http://127.0.0.1:8096/osm/0/0/0.png)
|
||||||
|
|
||||||
|
## Loading and updating OSM data
|
||||||
|
|
||||||
|
Use osm-loader.pl (Russia by now).
|
||||||
|
|
||||||
## Available Styles
|
## Available Styles
|
||||||
|
|
||||||
* [openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto),
|
* [openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto),
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
--- openstreetmap-carto.lua.orig 2018-04-04 15:59:52.904383753 +0300
|
||||||
|
+++ openstreetmap-carto.lua 2018-04-04 15:59:18.364384188 +0300
|
||||||
|
@@ -273,6 +273,11 @@ function filter_tags_generic(tags)
|
||||||
|
return 1, {}
|
||||||
|
end
|
||||||
|
|
||||||
|
+ -- Remove religion
|
||||||
|
+ if tags['religion'] ~= nil then
|
||||||
|
+ return 1, {}
|
||||||
|
+ end
|
||||||
|
+
|
||||||
|
-- Delete tags listed in delete_tags
|
||||||
|
for _, d in ipairs(delete_tags) do
|
||||||
|
tags[d] = nil
|
|
@ -0,0 +1,125 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_DIR + same Docker env vars
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use DBI;
|
||||||
|
use POSIX;
|
||||||
|
|
||||||
|
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
|
||||||
|
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
|
||||||
|
|
||||||
|
-e $dir || mkdir($dir);
|
||||||
|
chdir $dir or die "Failed to chdir $dir";
|
||||||
|
|
||||||
|
my $dbh = DBI->connect(
|
||||||
|
'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.$ENV{PG_ENV_OSM_PORT},
|
||||||
|
$ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
|
||||||
|
);
|
||||||
|
my ($version) = eval { $dbh->selectrow_array(
|
||||||
|
'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
|
||||||
|
) };
|
||||||
|
if (!$version)
|
||||||
|
{
|
||||||
|
$dbh->rollback;
|
||||||
|
$dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
|
||||||
|
if ($ENV{OSM_INIT})
|
||||||
|
{
|
||||||
|
my $cur = time()-86400;
|
||||||
|
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'.osm.pbf';
|
||||||
|
my $url = 'http://be.gis-lab.info/data/osm_dump/dump/RU/'.$fn;
|
||||||
|
system("curl -s -C - -f '$url' -o $dir/$fn");
|
||||||
|
if (-e "$dir/$fn")
|
||||||
|
{
|
||||||
|
$dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
|
||||||
|
$dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
|
||||||
|
$dbh->commit;
|
||||||
|
$dbh->do(
|
||||||
|
'INSERT INTO replication_state (name, value) VALUES (?, ?)',
|
||||||
|
{}, 'osm_version', POSIX::strftime("%Y-%m-%d", localtime($cur))
|
||||||
|
);
|
||||||
|
my $cmd =
|
||||||
|
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
|
||||||
|
" --style $carto_dir/openstreetmap-carto.style".
|
||||||
|
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
||||||
|
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
||||||
|
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$dir/$fn'";
|
||||||
|
system($cmd);
|
||||||
|
if ($?)
|
||||||
|
{
|
||||||
|
print "$cmd failed\n";
|
||||||
|
$dbh->rollback;
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$dbh->commit;
|
||||||
|
{
|
||||||
|
local $/ = undef;
|
||||||
|
my $fd;
|
||||||
|
open $fd, "$carto_dir/indexes.sql";
|
||||||
|
for my $index (split /;\s*/, <$fd>)
|
||||||
|
{
|
||||||
|
$dbh->do($index);
|
||||||
|
}
|
||||||
|
close $fd;
|
||||||
|
}
|
||||||
|
$dbh->commit;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
print "Failed to download $url\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
print "Current OSM version missing, run with OSM_INIT=1 environment variable to initialize\n";
|
||||||
|
}
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $ymd = [ split /-/, $version ];
|
||||||
|
my $cur = POSIX::mktime(0, 0, 0, $ymd->[2]-0, $ymd->[1]-1, $ymd->[0]-1900);
|
||||||
|
my $now = time();
|
||||||
|
my $apply = [];
|
||||||
|
while ($cur+86400 < $now)
|
||||||
|
{
|
||||||
|
my $next = $cur+86400;
|
||||||
|
my $fn = 'RU-'.POSIX::strftime("%y%m%d", localtime($cur)).'-'.POSIX::strftime("%y%m%d", localtime($next)).'.osc.gz';
|
||||||
|
my $url = 'http://be.gis-lab.info/data/osm_dump/diff/RU/'.$fn;
|
||||||
|
system("curl -C - -s -f '$url' -o $dir/$fn");
|
||||||
|
$cur = $next;
|
||||||
|
if (-e "$dir/$fn")
|
||||||
|
{
|
||||||
|
push @$apply, $fn;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (@$apply)
|
||||||
|
{
|
||||||
|
my $cmd =
|
||||||
|
"PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o $dir/expire.list -I -s --hstore".
|
||||||
|
" --style $carto_dir/openstreetmap-carto.style".
|
||||||
|
" --tag-transform-script $carto_dir/openstreetmap-carto.lua".
|
||||||
|
" -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
|
||||||
|
" -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
|
||||||
|
system($cmd);
|
||||||
|
if ($?)
|
||||||
|
{
|
||||||
|
print "$cmd failed\n";
|
||||||
|
$dbh->rollback;
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$dbh->do(
|
||||||
|
'UPDATE replication_state SET value=? WHERE name=?',
|
||||||
|
{}, POSIX::strftime("%Y-%m-%d", localtime($cur)), 'osm_version'
|
||||||
|
);
|
||||||
|
$dbh->commit;
|
||||||
|
if ($ENV{OSM_EXPIRE})
|
||||||
|
{
|
||||||
|
system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
|
||||||
|
system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$dbh->commit;
|
||||||
|
exit;
|
Loading…
Reference in New Issue