docker-renderd-osm/osm-loader.pl

#!/usr/bin/perl
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_DIR, URL_LATEST, URL_UPDATES,
# plus PG_ENV_OSM_{DB,HOST,PORT,USER,PASSWORD} like in README for the render server itself

use strict;
use DBI;
use POSIX;

my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
my $url_latest = $ENV{URL_LATEST} || 'http://download.geofabrik.de/russia-latest.osm.pbf';
my $url_updates = $ENV{URL_UPDATES} || 'http://download.geofabrik.de/russia-updates';

-e $dir || mkdir($dir);
chdir $dir or die "Failed to chdir $dir";

my $state = parse_geofabrik_state($url_updates, $dir);
my $dbh = DBI->connect(
    'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.($ENV{PG_ENV_OSM_PORT}||5432),
    $ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
);
my ($version) = eval { $dbh->selectrow_array(
    'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
) };
if (!$version)
{
    if (!$ENV{OSM_INIT})
    {
        die "Current OSM version missing, run with OSM_INIT=1 environment variable to initialize\n";
    }
    $dbh->rollback;
    my ($fn) = $url_latest =~ /([^\/]+)$/so;
    $fn =~ s/^([^\.]+)/$1-$state->{timestamp}/;
    system("curl -s -C - -f '$url_latest' -o $dir/$fn");
    if (!-e "$dir/$fn")
    {
        die "Failed to download $url_latest or $url_updates/state.txt\n";
    }
    load_and_init($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn", $carto_dir);
}
else
{
    my $apply = load_geofabrik_deltas($version, $state, $url_updates, $dir);
    apply_deltas($apply, $state->{timestamp} . ' ' . $state->{sequenceNumber}, $dir, $carto_dir);
}
exit;

sub parse_geofabrik_state
{
    my ($url_updates, $dir) = @_;
    system("curl -s -f '$url_updates/state.txt' -o $dir/state.txt");
    if (!-r "$dir/state.txt")
    {
        die "Error downloading $url_updates/state.txt";
    }
    else
    {
        my $state;
        if (open FD, "<$dir/state.txt")
        {
            local $/ = undef;
            $state = <FD>;
            close FD;
            $state = { map { (split /\s*=\s*/, $_, 2) } grep { !/^\s*(#.*)?$/so && /=/so } split /\n/, $state };
            if (!$state->{timestamp} || !$state->{sequenceNumber})
            {
                print "State file incorrect, should have timestamp=<ISO8601 date> and sequenceNumber=<integer>\n";
                exit;
            }
            $state->{timestamp} =~ s/\\//g;
            return $state;
        }
    }
    die "Error downloading $url_updates/state.txt";
}

sub load_and_init
{
    my ($dbh, $state_text, $path, $carto_dir) = @_;
    $dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
    $dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
    $dbh->commit;
    $dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
    $dbh->do(
        'INSERT INTO replication_state (name, value) VALUES (?, ?)',
        {}, 'osm_version', $state_text
    );
    my $cmd =
        "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
        " --style $carto_dir/openstreetmap-carto.style".
        " --tag-transform-script $carto_dir/openstreetmap-carto.lua".
        " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
        " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$path'";
    system($cmd);
    if ($?)
    {
        print "$cmd failed\n";
        $dbh->rollback;
        exit;
    }
    $dbh->commit;
    {
        local $/ = undef;
        my $fd;
        open $fd, "$carto_dir/indexes.sql";
        for my $index (split /;\s*/, <$fd>)
        {
            $dbh->do($index);
        }
        close $fd;
    }
    $dbh->commit;
}

sub load_geofabrik_deltas
{
    my ($version, $state, $url_updates, $dir) = @_;
    my ($timestamp, $i) = split /\s+/, $version;
    my $apply = [];
    while ($i <= $state->{sequenceNumber})
    {
        my $subdir = sprintf("%03d/%03d", $i / 1000000, ($i / 1000) % 1000);
        my $fn = sprintf("%03d.osc.gz", $i % 1000);
        system("mkdir -p $dir/$subdir && curl -C - -s -f '$url_updates/$subdir/$fn' -o $dir/$subdir/$fn");
        if (-e "$dir/$subdir/$fn")
        {
            push @$apply, "$subdir/$fn";
        }
        else
        {
            die "Delta not available: $url_updates/$subdir/$fn\n";
        }
        $i++;
    }
    return $apply;
}

sub apply_deltas
{
    my ($apply, $state_text, $dir, $carto_dir) = @_;
    if (@$apply)
    {
        chdir($dir);
        my $cmd =
            "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o $dir/expire.list -I -s --hstore".
            " --style $carto_dir/openstreetmap-carto.style".
            " --tag-transform-script $carto_dir/openstreetmap-carto.lua".
            " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
            " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
        system($cmd);
        if ($?)
        {
            print "$cmd failed\n";
            $dbh->rollback;
            exit;
        }
        $dbh->do(
            'UPDATE replication_state SET value=? WHERE name=?',
            {}, $state_text, 'osm_version'
        );
        $dbh->commit;
        if ($ENV{OSM_EXPIRE})
        {
            system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
            system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
        }
    }
    $dbh->commit;
}