#!/usr/bin/perl
# Configuration in env: OSM_CACHE_DIR, OSM_CARTO_VERSION, URL_LATEST, URL_UPDATES,
# plus PG_ENV_OSM_{DB,HOST,PORT,USER,PASSWORD} like in README for the render server itself

use strict;
use DBI;
use POSIX;

my $dbh;
my $dir = $ENV{OSM_CACHE_DIR} || '/var/lib/mod_tile/downloads';
my $method = $ENV{OSM_METHOD} || 'osm2pgsql-carto';
my $url_latest = $ENV{URL_LATEST} || 'http://download.geofabrik.de/russia-latest.osm.pbf';
my $url_updates = $ENV{URL_UPDATES} || 'http://download.geofabrik.de/russia-updates';

if (!$ENV{PG_ENV_OSM_DB})
{
    fatal("OSM database not specified");
}

info("Started OSM update for database: $ENV{PG_ENV_OSM_DB}");

if ($method ne 'osm2pgsql-carto' && $method ne 'imposm3-all')
{
    fatal("Incorrect OSM update mode: $method");
}

-e $dir || mkdir($dir);
chdir $dir or fatal("Failed to chdir $dir");
eval { run_update() };
if ($@)
{
    fatal("$@");
}
exit(0);

sub run_update
{
    my $state = parse_geofabrik_state($url_updates, $dir);
    my $dbh = DBI->connect(
        'dbi:Pg:dbname='.$ENV{PG_ENV_OSM_DB}.';host='.$ENV{PG_ENV_OSM_HOST}.';port='.($ENV{PG_ENV_OSM_PORT}||5432),
        $ENV{PG_ENV_OSM_USER}, $ENV{PG_ENV_OSM_PASSWORD}, {AutoCommit => 0, RaiseError => 1}
    );
    my ($version) = eval { $dbh->selectrow_array(
        'SELECT value FROM replication_state WHERE name=? FOR UPDATE', {}, 'osm_version'
    ) };
    if (!$version)
    {
        if (!$ENV{OSM_INIT})
        {
            fatal("Current OSM version missing, run with OSM_INIT=1 environment variable to initialize");
        }
        $dbh->rollback; # иначе postgresql пишет "statements until the end ignored"
        # создаём таблицу и оставляем её заблокированной
        init_state($dbh, $state);
        $dbh->do('CREATE EXTENSION IF NOT EXISTS postgis');
        $dbh->do('CREATE EXTENSION IF NOT EXISTS hstore');
        # качаем дамп
        my ($fn) = $url_latest =~ /([^\/]+)$/so;
        $fn =~ s/^([^\.]+)/$1-$state->{timestamp}/;
        info("Downloading $url_latest");
        system("curl -s -C - -f '$url_latest' -o $dir/$fn");
        if ($? || !-e "$dir/$fn")
        {
            fatal("Failed to download $url_latest");
        }
        if ($method eq 'osm2pgsql-carto')
        {
            init_osm2pgsql($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
        }
        else
        {
            init_imposm3($dbh, $state->{timestamp} . ' ' . $state->{sequenceNumber}, "$dir/$fn");
        }
        $dbh->commit;
        info("OSM database initialized with version: ".$state->{timestamp});
    }
    else
    {
        my $apply = load_geofabrik_deltas($version, $state, $url_updates, $dir);
        chdir($dir);
        if ($method eq 'osm2pgsql-carto')
        {
            apply_deltas_osm2pgsql($apply);
        }
        else
        {
            apply_deltas_imposm3($apply);
        }
        update_state($dbh, $state);
        $dbh->commit;
        info("OSM database updated to version: ".$state->{timestamp});
        if ($ENV{OSM_EXPIRE})
        {
            system("cat $dir/expire.list | render_expired --map=osm_carto --touch-from=11");
            system("cat $dir/expire.list | render_expired --map=osm_bright --touch-from=11");
        }
    }
}

sub parse_geofabrik_state
{
    my ($url_updates, $dir) = @_;
    system("curl -s -f '$url_updates/state.txt' -o $dir/state.txt");
    if (!-r "$dir/state.txt")
    {
        fatal("Error downloading $url_updates/state.txt");
    }
    else
    {
        my $state;
        if (open FD, "<$dir/state.txt")
        {
            local $/ = undef;
            $state = <FD>;
            close FD;
            $state = { map { (split /\s*=\s*/, $_, 2) } grep { !/^\s*(#.*)?$/so && /=/so } split /\n/, $state };
            if (!$state->{timestamp} || !$state->{sequenceNumber})
            {
                fatal("State file incorrect, should have timestamp=<ISO8601 date> and sequenceNumber=<integer>");
            }
            $state->{timestamp} =~ s/\\//g;
            return $state;
        }
    }
    fatal("Error downloading $url_updates/state.txt");
}

sub load_geofabrik_deltas
{
    my ($version, $state, $url_updates, $dir) = @_;
    my ($timestamp, $i) = split /\s+/, $version;
    my $apply = [];
    while ($i <= $state->{sequenceNumber})
    {
        my $subdir = sprintf("%03d/%03d", $i / 1000000, ($i / 1000) % 1000);
        my $fn = sprintf("%03d.osc.gz", $i % 1000);
        info("Downloading $url_updates/$subdir/$fn");
        system("mkdir -p $dir/$subdir && curl -C - -s -f '$url_updates/$subdir/$fn' -o $dir/$subdir/$fn");
        if (-e "$dir/$subdir/$fn")
        {
            push @$apply, "$subdir/$fn";
        }
        else
        {
            fatal("Delta not available: $url_updates/$subdir/$fn\n");
        }
        $i++;
    }
    return $apply;
}

sub init_state
{
    my ($dbh, $state) = @_;
    $dbh->do('CREATE TABLE IF NOT EXISTS replication_state (name varchar(1024) not null primary key, value text not null)');
    $dbh->do(
        'INSERT INTO replication_state (name, value) VALUES (?, ?)',
        {}, 'osm_version', $state->{timestamp} . ' ' . $state->{sequenceNumber}
    );
}

sub update_state
{
    my ($dbh, $state) = @_;
    $dbh->do(
        'UPDATE replication_state SET value=? WHERE name=?',
        {}, $state->{timestamp} . ' ' . $state->{sequenceNumber}, 'osm_version'
    );
}

sub init_osm2pgsql
{
    my ($dbh, $path) = @_;
    my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
    if (!$ENV{OSM_CARTO_VERSION})
    {
        fatal("osm-carto path not specified");
    }
    my $cmd =
        "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql -I -s -c --hstore".
        " --style $carto_dir/openstreetmap-carto.style".
        " --tag-transform-script $carto_dir/openstreetmap-carto.lua".
        " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
        " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '$path'";
    system($cmd);
    if ($?)
    {
        fatal("$cmd failed");
    }
    local $/ = undef;
    my $fd;
    open $fd, "$carto_dir/indexes.sql";
    for my $index (split /;\s*/, <$fd>)
    {
        $dbh->do($index);
    }
    close $fd;
}

sub init_imposm3
{
    my ($dbh, $path) = @_;
    my $cmd =
        "imposm3 import -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
        "@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
        "/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326 -diff".
        " -read '$path' -write";
    system($cmd);
    if ($?)
    {
        fatal("$cmd failed");
    }
    my $indexes = "SET SEARCH_PATH TO import, public;
CREATE INDEX IF NOT EXISTS osm_polygon_area     ON osm_polygon      (st_area(geometry));
CREATE INDEX IF NOT EXISTS osm_point_tags       ON osm_point        USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_linestring_tags  ON osm_linestring   USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_polygon_tags     ON osm_polygon      USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_relation_tags    ON osm_relation     USING gin (tags);
CREATE INDEX IF NOT EXISTS osm_point_text       ON osm_point        USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_linestring_text  ON osm_linestring   USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_polygon_text     ON osm_polygon      USING gin (to_tsvector('russian', tags::text));
CREATE INDEX IF NOT EXISTS osm_relation_text    ON osm_relation     USING gin (to_tsvector('russian', tags::text))";
    foreach my $index (split /;\n/, $indexes)
    {
        $dbh->do($index);
    }
}

sub apply_deltas_osm2pgsql
{
    my ($apply, $carto_dir) = @_;
    if (@$apply)
    {
        my $cmd =
            "PGPASSWORD='".$ENV{PG_ENV_OSM_PASSWORD}."' osm2pgsql --append -e15 -o '".$ENV{OSM_CACHE_DIR}."/expire.list' -I -s --hstore".
            " --style $carto_dir/openstreetmap-carto.style".
            " --tag-transform-script $carto_dir/openstreetmap-carto.lua".
            " -C 4000 -G -H '".$ENV{PG_ENV_OSM_HOST}."' -U '".$ENV{PG_ENV_OSM_USER}."' -d '".$ENV{PG_ENV_OSM_DB}."'".
            " -P ".($ENV{PG_ENV_OSM_PORT} || 5432)." '".join("' '", @$apply)."'";
        system($cmd);
        if ($?)
        {
            fatal("$cmd failed");
        }
    }
}

sub apply_deltas_imposm3
{
    my ($apply) = @_;
    my $carto_dir = '/usr/share/mapnik/openstreetmap-carto-'.$ENV{OSM_CARTO_VERSION};
    if (!$ENV{OSM_CARTO_VERSION})
    {
        fatal("osm-carto path not specified");
    }
    if (@$apply)
    {
        my $cmd =
            "imposm3 diff -connection 'postgis://".$ENV{PG_ENV_OSM_USER}.":".$ENV{PG_ENV_OSM_PASSWORD}.
            "@".$ENV{PG_ENV_OSM_HOST}.(($ENV{PG_ENV_OSM_PORT}||5432) != 5432 ? ":".$ENV{PG_ENV_OSM_PORT} : "").
            "/".$ENV{PG_ENV_OSM_DB}."' -cachedir '".$ENV{OSM_CACHE_DIR}."/imposm3-cache' -mapping '/home/imposm3-all.yml' -srid 4326".
            " '".join("' '", @$apply)."'";
        system($cmd);
        if ($?)
        {
            fatal("$cmd failed");
        }
    }
}

sub info
{
    my ($msg) = @_;
    print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [info] $msg\n";
}

sub fatal
{
    my ($msg) = @_;
    eval { $dbh->rollback } if $dbh;
    print POSIX::strftime("%Y-%m-%d %H:%M:%S", localtime)." [error] $msg\n";
    exit(1);
}