From bfae91e6e02f81413d8d3a8da921df8e9b5c5b4d Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 4 Sep 2021 10:28:39 +0300 Subject: [PATCH] Connect to external postgresql, add retina tiles, change mod_tile to standard --- Dockerfile | 48 +++-------- README.md | 191 ++++++++++++----------------------------- external-data.yml.diff | 13 +++ leaflet-demo.html | 2 +- run.sh | 116 ++++++++++++------------- 5 files changed, 141 insertions(+), 229 deletions(-) create mode 100644 external-data.yml.diff diff --git a/Dockerfile b/Dockerfile index b4565be..6fb7642 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,16 @@ -FROM ubuntu:20.04 +FROM debian:bullseye # Based on # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/ # Set up environment ENV TZ=UTC -ENV AUTOVACUUM=on ENV UPDATES=disabled RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Install dependencies -RUN apt-get update \ - && apt-get install -y wget gnupg2 lsb-core apt-transport-https ca-certificates curl \ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ - && echo "deb [ trusted=yes ] https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \ - && wget --quiet -O - https://deb.nodesource.com/setup_14.x | bash - \ - && apt-get update \ - && apt-get install -y nodejs +RUN echo "Install-Recommends false;" > /etc/apt.conf && apt-get update \ + && apt-get install -y wget gnupg2 apt-transport-https ca-certificates curl nodejs npm node-gyp node-pre-gyp RUN apt-get install -y --no-install-recommends \ apache2 \ @@ -52,19 +46,20 @@ RUN apt-get install -y --no-install-recommends \ libtiff5-dev \ libtool \ libxml2-dev \ + libiniparser-dev \ lua5.3 \ make \ mapnik-utils \ osmium-tool \ osmosis \ postgis \ - postgresql-12 \ - postgresql-contrib-12 \ - postgresql-server-dev-12 \ + postgresql-client-13 \ protobuf-c-compiler \ python-is-python3 \ python3-mapnik \ python3-lxml \ + python3-yaml \ + python3-requests \ python3-psycopg2 \ python3-shapely \ python3-pip \ @@ -78,21 +73,6 @@ RUN apt-get install -y --no-install-recommends \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ -# Install python libraries -RUN pip3 install requests \ - && pip3 install pyyaml - -# Set up PostGIS -RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \ - && mkdir -p postgis_src \ - && tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \ - && rm postgis.tar.gz \ - && cd postgis_src \ - && ./configure \ - && make -j $(nproc) \ - && make -j $(nproc) install \ - && cd .. && rm -rf postgis_src - # Set up renderer user RUN adduser --disabled-password --gecos "" renderer @@ -114,7 +94,7 @@ RUN mkdir -p /home/renderer/src \ # Install mod_tile and renderd RUN mkdir -p /home/renderer/src \ && cd /home/renderer/src \ - && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \ + && git clone https://github.com/openstreetmap/mod_tile.git --depth 1 \ && cd mod_tile \ && rm -rf .git \ && ./autogen.sh \ @@ -152,13 +132,6 @@ COPY leaflet-demo.html /var/www/html/index.html RUN ln -sf /dev/stdout /var/log/apache2/access.log \ && ln -sf /dev/stderr /var/log/apache2/error.log -# Configure PosgtreSQL -COPY postgresql.custom.conf.tmpl /etc/postgresql/12/main/ -RUN chown -R postgres:postgres /var/lib/postgresql \ - && chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \ - && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \ - && echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf - # Copy update scripts COPY openstreetmap-tiles-update-expire /usr/bin/ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ @@ -176,9 +149,12 @@ RUN mkdir -p /home/renderer/src \ && rm -rf .git \ && chmod u+x /home/renderer/src/regional/trim_osc.py +COPY ./external-data.yml.diff /home/renderer/src/openstreetmap-carto +RUN cd /home/renderer/src/openstreetmap-carto && patch -p1 < external-data.yml.diff + # Start running COPY run.sh / ENTRYPOINT ["/run.sh"] CMD [] -EXPOSE 80 5432 +EXPOSE 80 diff --git a/README.md b/README.md index 4d6f47b..7c0a652 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,39 @@ # openstreetmap-tile-server -[![Build Status](https://travis-ci.org/Overv/openstreetmap-tile-server.svg?branch=master)](https://travis-ci.org/Overv/openstreetmap-tile-server) [![](https://images.microbadger.com/badges/image/overv/openstreetmap-tile-server.svg)](https://microbadger.com/images/overv/openstreetmap-tile-server "openstreetmap-tile-server") - This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style. -## Setting up the server +## Set up PostgreSQL -First create a Docker volume to hold the PostgreSQL database that will contain the OpenStreetMap data: - - docker volume create openstreetmap-data - -Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example: +* Create a database +* Create extensions `postgis` and `hstore` within it +* Create a user and grant him access to the created database ``` -docker run \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - overv/openstreetmap-tile-server \ - import +CREATE DATABASE osm; +CREATE USER renderd WITH PASSWORD ''; +GRANT ALL PRIVILEGES ON DATABASE osm TO renderd; +CREATE EXTENSION postgis; +CREATE EXTENSION hstore; ``` -If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server. - -### Automatic updates (optional) - -If your import is an extract of the planet and has polygonal bounds associated with it, like those from geofabrik.de, then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the import process to facilitate this, and also include the `UPDATES=enabled` variable: +## Running the initial import ``` -docker run \ +podman run \ + -it \ + -e PGPASSWORD= \ + -e PGHOST= \ + -e PGUSER=renderd \ + -e PGPORT=5432 \ + -e PGDB=osm \ + -e DOWNLOAD_PBF=https://download.geofabrik.de/russia-latest.osm.pbf \ + -e DOWNLOAD_POLY=https://download.geofabrik.de/russia.poly \ -e UPDATES=enabled \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ - -v /absolute/path/to/luxembourg.poly:/data.poly \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - overv/openstreetmap-tile-server \ - import -``` - -Refer to the section *Automatic updating and tile expiry* to actually enable the updates while running the tile server. - -### Letting the container download the file - -It is also possible to let the container download files for you rather than mounting them in advance by using the `DOWNLOAD_PBF` and `DOWNLOAD_POLY` parameters: - -``` -docker run \ - -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \ - -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - overv/openstreetmap-tile-server \ + -v /run/postgresql:/run/postgresql \ + -v /home/osm/download:/data \ + -v /home/osm/mod_tile:/var/lib/mod_tile \ + --rm \ + osm \ import ``` @@ -56,92 +42,36 @@ docker run \ Run the server like this: ``` -docker run \ - -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ - run -``` - -Your tiles will now be available at `http://localhost:8080/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:8080`. Note that it will initially take quite a bit of time to render the larger tiles for the first time. - -### Using Docker Compose - -The `docker-compose.yml` file included with this repository shows how the aforementioned command can be used with Docker Compose to run your server. - -### Preserving rendered tiles - -Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it: - -``` -docker volume create openstreetmap-rendered-tiles -docker run \ - -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ - -d overv/openstreetmap-tile-server \ - run -``` - -**If you do this, then make sure to also run the import with the `openstreetmap-rendered-tiles` volume to make sure that caching works properly across updates!** - -### Enabling automatic updating (optional) - -Given that you've set up your import as described in the *Automatic updates* section during server setup, you can enable the updating process by setting the `UPDATES` variable while running your server as well: - -``` -docker run \ - -p 8080:80 \ +podman run \ + -d \ + -e PGPASSWORD= \ + -e PGHOST= \ + -e PGUSER=renderd \ + -e PGPORT=5432 \ + -e PGDB=osm \ -e UPDATES=enabled \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ - -d overv/openstreetmap-tile-server \ + -v /run/postgresql:/run/postgresql \ + -v /home/osm/download:/data \ + -v /home/osm/mod_tile:/var/lib/mod_tile \ + -p 8080:80 \ + osm \ run ``` -This will enable a background process that automatically downloads changes from the OpenStreetMap server, filters them for the relevant region polygon you specified, updates the database and finally marks the affected tiles for rerendering. +Your tiles will now be available at `http://localhost:8080/osm/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:8080`. Note that it will initially take quite a bit of time to render the larger tiles for the first time. + +`UPDATES=enabled` will enable a background process that automatically downloads changes from the OpenStreetMap server, filters them for the relevant region polygon you specified, updates the database and finally marks the affected tiles for rerendering. ### Cross-origin resource sharing To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles from other domains, simply set the `ALLOW_CORS` variable to `enabled`: ``` -docker run \ +podman run \ -p 8080:80 \ -v openstreetmap-data:/var/lib/postgresql/12/main \ -e ALLOW_CORS=enabled \ - -d overv/openstreetmap-tile-server \ - run -``` - -### Connecting to Postgres - -To connect to the PostgreSQL database inside the container, make sure to expose port 5432: - -``` -docker run \ - -p 8080:80 \ - -p 5432:5432 \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ - run -``` - -Use the user `renderer` and the database `gis` to connect. - -``` -psql -h localhost -U renderer gis -``` - -The default password is `renderer`, but it can be changed using the `PGPASSWORD` environment variable: - -``` -docker run \ - -p 8080:80 \ - -p 5432:5432 \ - -e PGPASSWORD=secret \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ + -d osm \ run ``` @@ -149,39 +79,31 @@ docker run \ Details for update procedure and invoked scripts can be found here [link](https://ircama.github.io/osm-carto-tutorials/updating-data/). +### POSTGRESQL + +https://pgtune.leopard.in.ua + ### THREADS The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example: ``` -docker run \ +podman run \ -p 8080:80 \ -e THREADS=24 \ -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ + -d osm \ run ``` ### CACHE -The import and tile serving processes use 800 MB RAM cache by default, but this number can be changed by option -C. For example: +This image uses 4096 MB RAM cache by default for osm2pgsql, but this number can be changed by option -C. For example: ``` -docker run \ +podman run \ -p 8080:80 \ - -e "OSM2PGSQL_EXTRA_ARGS=-C 4096" \ + -e "OSM2PGSQL_EXTRA_ARGS=-C 8192" \ -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ - run -``` - -### AUTOVACUUM - -The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example: -``` -docker run \ - -p 8080:80 \ - -e AUTOVACUUM=off \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ - -d overv/openstreetmap-tile-server \ + -d osm \ run ``` @@ -190,12 +112,12 @@ docker run \ If you are planning to import the entire planet or you are running into memory errors then you may want to enable the `--flat-nodes` option for osm2pgsql. You can then use it during the import process as follows: ``` -docker run \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ +podman run \ + -v /absolute/path/to/luxembourg.osm.pbf:/data/data.osm.pbf \ -v openstreetmap-nodes:/nodes \ -v openstreetmap-data:/var/lib/postgresql/12/main \ -e "OSM2PGSQL_EXTRA_ARGS=--flat-nodes /nodes/flat_nodes.bin" \ - overv/openstreetmap-tile-server \ + osm \ import ``` @@ -216,11 +138,11 @@ renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segm ``` To raise it use `--shm-size` parameter. For example: ``` -docker run \ +podman run \ -p 8080:80 \ -v openstreetmap-data:/var/lib/postgresql/12/main \ --shm-size="192m" \ - -d overv/openstreetmap-tile-server \ + -d osm \ run ``` For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself. @@ -233,6 +155,7 @@ You may be running into problems with memory usage during the import. Have a loo ``` Copyright 2019 Alexander Overvoorde +Copyright 2021 Vitaliy Filippov Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/external-data.yml.diff b/external-data.yml.diff new file mode 100644 index 0000000..b1f39a0 --- /dev/null +++ b/external-data.yml.diff @@ -0,0 +1,13 @@ +diff --git a/external-data.yml b/external-data.yml +index c93a865..f78a003 100644 +--- a/external-data.yml ++++ b/external-data.yml +@@ -62,7 +62,7 @@ sources: + + ne_110m_admin_0_boundary_lines_land: + type: shp +- url: https://naciscdn.org/naturalearth/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip ++ url: http://wiki.4intra.net/public/ne_110m_admin_0_boundary_lines_land.zip + file: ne_110m_admin_0_boundary_lines_land.shp + ogropts: &ne_opts + - "--config" diff --git a/leaflet-demo.html b/leaflet-demo.html index 82d09a8..c34842a 100644 --- a/leaflet-demo.html +++ b/leaflet-demo.html @@ -25,7 +25,7 @@