vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #76 from omniscale/system-test 

Rewrite system test from Python to Go
master
Oliver Tonnhofer 2015-12-21 13:25:28 +01:00
parent af4aff7d6b 36bf5f14b3
commit c6445e3fc7
12 changed files with 1435 additions and 1069 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -34,11 +34,13 @@ clean:
rm -f imposm3
(cd test && make clean)
test: test-unit test-system
test: imposm3
$(GO) test ./... -i
$(GO) test ./...
test-unit: imposm3
$(GO) test ./... -i
$(GO) test ./...
$(GO) test `$(GO) list ./... | grep -v 'imposm3/test'`
test-system: imposm3
(cd test && make test)

26
README.md
View File

@ -250,29 +250,23 @@ The GEOS package is released as LGPL3 and is linked dynamically. See LICENSE.bin
#### Unit tests ####
go test imposm3/...
To run all unit tests:
make test-unit
Or:
godep go test ./...
#### System tests ####
There are system test that import and update OSM data and verify the database content.
##### Dependencies #####
These tests are written in Python and requires `nose`, `shapely` and `psycopg2`.
On a recent Ubuntu can install the following packages for that: `python-nose python-shapely python-psycopg2`
Or you can [install a Python virtualenv](https://virtualenv.pypa.io/en/latest/installation.html):
virtualenv imposm3test
source imposm3test/bin/activate
pip install nose shapely psycopg2
You also need `osmosis` to create test PBF files.
There is a Makefile that (re)builds `imposm3` and creates all test files if necessary and then runs the test itself.
You need `osmosis` to create the test PBF files.
There is a Makefile that creates all test files if necessary and then runs the test itself.
make test
Call `make test-system` to skip the unit tests.
WARNING: It uses your local PostgeSQL database (`import` schema). Change the database with the standard `PGDATABASE`, `PGHOST`, etc. environment variables.
WARNING: It uses your local PostgeSQL database (`imposm3testimport`, `imposm3testproduction` and `imposm3testbackup` schema). Change the database with the standard `PGDATABASE`, `PGHOST`, etc. environment variables.

47
cmd/main.go
View File

@ -6,12 +6,9 @@ import (
"os"
"runtime"
"github.com/omniscale/imposm3/cache"
"github.com/omniscale/imposm3/cache/query"
"github.com/omniscale/imposm3/config"
"github.com/omniscale/imposm3/diff"
"github.com/omniscale/imposm3/geom/limit"
"github.com/omniscale/imposm3/import_"
"github.com/omniscale/imposm3/logging"
"github.com/omniscale/imposm3/stats"
@ -53,49 +50,7 @@ func Main(usage func()) {
if config.BaseOptions.Httpprofile != "" {
stats.StartHttpPProf(config.BaseOptions.Httpprofile)
}
if config.BaseOptions.Quiet {
logging.SetQuiet(true)
}
var geometryLimiter *limit.Limiter
if config.BaseOptions.LimitTo != "" {
var err error
step := log.StartStep("Reading limitto geometries")
geometryLimiter, err = limit.NewFromGeoJSON(
config.BaseOptions.LimitTo,
config.BaseOptions.LimitToCacheBuffer,
config.BaseOptions.Srid,
)
if err != nil {
log.Fatal(err)
}
log.StopStep(step)
}
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
err := osmCache.Open()
if err != nil {
log.Fatal("osm cache: ", err)
}
defer osmCache.Close()
diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
err = diffCache.Open()
if err != nil {
log.Fatal("diff cache: ", err)
}
for _, oscFile := range config.DiffFlags.Args() {
err := diff.Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false)
if err != nil {
osmCache.Close()
diffCache.Close()
log.Fatalf("unable to process %s: %v", oscFile, err)
}
}
// explicitly Close since os.Exit prevents defers
osmCache.Close()
diffCache.Close()
diff.Diff()
case "query-cache":
query.Query(os.Args[2:])

45
diff/process.go
View File

@ -24,6 +24,51 @@ import (
var log = logging.NewLogger("diff")
func Diff() {
if config.BaseOptions.Quiet {
logging.SetQuiet(true)
}
var geometryLimiter *limit.Limiter
if config.BaseOptions.LimitTo != "" {
var err error
step := log.StartStep("Reading limitto geometries")
geometryLimiter, err = limit.NewFromGeoJSON(
config.BaseOptions.LimitTo,
config.BaseOptions.LimitToCacheBuffer,
config.BaseOptions.Srid,
)
if err != nil {
log.Fatal(err)
}
log.StopStep(step)
}
osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir)
err := osmCache.Open()
if err != nil {
log.Fatal("osm cache: ", err)
}
defer osmCache.Close()
diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir)
err = diffCache.Open()
if err != nil {
log.Fatal("diff cache: ", err)
}
for _, oscFile := range config.DiffFlags.Args() {
err := Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false)
if err != nil {
osmCache.Close()
diffCache.Close()
log.Fatalf("unable to process %s: %v", oscFile, err)
}
}
// explicitly Close since os.Exit prevents defers
osmCache.Close()
diffCache.Close()
}
func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expireor, osmCache *cache.OSMCache, diffCache *cache.DiffCache, force bool) error {
state, err := diffstate.ParseFromOsc(oscFile)
if err != nil {

7
geom/geos/geos.go
View File

@ -246,6 +246,13 @@ func (this *Geos) IsValid(geom *Geom) bool {
return false
}
func (this *Geos) IsSimple(geom *Geom) bool {
if C.GEOSisSimple_r(this.v, geom.v) == 1 {
return true
}
return false
}
func (this *Geos) IsEmpty(geom *Geom) bool {
if C.GEOSisEmpty_r(this.v, geom.v) == 1 {
return true

28
test/Makefile
View File

@ -1,23 +1,19 @@
.PHONY: build all test clean
IMPOSM_BIN=../imposm3
.PHONY: all test clean
ifdef VERBOSE
NOSEOPTS = -vs
TESTOPTS = -v
else
NOSEOPTS = -v
TESTOPTS =
endif
all: build test
build:
cd ..; make build
$(IMPOSM_BIN): build
all: test
clean:
rm -rf build
PBF_FILES=$(addprefix build/,$(patsubst %.osm,%.pbf,$(wildcard *.osm)))
OSCGZ_FILES=$(addprefix build/,$(patsubst %.osc,%.osc.gz,$(wildcard *.osc)))
build/%.pbf: %.osm
@mkdir -p build
osmosis --read-xml $< --sort TypeThenId --write-pbf $@ omitmetadata=true
@ -26,12 +22,6 @@ build/%.osc.gz: %.osc
@mkdir -p build
gzip --stdout $< > $@
test: .lasttestrun_complete_db .lasttestrun_single_table
test: $(PBF_FILES) $(OSCGZ_FILES)
(cd .. && godep go test ./test $(TESTOPTS))
.lasttestrun_complete_db: $(IMPOSM_BIN) complete_db_test.py build/complete_db.osc.gz build/complete_db.pbf
nosetests complete_db_test.py $(NOSEOPTS)
@touch .lasttestrun_complete_db
.lasttestrun_single_table: $(IMPOSM_BIN) single_table_test.py build/single_table.osc.gz build/single_table.pbf
nosetests single_table_test.py $(NOSEOPTS)
@touch .lasttestrun_single_table

537
test/complete_db_test.py
View File

@ -1,537 +0,0 @@
import unittest
import helper as t
mapping_file = 'complete_db_mapping.json'
def setup():
t.setup()
def teardown():
t.teardown()
#######################################################################
def test_import():
"""Import succeeds"""
t.drop_schemas()
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
t.imposm3_import(t.db_conf, './build/complete_db.pbf', mapping_file)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
def test_deploy():
"""Deploy succeeds"""
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
t.imposm3_deploy(t.db_conf, mapping_file)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
#######################################################################
def test_imported_landusage():
"""Multipolygon relation is inserted"""
t.assert_cached_node(1001, (13, 47.5))
landusage_1001 = t.query_row(t.db_conf, 'osm_landusages', -1001)
# point in polygon
assert landusage_1001['geometry'].intersects(t.merc_point(13.4, 47.5))
# hole in multipolygon relation
assert not landusage_1001['geometry'].intersects(t.merc_point(14.75, 47.75))
def test_missing_nodes():
"""Cache does not contain nodes from previous imports"""
t.assert_missing_node(10001)
t.assert_missing_node(10002)
place_10000 = t.query_row(t.db_conf, 'osm_places', 10000)
assert place_10000['name'] == 'Foo', place_10000
def test_name_tags():
"""Road contains multiple names"""
road = t.query_row(t.db_conf, 'osm_roads', 1101)
assert road['name'] == 'name', road
assert road['name:de'] == 'name:de', road
assert road['name_en'] == 'name:en', road
def test_landusage_to_waterarea_1():
"""Parks inserted into landusages"""
t.assert_cached_way(11001)
t.assert_cached_way(12001)
t.assert_cached_way(13001)
assert not t.query_row(t.db_conf, 'osm_waterareas', 11001)
assert not t.query_row(t.db_conf, 'osm_waterareas', -12001)
assert not t.query_row(t.db_conf, 'osm_waterareas', -13001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', 11001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', -12001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', -13001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', 11001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', -12001)
assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', -13001)
assert t.query_row(t.db_conf, 'osm_landusages', 11001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages', -12001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages', -13001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen0', 11001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen0', -12001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen0', -13001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen1', 11001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen1', -12001)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages_gen1', -13001)['type'] == 'park'
def test_changed_hole_tags_1():
"""Multipolygon relation with untagged hole"""
t.assert_cached_way(14001)
t.assert_cached_way(14011)
assert not t.query_row(t.db_conf, 'osm_waterareas', 14011)
assert not t.query_row(t.db_conf, 'osm_waterareas', -14011)
assert t.query_row(t.db_conf, 'osm_landusages', -14001)['type'] == 'park'
def test_split_outer_multipolygon_way_1():
"""Single outer way of multipolygon was inserted."""
park_15001 = t.query_row(t.db_conf, 'osm_landusages', -15001)
assert park_15001['type'] == 'park'
t.assert_almost_equal(park_15001['geometry'].area, 9816216452, -1)
assert t.query_row(t.db_conf, 'osm_roads', 15002) == None
def test_merge_outer_multipolygon_way_1():
"""Splitted outer way of multipolygon was inserted."""
park_16001 = t.query_row(t.db_conf, 'osm_landusages', -16001)
assert park_16001['type'] == 'park'
t.assert_almost_equal(park_16001['geometry'].area, 12779350582, -1)
assert t.query_row(t.db_conf, 'osm_roads', 16002)['type'] == 'residential'
def test_broken_multipolygon_ways():
"""MultiPolygons with broken outer ways are handled."""
# outer way does not merge (17002 has one node)
assert t.query_row(t.db_conf, 'osm_landusages', -17001) == None
assert t.query_row(t.db_conf, 'osm_roads', 17001)['type'] == 'residential'
assert t.query_row(t.db_conf, 'osm_roads', 17002) == None
# outer way does not merge (17102 has no nodes)
assert t.query_row(t.db_conf, 'osm_landusages', -17101) == None
assert t.query_row(t.db_conf, 'osm_roads', 17101)['type'] == 'residential'
assert t.query_row(t.db_conf, 'osm_roads', 17102) == None
def test_node_way_inserted_twice():
"""Way with multiple mappings is inserted twice in same table"""
rows = t.query_row(t.db_conf, 'osm_roads', 18001)
rows.sort(key=lambda x: x['type'])
assert rows[0]['type'] == 'residential'
assert rows[1]['type'] == 'tram'
def test_outer_way_not_inserted():
"""Outer way with different tag is not inserted twice into same table"""
farm = t.query_row(t.db_conf, 'osm_landusages', -19001)
assert farm['type'] == 'farmland'
assert not t.query_row(t.db_conf, 'osm_landusages', 19001)
farmyard = t.query_row(t.db_conf, 'osm_landusages', 19002)
assert farmyard['type'] == 'farmyard'
def test_outer_way_inserted():
"""Outer way with different tag is inserted twice into different table"""
farm = t.query_row(t.db_conf, 'osm_landusages', 19101)
assert farm['type'] == 'farm'
assert not t.query_row(t.db_conf, 'osm_landusages', -19101)
farmyard = t.query_row(t.db_conf, 'osm_landusages', 19102)
assert farmyard['type'] == 'farmyard'
admin = t.query_row(t.db_conf, 'osm_admin', -19101)
assert admin['type'] == 'administrative'
def test_node_way_ref_after_delete_1():
"""Nodes refereces way"""
data = t.cache_query(nodes=[20001, 20002], deps=True)
assert '20001' in data['nodes']['20001']['ways']
assert '20001' in data['nodes']['20002']['ways']
assert t.query_row(t.db_conf, 'osm_roads', 20001)['type'] == 'residential'
assert t.query_row(t.db_conf, 'osm_barrierpoints', 20001)['type'] == 'block'
def test_way_rel_ref_after_delete_1():
"""Ways references relation"""
data = t.cache_query(ways=[21001], deps=True)
assert data['ways']['21001']['relations'].keys() == ['21001']
assert t.query_row(t.db_conf, 'osm_roads', 21001)['type'] == 'residential'
assert t.query_row(t.db_conf, 'osm_landusages', -21001)['type'] == 'park'
def test_relation_way_not_inserted():
"""Part of relation was inserted only once."""
park = t.query_row(t.db_conf, 'osm_landusages', -9001)
assert park['type'] == 'park'
assert park['name'] == 'rel 9001'
assert t.query_row(t.db_conf, 'osm_landusages', 9009) == None
park = t.query_row(t.db_conf, 'osm_landusages', -9101)
assert park['type'] == 'park'
assert park['name'] == 'rel 9101'
assert t.query_row(t.db_conf, 'osm_landusages', 9109) == None
scrub = t.query_row(t.db_conf, 'osm_landusages', 9110)
assert scrub['type'] == 'scrub'
def test_relation_ways_inserted():
"""Outer ways of multipolygon are inserted. """
park = t.query_row(t.db_conf, 'osm_landusages', -9201)
assert park['type'] == 'park'
assert park['name'] == '9209'
assert not t.query_row(t.db_conf, 'osm_landusages', 9201)
# outer ways of multipolygon stand for their own
road = t.query_row(t.db_conf, 'osm_roads', 9209)
assert road['type'] == 'secondary'
assert road['name'] == '9209'
road = t.query_row(t.db_conf, 'osm_roads', 9210)
assert road['type'] == 'residential'
assert road['name'] == '9210'
park = t.query_row(t.db_conf, 'osm_landusages', -9301)
assert park['type'] == 'park'
assert park['name'] == '' # no name on relation
# outer ways of multipolygon stand for their own
road = t.query_row(t.db_conf, 'osm_roads', 9309)
assert road['type'] == 'secondary'
assert road['name'] == '9309'
road = t.query_row(t.db_conf, 'osm_roads', 9310)
assert road['type'] == 'residential'
assert road['name'] == '9310'
def test_relation_way_inserted():
"""Part of relation was inserted twice."""
park = t.query_row(t.db_conf, 'osm_landusages', -8001)
assert park['type'] == 'park'
assert park['name'] == 'rel 8001'
assert t.query_row(t.db_conf, 'osm_roads', 8009)["type"] == 'residential'
def test_single_node_ways_not_inserted():
"""Ways with single/duplicate nodes are not inserted."""
assert not t.query_row(t.db_conf, 'osm_roads', 30001)
assert not t.query_row(t.db_conf, 'osm_roads', 30002)
assert not t.query_row(t.db_conf, 'osm_roads', 30003)
def test_polygon_with_duplicate_nodes_is_valid():
"""Polygon with duplicate nodes is valid."""
geom = t.query_row(t.db_conf, 'osm_landusages', 30005)['geometry']
assert geom.is_valid
assert len(geom.exterior.coords) == 4
def test_incomplete_polygons():
"""Non-closed/incomplete polygons are not inserted."""
assert not t.query_row(t.db_conf, 'osm_landusages', 30004)
assert not t.query_row(t.db_conf, 'osm_landusages', 30006)
def test_residential_to_secondary():
"""Residential road is not in roads_gen0/1."""
assert t.query_row(t.db_conf, 'osm_roads', 40001)['type'] == 'residential'
assert not t.query_row(t.db_conf, 'osm_roads_gen0', 40001)
assert not t.query_row(t.db_conf, 'osm_roads_gen1', 40001)
def test_relation_before_remove():
"""Relation and way is inserted."""
assert t.query_row(t.db_conf, 'osm_buildings', 50011)['type'] == 'yes'
assert t.query_row(t.db_conf, 'osm_landusages', -50021)['type'] == 'park'
def test_relation_without_tags():
"""Relation without tags is inserted."""
assert t.query_row(t.db_conf, 'osm_buildings', 50111) == None
assert t.query_row(t.db_conf, 'osm_buildings', -50121)['type'] == 'yes'
def test_duplicate_ids():
"""Relation/way with same ID is inserted."""
assert t.query_row(t.db_conf, 'osm_buildings', 51001)['type'] == 'way'
assert t.query_row(t.db_conf, 'osm_buildings', -51001)['type'] == 'mp'
assert t.query_row(t.db_conf, 'osm_buildings', 51011)['type'] == 'way'
assert t.query_row(t.db_conf, 'osm_buildings', -51011)['type'] == 'mp'
def test_generalized_banana_polygon_is_valid():
"""Generalized polygons are valid."""
park = t.query_row(t.db_conf, 'osm_landusages', 7101)
# geometry is valid
assert park['geometry'].is_valid, park
park = t.query_row(t.db_conf, 'osm_landusages_gen0', 7101)
# simplified geometies are valid too
assert park['geometry'].is_valid, park
park = t.query_row(t.db_conf, 'osm_landusages_gen1', 7101)
assert park['geometry'].is_valid, park
def test_generalized_linestring_is_valid():
"""Generalized linestring is valid."""
road = t.query_row(t.db_conf, 'osm_roads', 7201)
# geometry is not simple, but valid
# check that geometry 'survives' simplification
assert not road['geometry'].is_simple, road['geometry'].wkt
assert road['geometry'].is_valid, road['geometry'].wkt
assert road['geometry'].length > 1000000
road = t.query_row(t.db_conf, 'osm_roads_gen0', 7201)
# but simplified geometies are simple
assert road['geometry'].is_valid, road['geometry'].wkt
assert road['geometry'].length > 1000000
road = t.query_row(t.db_conf, 'osm_roads_gen1', 7201)
assert road['geometry'].is_valid, road['geometry'].wkt
assert road['geometry'].length > 1000000
def test_ring_with_gap():
"""Multipolygon and way with gap (overlapping but different endpoints) gets closed"""
park = t.query_row(t.db_conf, 'osm_landusages', -7301)
assert park['geometry'].is_valid, park
park = t.query_row(t.db_conf, 'osm_landusages', 7311)
assert park['geometry'].is_valid, park
def test_multipolygon_with_open_ring():
"""Multipolygon is inserted even if there is an open ring/member"""
park = t.query_row(t.db_conf, 'osm_landusages', -7401)
assert park['geometry'].is_valid, park
def test_updated_nodes1():
"""Zig-Zag line is inserted."""
road = t.query_row(t.db_conf, 'osm_roads', 60000)
t.assert_almost_equal(road['geometry'].length, 14035.61150207768)
def test_update_node_to_coord_1():
"""Node is inserted with tag."""
coords = t.cache_query(nodes=(70001, 70002))
assert coords['nodes']["70001"]["tags"] == {"amenity": "police"}
assert "tags" not in coords['nodes']["70002"]
assert t.query_row(t.db_conf, 'osm_amenities', 70001)
assert not t.query_row(t.db_conf, 'osm_amenities', 70002)
def test_enumerate_key():
"""Enumerate from key."""
assert t.query_row(t.db_conf, 'osm_landusages', 100001)['enum'] == 1
assert t.query_row(t.db_conf, 'osm_landusages', 100002)['enum'] == 0
assert t.query_row(t.db_conf, 'osm_landusages', 100003)['enum'] == 15
#######################################################################
def test_update():
"""Diff import applies"""
t.imposm3_update(t.db_conf, './build/complete_db.osc.gz', mapping_file)
#######################################################################
def test_no_duplicates():
"""
Relations/ways are only inserted once
Checks #66
"""
highways = t.query_duplicates(t.db_conf, 'osm_roads')
# one duplicate for test_node_way_inserted_twice is expected
assert highways == [[18001, 2]], highways
landusages = t.query_duplicates(t.db_conf, 'osm_landusages')
assert not landusages, landusages
def test_updated_landusage():
"""Multipolygon relation was modified"""
t.assert_cached_node(1001, (13.5, 47.5))
landusage_1001 = t.query_row(t.db_conf, 'osm_landusages', -1001)
# point not in polygon after update
assert not landusage_1001['geometry'].intersects(t.merc_point(13.4, 47.5))
def test_partial_delete():
"""Deleted relation but nodes are still cached"""
t.assert_cached_node(2001)
t.assert_cached_way(2001)
t.assert_cached_way(2002)
assert not t.query_row(t.db_conf, 'osm_landusages', -2001)
assert not t.query_row(t.db_conf, 'osm_landusages', 2001)
def test_updated_nodes():
"""Nodes were added, modified or deleted"""
t.assert_missing_node(10000)
t.assert_cached_node(10001, (10.0, 40.0))
t.assert_cached_node(10002, (10.1, 40.0))
place_10001 = t.query_row(t.db_conf, 'osm_places', 10001)
assert place_10001['name'] == 'Bar', place_10001
place_10002 = t.query_row(t.db_conf, 'osm_places', 10002)
assert place_10002['name'] == 'Baz', place_10002
def test_landusage_to_waterarea_2():
"""Parks converted to water moved from landusages to waterareas"""
t.assert_cached_way(11001)
t.assert_cached_way(12001)
t.assert_cached_way(13001)
assert not t.query_row(t.db_conf, 'osm_landusages', 11001)
assert not t.query_row(t.db_conf, 'osm_landusages', -12001)
assert not t.query_row(t.db_conf, 'osm_landusages', -13001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen0', 11001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen0', -12001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen0', -13001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen1', 11001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen1', -12001)
assert not t.query_row(t.db_conf, 'osm_landusages_gen1', -13001)
assert t.query_row(t.db_conf, 'osm_waterareas', 11001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas', -12001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas', -13001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen0', 11001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen0', -12001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen0', -13001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen1', 11001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen1', -12001)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_waterareas_gen1', -13001)['type'] == 'water'
def test_changed_hole_tags_2():
"""Newly tagged hole is inserted"""
t.assert_cached_way(14001)
t.assert_cached_way(14011)
assert t.query_row(t.db_conf, 'osm_waterareas', 14011)['type'] == 'water'
assert t.query_row(t.db_conf, 'osm_landusages', -14001)['type'] == 'park'
t.assert_almost_equal(t.query_row(t.db_conf, 'osm_waterareas', 14011)['geometry'].area, 26672000000, -6)
t.assert_almost_equal(t.query_row(t.db_conf, 'osm_landusages', -14001)['geometry'].area, 10373600000, -6)
def test_split_outer_multipolygon_way_2():
"""Splitted outer way of multipolygon was inserted"""
data = t.cache_query(ways=[15001, 15002], deps=True)
assert data['ways']['15001']['relations'].keys() == ['15001']
assert data['ways']['15002']['relations'].keys() == ['15001']
assert t.query_row(t.db_conf, 'osm_landusages', 15001) == None
park_15001 = t.query_row(t.db_conf, 'osm_landusages', -15001)
assert park_15001['type'] == 'park'
t.assert_almost_equal(park_15001['geometry'].area, 9816216452, -1)
assert t.query_row(t.db_conf, 'osm_roads', 15002)['type'] == 'residential'
def test_merge_outer_multipolygon_way_2():
"""Merged outer way of multipolygon was inserted"""
data = t.cache_query(ways=[16001, 16002], deps=True)
assert data['ways']['16001']['relations'].keys() == ['16001']
assert data['ways']['16002'] == None
data = t.cache_query(relations=[16001], full=True)
assert sorted(data['relations']['16001']['ways'].keys()) == ['16001', '16011']
assert t.query_row(t.db_conf, 'osm_landusages', 16001) == None
park_16001 = t.query_row(t.db_conf, 'osm_landusages', -16001)
assert park_16001['type'] == 'park'
t.assert_almost_equal(park_16001['geometry'].area, 12779350582, -1)
assert t.query_row(t.db_conf, 'osm_roads', 16002) == None
def test_node_way_ref_after_delete_2():
"""Node does not referece deleted way"""
data = t.cache_query(nodes=[20001, 20002], deps=True)
assert 'ways' not in data['nodes']['20001']
assert data['nodes']['20002'] == None
assert t.query_row(t.db_conf, 'osm_roads', 20001) == None
assert t.query_row(t.db_conf, 'osm_barrierpoints', 20001)['type'] == 'block'
def test_way_rel_ref_after_delete_2():
"""Way does not referece deleted relation"""
data = t.cache_query(ways=[21001], deps=True)
assert 'relations' not in data['ways']['21001']
assert t.query_row(t.db_conf, 'osm_roads', 21001)['type'] == 'residential'
assert t.query_row(t.db_conf, 'osm_landusages', 21001) == None
assert t.query_row(t.db_conf, 'osm_landusages', -21001) == None
def test_residential_to_secondary2():
"""New secondary (from residential) is now in roads_gen0/1."""
assert t.query_row(t.db_conf, 'osm_roads', 40001)['type'] == 'secondary'
assert t.query_row(t.db_conf, 'osm_roads_gen0', 40001)['type'] == 'secondary'
assert t.query_row(t.db_conf, 'osm_roads_gen1', 40001)['type'] == 'secondary'
def test_relation_after_remove():
"""Relation is deleted and way is still present."""
assert t.query_row(t.db_conf, 'osm_buildings', 50011)['type'] == 'yes'
assert t.query_row(t.db_conf, 'osm_landusages', 50021) == None
assert t.query_row(t.db_conf, 'osm_landusages', -50021) == None
def test_relation_without_tags2():
"""Relation without tags is removed."""
t.cache_query(ways=[50111], deps=True)
assert t.cache_query(relations=[50121], deps=True)['relations']["50121"] == None
assert t.query_row(t.db_conf, 'osm_buildings', 50111)['type'] == 'yes'
assert t.query_row(t.db_conf, 'osm_buildings', 50121) == None
assert t.query_row(t.db_conf, 'osm_buildings', -50121) == None
def test_duplicate_ids2():
"""Only relation/way with same ID was deleted."""
assert t.query_row(t.db_conf, 'osm_buildings', 51001)['type'] == 'way'
assert t.query_row(t.db_conf, 'osm_buildings', -51001) == None
assert t.query_row(t.db_conf, 'osm_buildings', -51011)['type'] == 'mp'
assert t.query_row(t.db_conf, 'osm_buildings', 51011) == None
def test_updated_way2():
"""All nodes of straightened way are updated."""
road = t.query_row(t.db_conf, 'osm_roads', 60000)
# new length 0.1 degree
t.assert_almost_equal(road['geometry'].length, 20037508.342789244/180.0/10.0)
def test_update_node_to_coord_2():
"""Node is becomes coord after tags are removed."""
coords = t.cache_query(nodes=(70001, 70002))
assert "tags" not in coords['nodes']["70001"]
assert coords['nodes']["70002"]["tags"] == {"amenity": "police"}
assert not t.query_row(t.db_conf, 'osm_amenities', 70001)
assert t.query_row(t.db_conf, 'osm_amenities', 70002)
def test_no_duplicate_insert():
"""
Relation is not inserted again if a nother relation with the same way was modified
Checks #65
"""
assert t.query_row(t.db_conf, 'osm_landusages', -201191)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages', -201192)['type'] == 'forest'
assert t.query_row(t.db_conf, 'osm_roads', 201151)['type'] == 'residential'
def test_unsupported_relation():
"""
Unsupported relation type is not inserted with update
"""
assert not t.query_row(t.db_conf, 'osm_landusages', -201291)
assert t.query_row(t.db_conf, 'osm_landusages', 201251)['type'] == 'park'
#######################################################################
def test_deploy_and_revert_deploy():
"""Revert deploy succeeds"""
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)
# import again to have a new import schema
t.imposm3_import(t.db_conf, './build/complete_db.pbf', mapping_file)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
t.imposm3_deploy(t.db_conf, mapping_file)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_revert_deploy(t.db_conf, mapping_file)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)
def test_remove_backup():
"""Remove backup succeeds"""
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_deploy(t.db_conf, mapping_file)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_remove_backups(t.db_conf, mapping_file)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP)

728
test/completedb_test.go Normal file
View File

@ -0,0 +1,728 @@
package test
import (
"database/sql"
"fmt"
"github.com/omniscale/imposm3/cache"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/proj"
"testing"
"github.com/omniscale/imposm3/geom/geos"
)
var ts importTestSuite
func TestPrepare(t *testing.T) {
ts.dir = "/tmp/imposm3test"
ts.config = importConfig{
connection: "postgis://",
cacheDir: ts.dir,
osmFileName: "build/complete_db.pbf",
mappingFileName: "complete_db_mapping.json",
}
ts.g = geos.NewGeos()
var err error
ts.db, err = sql.Open("postgres", "sslmode=disable")
if err != nil {
t.Fatal(err)
}
ts.dropSchemas()
}
func TestImport(t *testing.T) {
if ts.tableExists(t, dbschemaImport, "osm_roads") != false {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
ts.importOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_roads") != true {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport)
}
}
func TestDeploy(t *testing.T) {
ts.deployOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_roads") != false {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
if ts.tableExists(t, dbschemaProduction, "osm_roads") != true {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
}
func TestLandusageToWaterarea1(t *testing.T) {
// Parks inserted into landusages
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 11001)
assertCachedWay(t, cache, 12001)
assertCachedWay(t, cache, 13001)
assertRecords(t, []checkElem{
{"osm_waterareas", 11001, Missing, nil},
{"osm_waterareas", -12001, Missing, nil},
{"osm_waterareas", -13001, Missing, nil},
{"osm_waterareas_gen0", 11001, Missing, nil},
{"osm_waterareas_gen0", -12001, Missing, nil},
{"osm_waterareas_gen0", -13001, Missing, nil},
{"osm_waterareas_gen1", 11001, Missing, nil},
{"osm_waterareas_gen1", -12001, Missing, nil},
{"osm_waterareas_gen1", -13001, Missing, nil},
{"osm_landusages", 11001, "park", nil},
{"osm_landusages", -12001, "park", nil},
{"osm_landusages", -13001, "park", nil},
{"osm_landusages_gen0", 11001, "park", nil},
{"osm_landusages_gen0", -12001, "park", nil},
{"osm_landusages_gen0", -13001, "park", nil},
{"osm_landusages_gen1", 11001, "park", nil},
{"osm_landusages_gen1", -12001, "park", nil},
{"osm_landusages_gen1", -13001, "park", nil},
})
}
func TestChangedHoleTags1(t *testing.T) {
// Multipolygon relation with untagged hole
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 14001)
assertCachedWay(t, cache, 14011)
assertRecords(t, []checkElem{
{"osm_waterareas", 14011, Missing, nil},
{"osm_waterareas", -14011, Missing, nil},
{"osm_landusages", -14001, "park", nil},
})
}
func TestSplitOuterMultipolygonWay1(t *testing.T) {
// Single outer way of multipolygon was inserted.
assertRecords(t, []checkElem{
{"osm_roads", 15002, Missing, nil},
{"osm_landusages", -15001, "park", nil},
})
assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452)
}
func TestMergeOuterMultipolygonWay1(t *testing.T) {
// Splitted outer way of multipolygon was inserted.
assertRecords(t, []checkElem{
{"osm_landusages", -16001, "park", nil},
{"osm_roads", 16002, "residential", nil},
})
assertArea(t, checkElem{"osm_landusages", -16001, "park", nil}, 12779350582)
}
func TestBrokenMultipolygonWays(t *testing.T) {
// MultiPolygons with broken outer ways are handled.
// outer way does not merge (17002 has one node)
assertRecords(t, []checkElem{
{"osm_landusages", -17001, Missing, nil},
{"osm_roads", 17001, "residential", nil},
{"osm_roads", 17002, Missing, nil},
})
// outer way does not merge (17102 has no nodes)
assertRecords(t, []checkElem{
{"osm_landusages", -17101, Missing, nil},
{"osm_roads", 17101, "residential", nil},
{"osm_roads", 17102, Missing, nil},
})
}
func TestNodeWayInsertedTwice(t *testing.T) {
// Way with multiple mappings is inserted twice in same table
rows := ts.queryRows(t, "osm_roads", 18001)
if len(rows) != 2 || rows[0].osmType != "residential" || rows[1].osmType != "tram" {
t.Errorf("unexpected roads: %v", rows)
}
}
func TestOuterWayNotInserted(t *testing.T) {
// Outer way with different tag is not inserted twice into same table
assertRecords(t, []checkElem{
{"osm_landusages", -19001, "farmland", nil},
{"osm_landusages", 19002, "farmyard", nil},
{"osm_landusages", 19001, Missing, nil},
})
}
func TestOuterWayInserted(t *testing.T) {
// Outer way with different tag is inserted twice into different table
assertRecords(t, []checkElem{
{"osm_landusages", 19101, "farm", nil},
{"osm_landusages", 19102, "farmyard", nil},
{"osm_admin", -19101, "administrative", nil},
})
}
func TestNodeWayRefAfterDelete1(t *testing.T) {
// Nodes references way
cache := ts.diffCache(t)
defer cache.Close()
if ids := cache.Coords.Get(20001); len(ids) != 1 || ids[0] != 20001 {
t.Error("node does not references way")
}
if ids := cache.Coords.Get(20002); len(ids) != 1 || ids[0] != 20001 {
t.Error("node does not references way")
}
assertRecords(t, []checkElem{
{"osm_roads", 20001, "residential", nil},
{"osm_barrierpoints", 20001, "block", nil},
})
}
func TestWayRelRefAfterDelete1(t *testing.T) {
// Ways references relation
cache := ts.diffCache(t)
defer cache.Close()
if ids := cache.Ways.Get(21001); len(ids) != 1 || ids[0] != 21001 {
t.Error("way does not references relation")
}
assertRecords(t, []checkElem{
{"osm_roads", 21001, "residential", nil},
{"osm_landusages", -21001, "park", nil},
})
}
func TestRelationWayNotInserted(t *testing.T) {
// Part of relation was inserted only once.
assertRecords(t, []checkElem{
{"osm_landusages", -9001, "park", map[string]string{"name": "rel 9001"}},
{"osm_landusages", 9009, Missing, nil},
{"osm_landusages", -9101, "park", map[string]string{"name": "rel 9101"}},
{"osm_landusages", 9109, Missing, nil},
{"osm_landusages", 9110, "scrub", nil},
})
}
func TestRelationWaysInserted(t *testing.T) {
// Outer ways of multipolygon are inserted.
assertRecords(t, []checkElem{
{"osm_landusages", -9201, "park", map[string]string{"name": "9209"}},
{"osm_landusages", 9201, Missing, nil},
// outer ways of multipolygon stand for their own
{"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}},
{"osm_roads", 9210, "residential", map[string]string{"name": "9210"}},
// no name on relation
{"osm_landusages", -9301, "park", map[string]string{"name": ""}},
// outer ways of multipolygon stand for their own
{"osm_roads", 9309, "secondary", map[string]string{"name": "9309"}},
{"osm_roads", 9310, "residential", map[string]string{"name": "9310"}},
})
}
func TestRelationWayInserted(t *testing.T) {
// Part of relation was inserted twice.
assertRecords(t, []checkElem{
{"osm_landusages", -8001, "park", map[string]string{"name": "rel 8001"}},
{"osm_roads", 8009, "residential", nil},
})
}
func TestSingleNodeWaysNotInserted(t *testing.T) {
// Ways with single/duplicate nodes are not inserted.
assertRecords(t, []checkElem{
{"osm_landusages", 30001, Missing, nil},
{"osm_landusages", 30002, Missing, nil},
{"osm_landusages", 30003, Missing, nil},
})
}
func TestPolygonWithDuplicateNodesIsValid(t *testing.T) {
// Polygon with duplicate nodes is valid.
assertValid(t, checkElem{"osm_landusages", 30005, "park", nil})
}
func TestIncompletePolygons(t *testing.T) {
// Non-closed/incomplete polygons are not inserted.
assertRecords(t, []checkElem{
{"osm_landusages", 30004, Missing, nil},
{"osm_landusages", 30006, Missing, nil},
})
}
func TestResidentialToSecondary(t *testing.T) {
// Residential road is not in roads_gen0/1.
assertRecords(t, []checkElem{
{"osm_roads", 40001, "residential", nil},
{"osm_roads_gen0", 40001, Missing, nil},
{"osm_roads_gen1", 40002, Missing, nil},
})
}
func TestRelationBeforeRemove(t *testing.T) {
// Relation and way is inserted.
assertRecords(t, []checkElem{
{"osm_buildings", 50011, "yes", nil},
{"osm_landusages", -50021, "park", nil},
})
}
func TestRelationWithoutTags(t *testing.T) {
// Relation without tags is inserted.
assertRecords(t, []checkElem{
{"osm_buildings", 50111, Missing, nil},
{"osm_buildings", -50121, "yes", nil},
})
}
func TestDuplicateIds(t *testing.T) {
// Relation/way with same ID is inserted.
assertRecords(t, []checkElem{
{"osm_buildings", 51001, "way", nil},
{"osm_buildings", -51001, "mp", nil},
{"osm_buildings", 51011, "way", nil},
{"osm_buildings", -51011, "mp", nil},
})
}
func TestGeneralizedBananaPolygonIsValid(t *testing.T) {
// Generalized polygons are valid.
assertValid(t, checkElem{"osm_landusages", 7101, Missing, nil})
// simplified geometies are valid too
assertValid(t, checkElem{"osm_landusages_gen0", 7101, Missing, nil})
assertValid(t, checkElem{"osm_landusages_gen1", 7101, Missing, nil})
}
func TestGeneralizedLinestringIsValid(t *testing.T) {
// Generalized linestring is valid.
// geometry is not simple, but valid
assertLength(t, checkElem{"osm_roads", 7201, "primary", nil}, 1243660.044819)
if ts.g.IsSimple(ts.queryGeom(t, "osm_roads", 7201)) {
t.Errorf("expected non-simple geometry for 7201")
}
// check that geometry 'survives' simplification
assertLength(t, checkElem{"osm_roads_gen0", 7201, "primary", nil}, 1243660.044819)
assertLength(t, checkElem{"osm_roads_gen1", 7201, "primary", nil}, 1243660.044819)
}
func TestRingWithGap(t *testing.T) {
// Multipolygon and way with gap (overlapping but different endpoints) gets closed
assertValid(t, checkElem{"osm_landusages", -7301, Missing, nil})
assertValid(t, checkElem{"osm_landusages", 7311, Missing, nil})
}
func TestMultipolygonWithOpenRing(t *testing.T) {
// Multipolygon is inserted even if there is an open ring/member
assertValid(t, checkElem{"osm_landusages", -7401, Missing, nil})
}
func TestUpdatedNodes1(t *testing.T) {
// Zig-Zag line is inserted.
assertLength(t, checkElem{"osm_roads", 60000, Missing, nil}, 14035.61150207768)
}
func TestUpdateNodeToCoord1(t *testing.T) {
// Node is inserted with tag.
assertRecords(t, []checkElem{
{"osm_amenities", 70001, "police", nil},
{"osm_amenities", 70002, Missing, nil},
})
}
func TestEnumerateKey(t *testing.T) {
// Enumerate from key.
assertRecords(t, []checkElem{
{"osm_landusages", 100001, "park", map[string]string{"enum": "1"}},
{"osm_landusages", 100002, "park", map[string]string{"enum": "0"}},
{"osm_landusages", 100003, "wood", map[string]string{"enum": "15"}},
})
}
func TestUpdate(t *testing.T) {
ts.updateOsm(t, "./build/complete_db.osc.gz")
}
func TestNoDuplicates(t *testing.T) {
// Relations/ways are only inserted once Checks #66
for _, table := range []string{"osm_roads", "osm_landusages"} {
rows, err := ts.db.Query(
fmt.Sprintf(`SELECT osm_id, count(osm_id) FROM "%s"."%s" GROUP BY osm_id HAVING count(osm_id) > 1`,
dbschemaProduction, table))
if err != nil {
t.Fatal(err)
}
var osmId, count int64
for rows.Next() {
if err := rows.Scan(&osmId, &count); err != nil {
t.Fatal(err)
}
if table == "osm_roads" && osmId == 18001 {
// # duplicate for TestNodeWayInsertedTwice is expected
if count != 2 {
t.Error("highway not inserted twice", osmId, count)
}
} else {
t.Error("found duplicate way in osm_roads", osmId, count)
}
}
}
}
func TestUpdatedLandusage(t *testing.T) {
// Multipolygon relation was modified
nd := element.Node{Long: 13.4, Lat: 47.5}
proj.NodeToMerc(&nd)
point, err := geom.Point(ts.g, nd)
if err != nil {
t.Fatal(err)
}
poly := ts.queryGeom(t, "osm_landusages", -1001)
// point not in polygon after update
if ts.g.Intersects(point, poly) {
t.Error("point intersects polygon")
}
}
func TestPartialDelete(t *testing.T) {
// Deleted relation but nodes are still cached
cache := ts.cache(t)
defer cache.Close()
assertCachedNode(t, cache, 2001)
assertCachedWay(t, cache, 2001)
assertCachedWay(t, cache, 2002)
assertRecords(t, []checkElem{
{"osm_landusages", -2001, Missing, nil},
{"osm_landusages", 2001, Missing, nil},
})
}
func TestUpdatedNodes(t *testing.T) {
// Nodes were added, modified or deleted
c := ts.cache(t)
defer c.Close()
if _, err := c.Coords.GetCoord(10000); err != cache.NotFound {
t.Fatal("coord not missing")
}
assertRecords(t, []checkElem{
{"osm_places", 10001, "village", map[string]string{"name": "Bar"}},
{"osm_places", 10002, "city", map[string]string{"name": "Baz"}},
})
}
func TestLandusageToWaterarea2(t *testing.T) {
// Parks converted to water moved from landusages to waterareas
assertRecords(t, []checkElem{
{"osm_waterareas", 11001, "water", nil},
{"osm_waterareas", -12001, "water", nil},
{"osm_waterareas", -13001, "water", nil},
{"osm_waterareas_gen0", 11001, "water", nil},
{"osm_waterareas_gen0", -12001, "water", nil},
{"osm_waterareas_gen0", -13001, "water", nil},
{"osm_waterareas_gen1", 11001, "water", nil},
{"osm_waterareas_gen1", -12001, "water", nil},
{"osm_waterareas_gen1", -13001, "water", nil},
{"osm_landusages", 11001, Missing, nil},
{"osm_landusages", -12001, Missing, nil},
{"osm_landusages", -13001, Missing, nil},
{"osm_landusages_gen0", 11001, Missing, nil},
{"osm_landusages_gen0", -12001, Missing, nil},
{"osm_landusages_gen0", -13001, Missing, nil},
{"osm_landusages_gen1", 11001, Missing, nil},
{"osm_landusages_gen1", -12001, Missing, nil},
{"osm_landusages_gen1", -13001, Missing, nil},
})
}
func TestChangedHoleTags2(t *testing.T) {
// Newly tagged hole is inserted
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 14001)
assertCachedWay(t, cache, 14011)
assertArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779)
assertArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182)
}
func TestSplitOuterMultipolygonWay2(t *testing.T) {
// Splitted outer way of multipolygon was inserted
diffCache := ts.diffCache(t)
defer diffCache.Close()
if ids := diffCache.Ways.Get(15001); len(ids) != 1 || ids[0] != 15001 {
t.Error("way does not references relation")
}
if ids := diffCache.Ways.Get(15002); len(ids) != 1 || ids[0] != 15001 {
t.Error("way does not references relation")
}
assertRecords(t, []checkElem{
{"osm_landusages", 15001, Missing, nil},
{"osm_roads", 15002, "residential", nil},
})
assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452)
}
func TestMergeOuterMultipolygonWay2(t *testing.T) {
// Merged outer way of multipolygon was inserted
diffCache := ts.diffCache(t)
defer diffCache.Close()
if ids := diffCache.Ways.Get(16001); len(ids) != 1 || ids[0] != 16001 {
t.Error("way does not references relation")
}
if ids := diffCache.Ways.Get(16002); len(ids) != 0 {
t.Error("way references relation")
}
cache := ts.cache(t)
defer cache.Close()
rel, err := cache.Relations.GetRelation(16001)
if err != nil {
t.Fatal(err)
}
if len(rel.Members) != 2 || rel.Members[0].Id != 16001 || rel.Members[1].Id != 16011 {
t.Error("unexpected relation members", rel)
}
assertRecords(t, []checkElem{
{"osm_landusages", 16001, Missing, nil},
{"osm_roads", 16002, Missing, nil},
})
assertArea(t, checkElem{"osm_landusages", -16001, "park", nil}, 12779350582)
}
func TestNodeWayRefAfterDelete2(t *testing.T) {
// Node does not referece deleted way
diffCache := ts.diffCache(t)
defer diffCache.Close()
if ids := diffCache.Coords.Get(20001); len(ids) != 0 {
t.Error("node references way")
}
c := ts.cache(t)
defer c.Close()
_, err := c.Coords.GetCoord(20002)
if err != cache.NotFound {
t.Error("found deleted node")
}
assertRecords(t, []checkElem{
{"osm_roads", 20001, Missing, nil},
{"osm_barrierpoints", 20001, "block", nil},
})
}
func TestWayRelRefAfterDelete2(t *testing.T) {
// Way does not referece deleted relation
diffCache := ts.diffCache(t)
defer diffCache.Close()
if ids := diffCache.Ways.Get(21001); len(ids) != 0 {
t.Error("way references relation")
}
assertRecords(t, []checkElem{
{"osm_roads", 21001, "residential", nil},
{"osm_landusages", 21001, Missing, nil},
{"osm_landusages", -21001, Missing, nil},
})
}
func TestResidentialToSecondary2(t *testing.T) {
// New secondary (from residential) is now in roads_gen0/1.
assertRecords(t, []checkElem{
{"osm_roads", 40001, "secondary", nil},
{"osm_roads_gen0", 40001, "secondary", nil},
{"osm_roads_gen1", 40001, "secondary", nil},
})
}
func TestRelationAfterRemove(t *testing.T) {
// Relation is deleted and way is still present.
assertRecords(t, []checkElem{
{"osm_buildings", 50011, "yes", nil},
{"osm_landusages", 50021, Missing, nil},
{"osm_landusages", -50021, Missing, nil},
})
}
func TestRelationWithoutTags2(t *testing.T) {
// Relation without tags is removed.
c := ts.cache(t)
defer c.Close()
assertCachedWay(t, c, 50111)
_, err := c.Ways.GetWay(20002)
if err != cache.NotFound {
t.Error("found deleted node")
}
assertRecords(t, []checkElem{
{"osm_buildings", 50111, "yes", nil},
{"osm_buildings", 50121, Missing, nil},
{"osm_buildings", -50121, Missing, nil},
})
}
func TestDuplicateIds2(t *testing.T) {
// Only relation/way with same ID was deleted.
assertRecords(t, []checkElem{
{"osm_buildings", 51001, "way", nil},
{"osm_buildings", -51001, Missing, nil},
{"osm_buildings", 51011, Missing, nil},
{"osm_buildings", -51011, "mp", nil},
})
}
func TestUpdatedWay2(t *testing.T) {
// All nodes of straightened way are updated.
// new length 0.1 degree
assertLength(t, checkElem{"osm_roads", 60000, "park", nil}, 20037508.342789244/180.0/10.0)
}
func TestUpdateNodeToCoord2(t *testing.T) {
// Node is becomes coord after tags are removed.
assertRecords(t, []checkElem{
{"osm_amenities", 70001, Missing, nil},
{"osm_amenities", 70002, "police", nil},
})
}
func TestNoDuplicateInsert(t *testing.T) {
// Relation is not inserted again if a nother relation with the same way was modified
// Checks #65
assertRecords(t, []checkElem{
{"osm_landusages", -201191, "park", nil},
{"osm_landusages", -201192, "forest", nil},
{"osm_roads", 201151, "residential", nil},
})
}
func TestUnsupportedRelation(t *testing.T) {
// Unsupported relation type is not inserted with update
assertRecords(t, []checkElem{
{"osm_landusages", -201291, Missing, nil},
{"osm_landusages", 201251, "park", nil},
})
}
// #######################################################################
func TestDeployRevert(t *testing.T) {
if ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup)
}
ts.importOsm(t)
if !ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup)
}
ts.deployOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if !ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup)
}
ts.revertDeployOsm(t)
if !ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup)
}
}
func TestRemoveBackup(t *testing.T) {
if !ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup)
}
ts.deployOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if !ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup)
}
ts.removeBackupOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaImport)
}
if !ts.tableExists(t, dbschemaProduction, "osm_roads") {
t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction)
}
if ts.tableExists(t, dbschemaBackup, "osm_roads") {
t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup)
}
}

265
test/helper.py
View File

@ -1,265 +0,0 @@
import math
import tempfile
import shutil
import subprocess
import os
import psycopg2
import psycopg2.extras
import json
from shapely.wkb import loads as wkb_loads
from shapely.geometry import Point
import binascii
import unittest
__all__ = [
"assert_almost_equal",
"query_row",
"cache_query",
"merc_point",
"imposm3_import",
"imposm3_deploy",
"imposm3_update",
"imposm3_revert_deploy",
"imposm3_remove_backups",
"table_exists",
"drop_schemas",
"TEST_SCHEMA_IMPORT",
"TEST_SCHEMA_PRODUCTION",
"TEST_SCHEMA_BACKUP",
"db_conf",
"assert_missing_node",
"assert_cached_node",
"assert_cached_way",
]
class Dummy(unittest.TestCase):
def nop():
pass
_t = Dummy('nop')
assert_almost_equal = _t.assertAlmostEqual
tmpdir = None
def setup():
global tmpdir
tmpdir = tempfile.mkdtemp()
def teardown():
shutil.rmtree(tmpdir)
drop_schemas()
_close_test_connection(db_conf)
db_conf = {
'host': 'localhost',
}
TEST_SCHEMA_IMPORT = "imposm3testimport"
TEST_SCHEMA_PRODUCTION = "imposm3testpublic"
TEST_SCHEMA_BACKUP = "imposm3testbackup"
def merc_point(lon, lat):
pole = 6378137 * math.pi # 20037508.342789244
x = lon * pole / 180.0
y = math.log(math.tan((90.0+lat)*math.pi/360.0)) / math.pi * pole
return Point(x, y)
def pg_db_url(db_conf):
return 'postgis://%(host)s' % db_conf
def create_geom_in_row(rowdict):
if rowdict:
rowdict['geometry'] = wkb_loads(binascii.unhexlify(rowdict['geometry']))
return rowdict
def query_row(db_conf, table, osmid):
conn = _test_connection(db_conf)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute('select * from %s.%s where osm_id = %%s' % (TEST_SCHEMA_PRODUCTION, table), [osmid])
results = []
for row in cur.fetchall():
create_geom_in_row(row)
results.append(row)
cur.close()
if not results:
return None
if len(results) == 1:
return results[0]
return results
def query_duplicates(db_conf, table):
conn = _test_connection(db_conf)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute('select osm_id, count(osm_id) from %s.%s group by osm_id having count(osm_id) > 1' % (TEST_SCHEMA_PRODUCTION, table))
results = []
for row in cur.fetchall():
results.append(row)
cur.close()
return results
def imposm3_import(db_conf, pbf, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)
try:
print subprocess.check_output((
os.path.join("..", "imposm3") +
" import -connection %s -read %s"
" -write"
" -cachedir %s"
" -diff"
" -overwritecache"
" -dbschema-import " + TEST_SCHEMA_IMPORT +
" -optimize"
" -mapping %s ") % (
conn, pbf, tmpdir, mapping_file
), shell=True)
except subprocess.CalledProcessError, ex:
print ex.output
raise
def imposm3_deploy(db_conf, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)
try:
print subprocess.check_output((
os.path.join("..", "imposm3") +
" import -connection %s"
" -dbschema-import " + TEST_SCHEMA_IMPORT +
" -dbschema-production " + TEST_SCHEMA_PRODUCTION +
" -dbschema-backup " + TEST_SCHEMA_BACKUP +
" -deployproduction"
" -mapping %s ") % (
conn, mapping_file,
), shell=True)
except subprocess.CalledProcessError, ex:
print ex.output
raise
def imposm3_revert_deploy(db_conf, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)
try:
print subprocess.check_output((
os.path.join("..", "imposm3") +
" import -connection %s"
" -dbschema-import " + TEST_SCHEMA_IMPORT +
" -dbschema-production " + TEST_SCHEMA_PRODUCTION +
" -dbschema-backup " + TEST_SCHEMA_BACKUP +
" -revertdeploy"
" -mapping %s ") % (
conn, mapping_file,
), shell=True)
except subprocess.CalledProcessError, ex:
print ex.output
raise
def imposm3_remove_backups(db_conf, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)
try:
print subprocess.check_output((
os.path.join("..", "imposm3") +
" import -connection %s"
" -dbschema-backup " + TEST_SCHEMA_BACKUP +
" -removebackup"
" -mapping %s ") % (
conn, mapping_file,
), shell=True)
except subprocess.CalledProcessError, ex:
print ex.output
raise
def imposm3_update(db_conf, osc, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)
try:
print subprocess.check_output((
os.path.join("..", "imposm3") +
" diff -connection %s"
" -cachedir %s"
" -limitto clipping.geojson"
" -dbschema-production " + TEST_SCHEMA_PRODUCTION +
" -mapping %s %s") % (
conn, tmpdir, mapping_file, osc,
), shell=True)
except subprocess.CalledProcessError, ex:
print ex.output
raise
def cache_query(nodes='', ways='', relations='', deps='', full=''):
if nodes:
nodes = '-node ' + ','.join(map(str, nodes))
if ways:
ways = '-way ' + ','.join(map(str, ways))
if relations:
relations = '-rel ' + ','.join(map(str, relations))
if deps:
deps = '-deps'
if full:
full = '-full'
out = subprocess.check_output(
os.path.join("..", "imposm3") +
" query-cache -cachedir %s %s %s %s %s %s" % (
tmpdir, nodes, ways, relations, deps, full),
shell=True)
print out
return json.loads(out)
def _test_connection(db_conf):
if '_connection' in db_conf:
return db_conf['_connection']
db_conf['_connection'] = psycopg2.connect(**db_conf)
return db_conf['_connection']
def _close_test_connection(db_conf):
if '_connection' in db_conf:
db_conf['_connection'].close()
del db_conf['_connection']
def table_exists(table, schema=TEST_SCHEMA_IMPORT):
conn = _test_connection(db_conf)
cur = conn.cursor()
cur.execute("SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s' AND table_schema='%s')"
% (table, schema))
exists = cur.fetchone()[0]
cur.close()
return exists
def assert_missing_node(id):
data = cache_query(nodes=[id])
if data['nodes'][str(id)]:
raise AssertionError('node %d found' % id)
def assert_cached_node(id, (lon, lat)=(None, None)):
data = cache_query(nodes=[id])
node = data['nodes'][str(id)]
if not node:
raise AssertionError('node %d not found' % id)
if lon and lat:
assert_almost_equal(lon, node['lon'], 6)
assert_almost_equal(lat, node['lat'], 6)
def assert_cached_way(id):
data = cache_query(ways=[id])
if not data['ways'][str(id)]:
raise AssertionError('way %d not found' % id)
def drop_schemas():
conn = _test_connection(db_conf)
cur = conn.cursor()
cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_IMPORT)
cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_PRODUCTION)
cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_BACKUP)
conn.commit()

446
test/helper_test.go Normal file
View File

@ -0,0 +1,446 @@
package test
import (
"database/sql"
"fmt"
"log"
"math"
"strings"
"testing"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/cache"
"github.com/lib/pq/hstore"
"github.com/omniscale/imposm3/diff"
"github.com/omniscale/imposm3/geom/geos"
"github.com/omniscale/imposm3/config"
"github.com/omniscale/imposm3/import_"
)
const (
dbschemaImport = "imposm3testimport"
dbschemaProduction = "imposm3testproduction"
dbschemaBackup = "imposm3testbackup"
)
type importConfig struct {
connection string
osmFileName string
mappingFileName string
cacheDir string
verbose bool
}
type importTestSuite struct {
dir string
config importConfig
db *sql.DB
g *geos.Geos
}
const Missing = ""
func (s *importTestSuite) importOsm(t *testing.T) {
importArgs := []string{
"-connection", s.config.connection,
"-read", s.config.osmFileName,
"-write",
"-cachedir", s.config.cacheDir,
"-diff",
"-overwritecache",
"-dbschema-import", dbschemaImport,
// "-optimize",
"-mapping", s.config.mappingFileName,
"-quiet",
"-revertdeploy=false",
"-deployproduction=false",
"-removebackup=false",
}
config.ParseImport(importArgs)
import_.Import()
}
func (s *importTestSuite) deployOsm(t *testing.T) {
importArgs := []string{
"-read=", // overwrite previous options
"-write=false",
"-optimize=false",
"-revertdeploy=false",
"-deployproduction",
"-removebackup=false",
"-connection", s.config.connection,
"-dbschema-import", dbschemaImport,
"-dbschema-production", dbschemaProduction,
"-dbschema-backup", dbschemaBackup,
"-deployproduction",
"-mapping", s.config.mappingFileName,
"-quiet",
}
config.ParseImport(importArgs)
import_.Import()
}
func (s *importTestSuite) revertDeployOsm(t *testing.T) {
importArgs := []string{
"-read=", // overwrite previous options
"-write=false",
"-optimize=false",
"-revertdeploy",
"-deployproduction=false",
"-removebackup=false",
"-connection", s.config.connection,
"-dbschema-import", dbschemaImport,
"-dbschema-production", dbschemaProduction,
"-dbschema-backup", dbschemaBackup,
"-revertdeploy",
"-deployproduction=false",
"-removebackup=false",
"-mapping", s.config.mappingFileName,
"-quiet",
}
config.ParseImport(importArgs)
import_.Import()
}
func (s *importTestSuite) cache(t *testing.T) *cache.OSMCache {
c := cache.NewOSMCache(s.config.cacheDir)
if err := c.Open(); err != nil {
t.Fatal(err)
}
return c
}
func (s *importTestSuite) diffCache(t *testing.T) *cache.DiffCache {
c := cache.NewDiffCache(s.config.cacheDir)
if err := c.Open(); err != nil {
t.Fatal(err)
}
return c
}
func (s *importTestSuite) removeBackupOsm(t *testing.T) {
importArgs := []string{
"-read=", // overwrite previous options
"-write=false",
"-optimize=false",
"-revertdeploy=false",
"-deployproduction=false",
"-removebackup",
"-connection", s.config.connection,
"-dbschema-import", dbschemaImport,
"-dbschema-production", dbschemaProduction,
"-dbschema-backup", dbschemaBackup,
"-mapping", s.config.mappingFileName,
"-quiet",
}
config.ParseImport(importArgs)
import_.Import()
}
func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) {
args := []string{
"-connection", s.config.connection,
"-cachedir", s.config.cacheDir,
"-limitto", "clipping.geojson",
"-dbschema-production", dbschemaProduction,
"-mapping", s.config.mappingFileName,
diffFile,
}
config.ParseDiffImport(args)
diff.Diff()
}
func (s *importTestSuite) dropSchemas() {
var err error
_, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaImport))
if err != nil {
log.Fatal(err)
}
_, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaProduction))
if err != nil {
log.Fatal(err)
}
_, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaBackup))
if err != nil {
log.Fatal(err)
}
}
func (s *importTestSuite) tableExists(t *testing.T, schema, table string) bool {
row := s.db.QueryRow(fmt.Sprintf(`SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s' AND table_schema='%s')`, table, schema))
var exists bool
if err := row.Scan(&exists); err != nil {
t.Error(err)
return false
}
return exists
}
type record struct {
id int
name string
osmType string
wkt string
missing bool
tags map[string]string
}
func (s *importTestSuite) queryExists(t *testing.T, table string, id int64) bool {
row := s.db.QueryRow(fmt.Sprintf(`SELECT EXISTS(SELECT * FROM "%s"."%s" WHERE osm_id=$1)`, dbschemaProduction, table), id)
var exists bool
if err := row.Scan(&exists); err != nil {
t.Error(err)
return false
}
return exists
}
func (s *importTestSuite) query(t *testing.T, table string, id int64, keys []string) record {
kv := make([]string, len(keys))
for i, k := range keys {
kv[i] = "'" + k + "', " + k + "::varchar"
}
columns := strings.Join(kv, ", ")
if columns == "" {
columns = "''::hstore"
} else {
columns = "hstore(ARRAY[" + columns + "])"
}
stmt := fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry), %s FROM "%s"."%s" WHERE osm_id=$1`, columns, dbschemaProduction, table)
row := s.db.QueryRow(stmt, id)
r := record{}
h := hstore.Hstore{}
if err := row.Scan(&r.id, &r.name, &r.osmType, &r.wkt, &h); err != nil {
if err == sql.ErrNoRows {
r.missing = true
} else {
t.Fatal(err)
}
}
if len(h.Map) > 0 {
r.tags = make(map[string]string)
}
for k, v := range h.Map {
if v.Valid {
r.tags[k] = v.String
}
}
return r
}
func (s *importTestSuite) queryTags(t *testing.T, table string, id int64) record {
stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table)
row := s.db.QueryRow(stmt, id)
r := record{}
h := hstore.Hstore{}
if err := row.Scan(&r.id, &r.wkt, &h); err != nil {
if err == sql.ErrNoRows {
r.missing = true
} else {
t.Fatal(err)
}
}
if len(h.Map) > 0 {
r.tags = make(map[string]string)
}
for k, v := range h.Map {
if v.Valid {
r.tags[k] = v.String
}
}
return r
}
func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []record {
rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name, ST_GeometryType(geometry)`, dbschemaProduction, table), id)
if err != nil {
t.Fatal(err)
}
rs := []record{}
for rows.Next() {
var r record
if err := rows.Scan(&r.id, &r.name, &r.osmType, &r.wkt); err != nil {
t.Fatal(err)
}
rs = append(rs, r)
}
return rs
}
func (s *importTestSuite) queryRowsTags(t *testing.T, table string, id int64) []record {
rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1 ORDER BY ST_GeometryType(geometry)`, dbschemaProduction, table), id)
if err != nil {
t.Fatal(err)
}
rs := []record{}
for rows.Next() {
var r record
h := hstore.Hstore{}
if err := rows.Scan(&r.id, &r.wkt, &h); err != nil {
t.Fatal(err)
}
if len(h.Map) > 0 {
r.tags = make(map[string]string)
}
for k, v := range h.Map {
if v.Valid {
r.tags[k] = v.String
}
}
rs = append(rs, r)
}
return rs
}
func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos.Geom {
stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table)
row := s.db.QueryRow(stmt, id)
r := record{}
if err := row.Scan(&r.id, &r.wkt); err != nil {
if err == sql.ErrNoRows {
r.missing = true
} else {
t.Fatal(err)
}
}
g := geos.NewGeos()
defer g.Finish()
geom := g.FromWkt(r.wkt)
if geom == nil {
t.Fatalf("unable to read WKT for %s", id)
}
return geom
}
type checkElem struct {
table string
id int64
osmType string
tags map[string]string
}
func assertRecordsMissing(t *testing.T, elems []checkElem) {
for _, e := range elems {
if ts.queryExists(t, e.table, e.id) {
t.Errorf("found %d in %d", e.id, e.table)
}
}
}
func assertRecords(t *testing.T, elems []checkElem) {
for _, e := range elems {
keys := make([]string, 0, len(e.tags))
for k, _ := range e.tags {
keys = append(keys, k)
}
r := ts.query(t, e.table, e.id, keys)
if e.osmType == "" {
if r.missing {
continue
}
t.Errorf("got unexpected record %d", r.id)
}
if r.osmType != e.osmType {
t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id)
}
for k, v := range e.tags {
if r.tags[k] != v {
t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v)
}
}
}
}
func assertHstore(t *testing.T, elems []checkElem) {
for _, e := range elems {
r := ts.queryTags(t, e.table, e.id)
if e.osmType == "" {
if r.missing {
continue
}
t.Errorf("got unexpected record %d", r.id)
}
if len(e.tags) != len(r.tags) {
t.Errorf("tags for %d differ %v != %v", e.id, r.tags, e.tags)
}
for k, v := range e.tags {
if r.tags[k] != v {
t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v)
}
}
}
}
func assertValid(t *testing.T, e checkElem) {
geom := ts.queryGeom(t, e.table, e.id)
if !ts.g.IsValid(geom) {
t.Fatalf("geometry of %d is invalid", e.id)
}
}
func assertArea(t *testing.T, e checkElem, expect float64) {
geom := ts.queryGeom(t, e.table, e.id)
if !ts.g.IsValid(geom) {
t.Fatalf("geometry of %d is invalid", e.id)
}
actual := geom.Area()
if math.Abs(expect-actual) > 1 {
t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect)
}
}
func assertLength(t *testing.T, e checkElem, expect float64) {
geom := ts.queryGeom(t, e.table, e.id)
if !ts.g.IsValid(geom) {
t.Fatalf("geometry of %d is invalid", e.id)
}
actual := geom.Length()
if math.Abs(expect-actual) > 1 {
t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect)
}
}
func assertGeomType(t *testing.T, e checkElem, expect string) {
actual := ts.g.Type(ts.queryGeom(t, e.table, e.id))
if actual != expect {
t.Errorf("expected %s geometry for %d, got %s", expect, e.id, actual)
}
}
func assertCachedWay(t *testing.T, c *cache.OSMCache, id int64) *element.Way {
way, err := c.Ways.GetWay(id)
if err == cache.NotFound {
t.Errorf("missing way %d", id)
} else if err != nil {
t.Fatal(err)
}
if way.Id != id {
t.Errorf("cached way contains invalid id, %d != %d", way.Id, id)
}
return way
}
func assertCachedNode(t *testing.T, c *cache.OSMCache, id int64) *element.Node {
node, err := c.Nodes.GetNode(id)
if err == cache.NotFound {
node, err = c.Coords.GetCoord(id)
if err == cache.NotFound {
t.Errorf("missing node %d", id)
return nil
}
} else if err != nil {
t.Fatal(err)
}
if node.Id != id {
t.Errorf("cached node contains invalid id, %d != %d", node.Id, id)
}
return node
}

185
test/single_table_test.go Normal file
View File

@ -0,0 +1,185 @@
package test
import (
"database/sql"
"strings"
"testing"
"github.com/omniscale/imposm3/geom/geos"
)
const RelOffset = -1e17
func TestSingleTable_Prepare(t *testing.T) {
ts.dir = "/tmp/imposm3test"
ts.config = importConfig{
connection: "postgis://",
cacheDir: ts.dir,
osmFileName: "build/single_table.pbf",
mappingFileName: "single_table_mapping.json",
}
ts.g = geos.NewGeos()
var err error
ts.db, err = sql.Open("postgres", "sslmode=disable")
if err != nil {
t.Fatal(err)
}
ts.dropSchemas()
}
func TestSingleTable_Import(t *testing.T) {
if ts.tableExists(t, dbschemaImport, "osm_all") != false {
t.Fatalf("table osm_all exists in schema %s", dbschemaImport)
}
ts.importOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_all") != true {
t.Fatalf("table osm_all does not exists in schema %s", dbschemaImport)
}
}
func TestSingleTable_Deploy(t *testing.T) {
ts.deployOsm(t)
if ts.tableExists(t, dbschemaImport, "osm_all") != false {
t.Fatalf("table osm_all exists in schema %s", dbschemaImport)
}
if ts.tableExists(t, dbschemaProduction, "osm_all") != true {
t.Fatalf("table osm_all does not exists in schema %s", dbschemaProduction)
}
}
func TestSingleTable_NonMappedNodeIsMissing(t *testing.T) {
// Node without mapped tags is missing.
cache := ts.cache(t)
defer cache.Close()
assertCachedNode(t, cache, 10001)
assertHstore(t, []checkElem{
{"osm_all", 10001, Missing, nil},
})
}
func TestSingleTable_MappedNode(t *testing.T) {
// Node is stored with all tags.
cache := ts.cache(t)
defer cache.Close()
assertCachedNode(t, cache, 10002)
assertHstore(t, []checkElem{
{"osm_all", 10002, "*", map[string]string{"random": "tag", "but": "mapped", "poi": "unicorn"}},
})
}
func TestSingleTable_NonMappedWayIsMissing(t *testing.T) {
// Way without mapped tags is missing.
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 20101)
assertCachedWay(t, cache, 20102)
assertCachedWay(t, cache, 20103)
assertHstore(t, []checkElem{
{"osm_all", 20101, Missing, nil},
{"osm_all", 20102, Missing, nil},
{"osm_all", 20103, Missing, nil},
})
}
func TestSingleTable_MappedWay(t *testing.T) {
// Way is stored with all tags.
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 20201)
assertHstore(t, []checkElem{
{"osm_all", -20201, "*", map[string]string{"random": "tag", "highway": "yes"}},
})
}
func TestSingleTable_NonMappedClosedWayIsMissing(t *testing.T) {
// Closed way without mapped tags is missing.
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 20301)
assertHstore(t, []checkElem{
{"osm_all", 20301, Missing, nil},
{"osm_all", -20301, Missing, nil},
})
}
func TestSingleTable_MappedClosedWay(t *testing.T) {
// Closed way is stored with all tags.
assertHstore(t, []checkElem{
{"osm_all", -20401, "*", map[string]string{"random": "tag", "building": "yes"}},
})
}
func TestSingleTable_MappedClosedWayAreaYes(t *testing.T) {
// Closed way with area=yes is not stored as linestring.
assertHstore(t, []checkElem{
{"osm_all", -20501, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "yes"}},
})
assertGeomType(t, checkElem{"osm_all", -20501, "*", nil}, "Polygon")
}
func TestSingleTable_MappedClosedWayAreaNo(t *testing.T) {
// Closed way with area=no is not stored as polygon.
assertHstore(t, []checkElem{
{"osm_all", -20502, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "no"}},
})
assertGeomType(t, checkElem{"osm_all", -20502, "*", nil}, "LineString")
}
func TestSingleTable_MappedClosedWayWithoutArea(t *testing.T) {
// Closed way without area is stored as mapped (linestring and polygon).
rows := ts.queryRowsTags(t, "osm_all", -20601)
if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") {
t.Errorf("unexpected geometries: %v", rows)
}
}
func TestSingleTable_DuplicateIds1(t *testing.T) {
// Points/lines/polygons with same ID are inserted.
assertHstore(t, []checkElem{
{"osm_all", 31101, "*", map[string]string{"amenity": "cafe"}},
})
rows := ts.queryRowsTags(t, "osm_all", -31101)
if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") {
t.Errorf("unexpected geometries: %v", rows)
}
assertHstore(t, []checkElem{
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
})
assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
}
// #######################################################################
func TestSingleTable_Update(t *testing.T) {
ts.updateOsm(t, "./build/single_table.osc.gz")
}
// #######################################################################
func TestSingleTable_DuplicateIds2(t *testing.T) {
// Node moved and ways/rels with same ID are still present.
assertHstore(t, []checkElem{
{"osm_all", 31101, "*", map[string]string{"amenity": "cafe"}},
})
rows := ts.queryRowsTags(t, "osm_all", -31101)
if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") {
t.Errorf("unexpected geometries: %v", rows)
}
assertHstore(t, []checkElem{
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
})
assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
}

184
test/single_table_test.py
View File

@ -1,184 +0,0 @@
import psycopg2
import psycopg2.extras
import helper as t
psycopg2.extras.register_hstore(psycopg2.connect(**t.db_conf), globally=True)
mapping_file = 'single_table_mapping.json'
def setup():
t.setup()
def teardown():
t.teardown()
RELOFFSET = int(-1e17)
#######################################################################
def test_import():
"""Import succeeds"""
t.drop_schemas()
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
t.imposm3_import(t.db_conf, './build/single_table.pbf', mapping_file)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
def test_deploy():
"""Deploy succeeds"""
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
t.imposm3_deploy(t.db_conf, mapping_file)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
#######################################################################
def test_non_mapped_node_is_missing():
"""Node without mapped tags is missing."""
t.assert_cached_node(10001, (10, 42))
assert not t.query_row(t.db_conf, 'osm_all', 10001)
def test_mapped_node():
"""Node is stored with all tags."""
t.assert_cached_node(10002, (11, 42))
poi = t.query_row(t.db_conf, 'osm_all', 10002)
assert poi['tags'] == {'random': 'tag', 'but': 'mapped', 'poi': 'unicorn'}
def test_non_mapped_way_is_missing():
"""Way without mapped tags is missing."""
t.assert_cached_way(20101)
assert not t.query_row(t.db_conf, 'osm_all', 20101)
t.assert_cached_way(20102)
assert not t.query_row(t.db_conf, 'osm_all', 20102)
t.assert_cached_way(20103)
assert not t.query_row(t.db_conf, 'osm_all', 20103)
def test_mapped_way():
"""Way is stored with all tags."""
t.assert_cached_way(20201)
highway = t.query_row(t.db_conf, 'osm_all', -20201)
assert highway['tags'] == {'random': 'tag', 'highway': 'yes'}
def test_non_mapped_closed_way_is_missing():
"""Closed way without mapped tags is missing."""
t.assert_cached_way(20301)
assert not t.query_row(t.db_conf, 'osm_all', -20301)
def test_mapped_closed_way():
"""Closed way is stored with all tags."""
t.assert_cached_way(20401)
building = t.query_row(t.db_conf, 'osm_all', -20401)
assert building['tags'] == {'random': 'tag', 'building': 'yes'}
def test_mapped_closed_way_area_yes():
"""Closed way with area=yes is not stored as linestring."""
t.assert_cached_way(20501)
elem = t.query_row(t.db_conf, 'osm_all', -20501)
assert elem['geometry'].type == 'Polygon', elem['geometry'].type
assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'yes'}
def test_mapped_closed_way_area_no():
"""Closed way with area=no is not stored as polygon."""
t.assert_cached_way(20502)
elem = t.query_row(t.db_conf, 'osm_all', -20502)
assert elem['geometry'].type == 'LineString', elem['geometry'].type
assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'no'}
def test_mapped_closed_way_without_area():
"""Closed way without area is stored as mapped (linestring and polygon)."""
t.assert_cached_way(20601)
elems = t.query_row(t.db_conf, 'osm_all', -20601)
assert len(elems) == 2
elems.sort(key=lambda x: x['geometry'].type)
assert elems[0]['geometry'].type == 'LineString', elems[0]['geometry'].type
assert elems[0]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'}
assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type
assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'}
def test_duplicate_ids_1():
"""Points/lines/polygons with same ID are inserted."""
node = t.query_row(t.db_conf, 'osm_all', 31101)
assert node['geometry'].type == 'Point', node['geometry'].type
assert node['tags'] == {'amenity': 'cafe'}
assert node['geometry'].distance(t.merc_point(80, 47)) < 1
ways = t.query_row(t.db_conf, 'osm_all', -31101)
ways.sort(key=lambda x: x['geometry'].type)
assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type
assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type
assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L)
assert rel['geometry'].type == 'Polygon', rel['geometry'].type
assert rel['tags'] == {'building': 'yes'}
#######################################################################
def test_update():
"""Diff import applies"""
t.imposm3_update(t.db_conf, './build/single_table.osc.gz', mapping_file)
#######################################################################
def test_duplicate_ids_2():
"""Node moved and ways/rels with same ID are still present."""
node = t.query_row(t.db_conf, 'osm_all', 31101)
assert node['geometry'].type == 'Point', node['geometry'].type
assert node['tags'] == {'amenity': 'cafe'}
assert node['geometry'].distance(t.merc_point(81, 47)) < 1
ways = t.query_row(t.db_conf, 'osm_all', -31101)
ways.sort(key=lambda x: x['geometry'].type)
assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type
assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type
assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L)
assert rel['geometry'].type == 'Polygon', rel['geometry'].type
assert rel['tags'] == {'building': 'yes'}
#######################################################################
def test_deploy_and_revert_deploy():
"""Revert deploy succeeds"""
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)
# import again to have a new import schema
t.imposm3_import(t.db_conf, './build/single_table.pbf', mapping_file)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
t.imposm3_deploy(t.db_conf, mapping_file)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_revert_deploy(t.db_conf, mapping_file)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)
def test_remove_backup():
"""Remove backup succeeds"""
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_deploy(t.db_conf, mapping_file)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)
t.imposm3_remove_backups(t.db_conf, mapping_file)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT)
assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION)
assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP)