From d3452eafda705c022809cb1abd2ed04c3413eff9 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 30 Sep 2015 14:30:55 +0200 Subject: [PATCH 01/36] improve tests and actually fix #65 and #66 previous fix only worked by accident because way and relation ids were the same --- diff/deleter.go | 6 ++++-- test/complete_db.osc | 14 +++++++------- test/complete_db.osm | 24 ++++++++++++------------ test/complete_db_test.py | 10 +++++----- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/diff/deleter.go b/diff/deleter.go index 75d039d..c872f68 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -116,8 +116,10 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) if _, ok := d.deletedWays[member.Id]; ok { continue } - if err := d.deleteRelation(member.Id, false, false); err != nil { - return err + for _, r := range d.diffCache.Ways.Get(member.Id) { + if err := d.deleteRelation(r, false, false); err != nil { + return err + } } if err := d.deleteWay(member.Id, false); err != nil { return err diff --git a/test/complete_db.osc b/test/complete_db.osc index a5c45f7..5f411d2 100644 --- a/test/complete_db.osc +++ b/test/complete_db.osc @@ -190,9 +190,9 @@ - - - + + + @@ -200,8 +200,8 @@ - - + + @@ -210,8 +210,8 @@ - - + + diff --git a/test/complete_db.osm b/test/complete_db.osm index 6073721..4839da7 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -1055,9 +1055,9 @@ - - - + + + @@ -1065,8 +1065,8 @@ - - + + @@ -1075,9 +1075,9 @@ relation (201101) with shared way (checks #65) --> - - - + + + @@ -1085,13 +1085,13 @@ - - + + - - + + diff --git a/test/complete_db_test.py b/test/complete_db_test.py index 3a675c9..a4daa44 100644 --- a/test/complete_db_test.py +++ b/test/complete_db_test.py @@ -319,9 +319,9 @@ def test_no_duplicates(): """ highways = t.query_duplicates(t.db_conf, 'osm_roads') # one duplicate for test_node_way_inserted_twice is expected - assert highways == [[18001, 2]] + assert highways == [[18001, 2]], highways landusages = t.query_duplicates(t.db_conf, 'osm_landusages') - assert not landusages + assert not landusages, landusages def test_updated_landusage(): """Multipolygon relation was modified""" @@ -480,9 +480,9 @@ def test_no_duplicate_insert(): Relation is not inserted again if a nother relation with the same way was modified Checks #65 """ - assert t.query_row(t.db_conf, 'osm_landusages', -201101)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages', -201102)['type'] == 'forest' - assert t.query_row(t.db_conf, 'osm_roads', 201101)['type'] == 'residential' + assert t.query_row(t.db_conf, 'osm_landusages', -201191)['type'] == 'park' + assert t.query_row(t.db_conf, 'osm_landusages', -201192)['type'] == 'forest' + assert t.query_row(t.db_conf, 'osm_roads', 201151)['type'] == 'residential' ####################################################################### def test_deploy_and_revert_deploy(): From 1b347cde96c88c7d5d65ab8602b39a2833ee7305 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 7 Oct 2015 10:21:57 +0200 Subject: [PATCH 02/36] force sort order of pbf test files --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 516540e..433405f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -20,7 +20,7 @@ clean: build/%.pbf: %.osm @mkdir -p build - osmosis --read-xml $< --write-pbf $@ omitmetadata=true + osmosis --read-xml $< --sort TypeThenId --write-pbf $@ omitmetadata=true build/%.osc.gz: %.osc @mkdir -p build From 4195e24dee253c802b08b6046f7d03337525f1f9 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 7 Oct 2015 10:22:30 +0200 Subject: [PATCH 03/36] do not insert relations with unsupported types during diff import --- diff/process.go | 6 +++++- test/complete_db.osc | 8 ++++++++ test/complete_db.osm | 14 ++++++++++++++ test/complete_db_test.py | 7 +++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/diff/process.go b/diff/process.go index 3f7f46f..a77f664 100644 --- a/diff/process.go +++ b/diff/process.go @@ -290,7 +290,11 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi } // insert new relation progress.AddRelations(1) - relations <- rel + // filter out unsupported relation types, otherwise they might + // get inserted with the tags from an outer way + if relTagFilter.Filter(&rel.Tags) { + relations <- rel + } } for wayId, _ := range wayIds { diff --git a/test/complete_db.osc b/test/complete_db.osc index 5f411d2..b3c8c1c 100644 --- a/test/complete_db.osc +++ b/test/complete_db.osc @@ -217,4 +217,12 @@ + + + + + + + + diff --git a/test/complete_db.osm b/test/complete_db.osm index 4839da7..1c84f7c 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -1096,4 +1096,18 @@ + + + + + + + + + + + + + + diff --git a/test/complete_db_test.py b/test/complete_db_test.py index a4daa44..5d51b02 100644 --- a/test/complete_db_test.py +++ b/test/complete_db_test.py @@ -484,6 +484,13 @@ def test_no_duplicate_insert(): assert t.query_row(t.db_conf, 'osm_landusages', -201192)['type'] == 'forest' assert t.query_row(t.db_conf, 'osm_roads', 201151)['type'] == 'residential' +def test_unsupported_relation(): + """ + Unsupported relation type is not inserted with update + """ + assert not t.query_row(t.db_conf, 'osm_landusages', -201291) + assert t.query_row(t.db_conf, 'osm_landusages', 201251)['type'] == 'park' + ####################################################################### def test_deploy_and_revert_deploy(): """Revert deploy succeeds""" From d384f35a5414cc22baf636f8da2ac7bc0d1fa3cc Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 9 Oct 2015 16:46:46 +0000 Subject: [PATCH 04/36] Setting password without escaping For me on Debian testing setting the password only worked without the escaping of the single quotes. --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 9e9119a..2a54101 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -24,7 +24,7 @@ This is step zero, since you have to do it only once. The following commands cre createdb -E UTF8 -O osm osm psql -d osm -c "CREATE EXTENSION postgis;" psql -d osm -c "CREATE EXTENSION hstore;" # only required for hstore support - echo "ALTER USER osm WITH PASSWORD \'osm\';" |psql -d osm + echo "ALTER USER osm WITH PASSWORD 'osm';" |psql -d osm You can change the names if you like, but we will use `osm` for user name, password and database name in all following examples. From 12f92b99cc23ac0dfecd7161da29a79d1fc04eac Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 14 Oct 2015 15:01:45 +0200 Subject: [PATCH 05/36] update "Planned features" --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b9138b0..633ae2e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ It is designed to create databases that are optimized for rendering (i.e. genera Imposm 3 is written in Go and it is a complete rewrite of the previous Python implementation. Configurations/mappings and cache files are not compatible with Imposm 2, but they share a similar architecture. -The development of Imposm 3 was sponsored by [Omniscale](http://omniscale.com/) and development will continue as resources permit. -Please get in touch if you need commercial support or if you need specific features. +The development of Imposm 3 was sponsored by [Omniscale](http://omniscale.com/). There are [commercial licenses available for Imposm](http://omniscale.com/opensource/soss) to support the long-term development of Imposm. +There is also commercial support available from Omniscale. Features @@ -80,15 +80,20 @@ Import of Europe 11GB PBF with generalized tables: Current status -------------- -Imposm 3 is used in production but there is no official release yet. +Imposm 3 is used in production but there is no official 3.0 release yet. -### Missing ### +### Planned features ### -Compared to Imposm 2: +There are a few features we like to see in Imposm 3: +* Automatic download and import of differential files * Support for other projections than EPSG:3857 or EPSG:4326 -* Import of XML files (unlikely to be implemented in the future, use [osmosis](http://wiki.openstreetmap.org/wiki/Osmosis) to convert XML to PBF first) +* Support for route relations +* Improved integration with tile servers (expiration of updated tiles) * Custom field/filter functions +* Official releases with binaries for more platforms + +There is no roadmap however, as the implementation of these features largely depends on external funding. There are [commercial licenses available for Imposm](http://omniscale.com/opensource/soss) if you like to help with this development. Installation ------------ From b3b388b11b0e4090f7a185b30ca752bdc6a9642c Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 20 Oct 2015 13:20:48 +0200 Subject: [PATCH 06/36] import multipolygons if at least on ring was build --- geom/multipolygon.go | 14 ++++----- geom/multipolygon_test.go | 61 ++++++++++++++++++++++++++++++++++++++- test/complete_db.osm | 25 ++++++++++++++++ test/complete_db_test.py | 5 ++++ 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/geom/multipolygon.go b/geom/multipolygon.go index 2d2b887..f5f4ca8 100644 --- a/geom/multipolygon.go +++ b/geom/multipolygon.go @@ -93,23 +93,23 @@ func buildRings(rel *element.Relation, maxRingGap float64) ([]*ring, error) { } // merge incomplete rings mergedRings = mergeRings(incompleteRings) - if len(completeRings)+len(mergedRings) == 0 { - err = ErrorNoRing // for defer - return nil, err - } + // create geometries for merged rings for _, ring := range mergedRings { if !ring.isClosed() && !ring.tryClose(maxRingGap) { - err = ErrorNoRing // for defer - return nil, err + continue } ring.geom, err = Polygon(g, ring.nodes) if err != nil { return nil, err } + completeRings = append(completeRings, ring) } - completeRings = append(completeRings, mergedRings...) + if len(completeRings) == 0 { + err = ErrorNoRing // for defer + return nil, err + } // sort by area (large to small) for _, r := range completeRings { diff --git a/geom/multipolygon_test.go b/geom/multipolygon_test.go index fae9b14..bdc6cad 100644 --- a/geom/multipolygon_test.go +++ b/geom/multipolygon_test.go @@ -31,7 +31,6 @@ func buildRelation(rel *element.Relation, srid int) (Geometry, error) { if err != nil { return Geometry{}, err } - return prep.Build() } @@ -599,3 +598,63 @@ func TestBrokenPolygonSelfIntersectTriangle(t *testing.T) { } } + +func TestOpenRing(t *testing.T) { + w1 := makeWay(1, element.Tags{}, []coord{ + {1, 0, 0}, + {2, 10, 0}, + {3, 10, 10}, + {4, 0, 10}, + }) + + rel := element.Relation{ + OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{}}} + rel.Members = []element.Member{ + {1, element.WAY, "outer", &w1}, + } + + _, err := buildRelation(&rel, 3857) + if err == nil { + t.Fatal("no error from open ring") + } +} + +func TestClosedAndOpenRing(t *testing.T) { + w1 := makeWay(1, element.Tags{}, []coord{ + {1, 0, 0}, + {2, 10, 0}, + {3, 10, 10}, + {4, 0, 10}, + {1, 0, 0}, + }) + w2 := makeWay(2, element.Tags{}, []coord{ + {5, 0, 0}, + {6, -5, -2}, + }) + rel := element.Relation{ + OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{}}} + rel.Members = []element.Member{ + {1, element.WAY, "outer", &w1}, + {2, element.WAY, "outer", &w2}, + } + + prep, err := PrepareRelation(&rel, 3857, 0.1) + if err != nil { + t.Fatal(err) + } + // open ring is excluded + if len(prep.rings) != 1 { + t.Fatal("expected single ring") + } + geom, err := prep.Build() + if err != nil { + t.Fatal(err) + } + + g := geos.NewGeos() + defer g.Finish() + + if !g.IsValid(geom.Geom) { + t.Fatal("geometry not valid", g.AsWkt(geom.Geom)) + } +} diff --git a/test/complete_db.osm b/test/complete_db.osm index 1c84f7c..a6b7b8c 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -386,6 +386,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/complete_db_test.py b/test/complete_db_test.py index 5d51b02..aa5cf48 100644 --- a/test/complete_db_test.py +++ b/test/complete_db_test.py @@ -284,6 +284,11 @@ def test_ring_with_gap(): park = t.query_row(t.db_conf, 'osm_landusages', 7311) assert park['geometry'].is_valid, park +def test_multipolygon_with_open_ring(): + """Multipolygon is inserted even if there is an open ring/member""" + park = t.query_row(t.db_conf, 'osm_landusages', -7401) + assert park['geometry'].is_valid, park + def test_updated_nodes1(): """Zig-Zag line is inserted.""" road = t.query_row(t.db_conf, 'osm_roads', 60000) From b466940a19cdd60cf1d0b4438cfc499c8f207390 Mon Sep 17 00:00:00 2001 From: Luc-Edmond Gaspard Date: Wed, 18 Nov 2015 11:48:51 -0500 Subject: [PATCH 07/36] use the right character to separate the directories in the pathname for the current os --- test/helper.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/helper.py b/test/helper.py index 3361754..d89492e 100644 --- a/test/helper.py +++ b/test/helper.py @@ -2,6 +2,7 @@ import math import tempfile import shutil import subprocess +import os import psycopg2 import psycopg2.extras import json @@ -106,7 +107,8 @@ def imposm3_import(db_conf, pbf, mapping_file): try: print subprocess.check_output(( - "../imposm3 import -connection %s -read %s" + os.path.join("..", "imposm3") + + " import -connection %s -read %s" " -write" " -cachedir %s" " -diff" @@ -126,7 +128,8 @@ def imposm3_deploy(db_conf, mapping_file): try: print subprocess.check_output(( - "../imposm3 import -connection %s" + os.path.join("..", "imposm3") + + " import -connection %s" " -dbschema-import " + TEST_SCHEMA_IMPORT + " -dbschema-production " + TEST_SCHEMA_PRODUCTION + " -dbschema-backup " + TEST_SCHEMA_BACKUP + @@ -144,7 +147,8 @@ def imposm3_revert_deploy(db_conf, mapping_file): try: print subprocess.check_output(( - "../imposm3 import -connection %s" + os.path.join("..", "imposm3") + + " import -connection %s" " -dbschema-import " + TEST_SCHEMA_IMPORT + " -dbschema-production " + TEST_SCHEMA_PRODUCTION + " -dbschema-backup " + TEST_SCHEMA_BACKUP + @@ -162,7 +166,8 @@ def imposm3_remove_backups(db_conf, mapping_file): try: print subprocess.check_output(( - "../imposm3 import -connection %s" + os.path.join("..", "imposm3") + + " import -connection %s" " -dbschema-backup " + TEST_SCHEMA_BACKUP + " -removebackup" " -mapping %s ") % ( @@ -178,7 +183,8 @@ def imposm3_update(db_conf, osc, mapping_file): try: print subprocess.check_output(( - "../imposm3 diff -connection %s" + os.path.join("..", "imposm3") + + " diff -connection %s" " -cachedir %s" " -limitto clipping.geojson" " -dbschema-production " + TEST_SCHEMA_PRODUCTION + @@ -201,7 +207,8 @@ def cache_query(nodes='', ways='', relations='', deps='', full=''): if full: full = '-full' out = subprocess.check_output( - "../imposm3 query-cache -cachedir %s %s %s %s %s %s" % ( + os.path.join("..", "imposm3") + + " query-cache -cachedir %s %s %s %s %s %s" % ( tmpdir, nodes, ways, relations, deps, full), shell=True) print out From 838123acc20661ba1fdd4822ebe4e2f198623f38 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 20 Nov 2015 15:03:08 +0100 Subject: [PATCH 08/36] fix Benchmarks --- cache/diff_test.go | 3 ++- geom/geom_test.go | 3 ++- mapping/matcher_test.go | 2 +- parser/pbf/pbf_test.go | 10 ++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cache/diff_test.go b/cache/diff_test.go index 1993cd7..e00fd1f 100644 --- a/cache/diff_test.go +++ b/cache/diff_test.go @@ -90,9 +90,10 @@ func BenchmarkWriteDiff(b *testing.B) { for i := 0; i < b.N; i++ { for w := 0; w < 5; w++ { for n := 0; n < 200; n++ { - cache.addc <- idRef{id: int64(n), ref: int64(w)} + cache.Add(int64(n), int64(w)) } } + cache.Flush() } } diff --git a/geom/geom_test.go b/geom/geom_test.go index 658dd05..6db71e7 100644 --- a/geom/geom_test.go +++ b/geom/geom_test.go @@ -1,9 +1,10 @@ package geom import ( + "testing" + "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom/geos" - "testing" ) func TestLineString(t *testing.T) { diff --git a/mapping/matcher_test.go b/mapping/matcher_test.go index 9a4b062..8e5efa2 100644 --- a/mapping/matcher_test.go +++ b/mapping/matcher_test.go @@ -7,7 +7,7 @@ import ( ) func BenchmarkTagMatch(b *testing.B) { - m, err := NewMapping("matcher_test_mapping.yml") + m, err := NewMapping("test_mapping.yml") if err != nil { b.Fatal(err) } diff --git a/parser/pbf/pbf_test.go b/parser/pbf/pbf_test.go index d7de08c..a4d1bd4 100644 --- a/parser/pbf/pbf_test.go +++ b/parser/pbf/pbf_test.go @@ -3,7 +3,6 @@ package pbf import ( "bytes" "compress/zlib" - "fmt" "io" "log" "os" @@ -15,13 +14,12 @@ import ( func BenchmarkHello(b *testing.B) { b.StopTimer() - pbf, err := Open("../azores.osm.pbf") + pbf, err := Open("./monaco-20150428.osm.pbf") if err != nil { panic(err) } for pos := range pbf.BlockPositions() { - fmt.Println(pos.size, pos.offset) b.StartTimer() for i := 0; i < b.N; i++ { readPrimitiveBlock(pos) @@ -43,7 +41,7 @@ func BenchmarkHello(b *testing.B) { func BenchmarkPrimitiveBlock(b *testing.B) { b.StopTimer() - file, err := os.Open("../azores.osm.pbf") + file, err := os.Open("./monaco-20150428.osm.pbf") if err != nil { log.Panic(err) } @@ -52,8 +50,8 @@ func BenchmarkPrimitiveBlock(b *testing.B) { var block = &osmpbf.PrimitiveBlock{} var blob = &osmpbf.Blob{} - var size = 56092 - var offset int64 = 197 + var size = 79566 + var offset int64 = 155 blobData := make([]byte, size) file.Seek(offset, 0) From b8530cb803abaa5512b978580f5d6c6628a9cf51 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 20 Nov 2015 22:54:53 +0100 Subject: [PATCH 09/36] remove freeBuffer/freeNode hack --- cache/delta.go | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/cache/delta.go b/cache/delta.go index 871593e..39c0684 100644 --- a/cache/delta.go +++ b/cache/delta.go @@ -322,10 +322,6 @@ func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error { return nil } -var ( - freeBuffer = make(chan []byte, 4) -) - func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error { keyBuf := idToKeyBuf(bunchId) @@ -333,12 +329,7 @@ func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) return p.db.Delete(p.wo, keyBuf) } - var data []byte - select { - case data = <-freeBuffer: - default: - } - + data := make([]byte, 512) data = binary.MarshalDeltaNodes(nodes, data) err := p.db.Put(p.wo, keyBuf, data) @@ -346,11 +337,6 @@ func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) return err } - select { - case freeBuffer <- data: - default: - } - return nil } @@ -377,10 +363,6 @@ func (self *DeltaCoordsCache) getBunchId(nodeId int64) int64 { return nodeId / self.bunchSize } -var ( - freeNodes = make(chan []element.Node, 4) -) - func (self *DeltaCoordsCache) getBunch(bunchId int64) (*coordsBunch, error) { self.mu.Lock() bunch, ok := self.table[bunchId] @@ -388,12 +370,7 @@ func (self *DeltaCoordsCache) getBunch(bunchId int64) (*coordsBunch, error) { needsGet := false if !ok { elem := self.lruList.PushFront(bunchId) - select { - case nodes = <-freeNodes: - nodes = nodes[:0] - default: - nodes = make([]element.Node, 0, self.bunchSize) - } + nodes = make([]element.Node, 0, self.bunchSize) bunch = &coordsBunch{id: bunchId, coords: nodes, elem: elem} needsGet = true self.table[bunchId] = bunch @@ -429,10 +406,6 @@ func (self *DeltaCoordsCache) CheckCapacity() error { return err } } - select { - case freeNodes <- bunch.coords: - default: - } delete(self.table, bunchId) } return nil From f5d16e170b87b00d81bdf4c9cef3c4b216e0e433 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 20 Nov 2015 23:07:26 +0100 Subject: [PATCH 10/36] remove unused deltacoords proto code --- cache/binary/deltacoords_test.go | 101 ------------------------------- cache/delta.go | 52 ---------------- 2 files changed, 153 deletions(-) diff --git a/cache/binary/deltacoords_test.go b/cache/binary/deltacoords_test.go index 33869e9..6873bc7 100644 --- a/cache/binary/deltacoords_test.go +++ b/cache/binary/deltacoords_test.go @@ -6,7 +6,6 @@ import ( "runtime" "testing" - "github.com/golang/protobuf/proto" "github.com/omniscale/imposm3/element" ) @@ -73,103 +72,3 @@ func BenchmarkUnmarshalDeltaCoords(b *testing.B) { runtime.GC() } - -func BenchmarkMarshalDeltaCoordsProto(b *testing.B) { - var buf []byte - var err error - - for n := 0; n < b.N; n++ { - deltaCoords := packNodes(nodes) - buf, err = proto.Marshal(deltaCoords) - if err != nil { - panic(err) - } - } - - deltaCoords := &DeltaCoords{} - err = proto.Unmarshal(buf, deltaCoords) - if err != nil { - panic(err) - } - - nodes2 := unpackNodes(deltaCoords, nodes) - - compareNodes(b, nodes, nodes2) - runtime.GC() - -} - -func BenchmarkUnmarshalDeltaCoordsProto(b *testing.B) { - var buf []byte - var err error - - deltaCoords := packNodes(nodes) - buf, err = proto.Marshal(deltaCoords) - if err != nil { - panic(err) - } - var nodes2 []element.Node - for n := 0; n < b.N; n++ { - deltaCoords := &DeltaCoords{} - err = proto.Unmarshal(buf, deltaCoords) - if err != nil { - panic(err) - } - nodes2 = unpackNodes(deltaCoords, nodes) - } - compareNodes(b, nodes, nodes2) - runtime.GC() - -} - -func packNodes(nodes []element.Node) *DeltaCoords { - var lastLon, lastLat int64 - var lon, lat int64 - var lastId int64 - ids := make([]int64, len(nodes)) - lons := make([]int64, len(nodes)) - lats := make([]int64, len(nodes)) - - i := 0 - for _, nd := range nodes { - lon = int64(CoordToInt(nd.Long)) - lat = int64(CoordToInt(nd.Lat)) - ids[i] = nd.Id - lastId - lons[i] = lon - lastLon - lats[i] = lat - lastLat - - lastId = nd.Id - lastLon = lon - lastLat = lat - i++ - } - return &DeltaCoords{Ids: ids, Lats: lats, Lons: lons} -} - -func unpackNodes(deltaCoords *DeltaCoords, nodes []element.Node) []element.Node { - if len(deltaCoords.Ids) > cap(nodes) { - nodes = make([]element.Node, len(deltaCoords.Ids)) - } else { - nodes = nodes[:len(deltaCoords.Ids)] - } - - var lastLon, lastLat int64 - var lon, lat int64 - var lastId, id int64 - - for i := 0; i < len(deltaCoords.Ids); i++ { - id = lastId + deltaCoords.Ids[i] - lon = lastLon + deltaCoords.Lons[i] - lat = lastLat + deltaCoords.Lats[i] - nodes[i] = element.Node{ - OSMElem: element.OSMElem{Id: int64(id)}, - Long: IntToCoord(uint32(lon)), - Lat: IntToCoord(uint32(lat)), - } - - lastId = id - lastLon = lon - lastLat = lat - } - return nodes -} diff --git a/cache/delta.go b/cache/delta.go index 39c0684..569e3de 100644 --- a/cache/delta.go +++ b/cache/delta.go @@ -15,58 +15,6 @@ func (s byId) Len() int { return len(s) } func (s byId) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s byId) Less(i, j int) bool { return s[i].Id < s[j].Id } -func packNodes(nodes []element.Node) *binary.DeltaCoords { - var lastLon, lastLat int64 - var lon, lat int64 - var lastId int64 - ids := make([]int64, len(nodes)) - lons := make([]int64, len(nodes)) - lats := make([]int64, len(nodes)) - - i := 0 - for _, nd := range nodes { - lon = int64(binary.CoordToInt(nd.Long)) - lat = int64(binary.CoordToInt(nd.Lat)) - ids[i] = nd.Id - lastId - lons[i] = lon - lastLon - lats[i] = lat - lastLat - - lastId = nd.Id - lastLon = lon - lastLat = lat - i++ - } - return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons} -} - -func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node { - if len(deltaCoords.Ids) > cap(nodes) { - nodes = make([]element.Node, len(deltaCoords.Ids)) - } else { - nodes = nodes[:len(deltaCoords.Ids)] - } - - var lastLon, lastLat int64 - var lon, lat int64 - var lastId, id int64 - - for i := 0; i < len(deltaCoords.Ids); i++ { - id = lastId + deltaCoords.Ids[i] - lon = lastLon + deltaCoords.Lons[i] - lat = lastLat + deltaCoords.Lats[i] - nodes[i] = element.Node{ - OSMElem: element.OSMElem{Id: int64(id)}, - Long: binary.IntToCoord(uint32(lon)), - Lat: binary.IntToCoord(uint32(lat)), - } - - lastId = id - lastLon = lon - lastLat = lat - } - return nodes -} - type coordsBunch struct { sync.Mutex id int64 From b3110b3bb5bf1e9736488f1e992526468e46ede8 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Sat, 21 Nov 2015 18:05:07 +0100 Subject: [PATCH 11/36] add benchmarks for deltacoords --- cache/delta_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/cache/delta_test.go b/cache/delta_test.go index 15d431b..f6a4093 100644 --- a/cache/delta_test.go +++ b/cache/delta_test.go @@ -1,12 +1,13 @@ package cache import ( - "github.com/omniscale/imposm3/element" "io/ioutil" "math/rand" "os" "sort" "testing" + + "github.com/omniscale/imposm3/element" ) func mknode(id int64) element.Node { @@ -185,3 +186,64 @@ func TestSingleUpdate(t *testing.T) { insertAndCheck(t, cache, 4, 4, 4) } + +func BenchmarkWriteDeltaCoords(b *testing.B) { + b.StopTimer() + cache_dir, _ := ioutil.TempDir("", "imposm3_test") + defer os.RemoveAll(cache_dir) + + cache, err := newDeltaCoordsCache(cache_dir) + if err != nil { + b.Fatal() + } + defer cache.Close() + + nodes := make([]element.Node, 10000) + for i := range nodes { + nodes[i].Id = rand.Int63n(50000) + nodes[i].Long = rand.Float64() - 0.5*360 + nodes[i].Lat = rand.Float64() - 0.5*180 + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + for _, n := range nodes { + if err := cache.PutCoords([]element.Node{n}); err != nil { + b.Fatal(err) + } + } + } +} + +func BenchmarkReadDeltaCoords(b *testing.B) { + b.StopTimer() + cache_dir, _ := ioutil.TempDir("", "imposm3_test") + defer os.RemoveAll(cache_dir) + + cache, err := newDeltaCoordsCache(cache_dir) + if err != nil { + b.Fatal() + } + defer cache.Close() + + nodes := make([]element.Node, 10000) + for i := range nodes { + nodes[i].Id = rand.Int63n(50000) + nodes[i].Long = rand.Float64() - 0.5*360 + nodes[i].Lat = rand.Float64() - 0.5*180 + } + for _, n := range nodes { + if err := cache.PutCoords([]element.Node{n}); err != nil { + b.Fatal(err) + } + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + for n := 0; n < 10000; n++ { + if _, err := cache.GetCoord(int64(n)); err != nil && err != NotFound { + b.Fatal(err) + } + } + } +} From 7f1a5cb999f2c4efe0168030c512797d95d35161 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Sat, 21 Nov 2015 18:10:20 +0100 Subject: [PATCH 12/36] validate polygons from ways --- geom/geos/geos.go | 17 ++++++++++++++++- geom/multipolygon.go | 11 ++++------- test/complete_db_test.py | 6 +++--- writer/ways.go | 3 +++ 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/geom/geos/geos.go b/geom/geos/geos.go index 77b59d3..dfc9cb3 100644 --- a/geom/geos/geos.go +++ b/geom/geos/geos.go @@ -13,9 +13,11 @@ extern void initGEOS_debug(); import "C" import ( - "github.com/omniscale/imposm3/logging" + "errors" "runtime" "unsafe" + + "github.com/omniscale/imposm3/logging" ) var log = logging.NewLogger("GEOS") @@ -268,6 +270,19 @@ func (this *Geos) Equals(a, b *Geom) bool { return false } +func (g *Geos) MakeValid(geom *Geom) (*Geom, error) { + if g.IsValid(geom) { + return geom, nil + } + fixed := g.Buffer(geom, 0) + if fixed == nil { + return nil, errors.New("Error while fixing geom with buffer(0)") + } + g.Destroy(geom) + + return fixed, nil +} + func (this *Geom) Area() float64 { var area C.double if ret := C.GEOSArea(this.v, &area); ret == 1 { diff --git a/geom/multipolygon.go b/geom/multipolygon.go index f5f4ca8..e5e631a 100644 --- a/geom/multipolygon.go +++ b/geom/multipolygon.go @@ -196,13 +196,10 @@ func buildRelGeometry(g *geos.Geos, rel *element.Relation, rings []*ring) (*geos return nil, errors.New("Error while building multi-polygon.") } } - if !g.IsValid(result) { - buffered := g.Buffer(result, 0) - if buffered == nil { - return nil, errors.New("Error while fixing geom with buffer(0)") - } - g.Destroy(result) - result = buffered + var err error + result, err = g.MakeValid(result) + if err != nil { + return nil, err } g.DestroyLater(result) diff --git a/test/complete_db_test.py b/test/complete_db_test.py index aa5cf48..ecde4f8 100644 --- a/test/complete_db_test.py +++ b/test/complete_db_test.py @@ -252,10 +252,10 @@ def test_duplicate_ids(): def test_generalized_banana_polygon_is_valid(): """Generalized polygons are valid.""" park = t.query_row(t.db_conf, 'osm_landusages', 7101) - # geometry is not valid - assert not park['geometry'].is_valid, park + # geometry is valid + assert park['geometry'].is_valid, park park = t.query_row(t.db_conf, 'osm_landusages_gen0', 7101) - # but simplified geometies are valid + # simplified geometies are valid too assert park['geometry'].is_valid, park park = t.query_row(t.db_conf, 'osm_landusages_gen1', 7101) assert park['geometry'].is_valid, park diff --git a/writer/ways.go b/writer/ways.go index be0be00..e9a1b2c 100644 --- a/writer/ways.go +++ b/writer/ways.go @@ -129,6 +129,9 @@ func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches []mapp if isPolygon { geosgeom, err = geomp.Polygon(g, way.Nodes) + if err == nil { + geosgeom, err = g.MakeValid(geosgeom) + } } else { geosgeom, err = geomp.LineString(g, way.Nodes) } From 3e5c83695758b96180c0485f805743fc48092ba0 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Sat, 21 Nov 2015 18:14:26 +0100 Subject: [PATCH 13/36] add extraTags to geometry tables --- mapping/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapping/config.go b/mapping/config.go index 8264dca..78de91d 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -242,7 +242,7 @@ func (m *Mapping) tables(tableType TableType) map[string]*TableFields { func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { for _, t := range m.Tables { - if t.Type != tableType { + if t.Type != tableType && t.Type != "geometry" { continue } for key, _ := range t.ExtraTags() { From 23ff4867e8f9334860f3df888327ef711a6a3910 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Sun, 22 Nov 2015 11:09:15 +0100 Subject: [PATCH 14/36] add first go system test --- geom/geos/geos.go | 7 + test/completedb_test.go | 412 ++++++++++++++++++++++++++++++++++++++++ test/helper_test.go | 179 +++++++++++++++++ 3 files changed, 598 insertions(+) create mode 100644 test/completedb_test.go create mode 100644 test/helper_test.go diff --git a/geom/geos/geos.go b/geom/geos/geos.go index dfc9cb3..d4d3a0d 100644 --- a/geom/geos/geos.go +++ b/geom/geos/geos.go @@ -246,6 +246,13 @@ func (this *Geos) IsValid(geom *Geom) bool { return false } +func (this *Geos) IsSimple(geom *Geom) bool { + if C.GEOSisSimple_r(this.v, geom.v) == 1 { + return true + } + return false +} + func (this *Geos) IsEmpty(geom *Geom) bool { if C.GEOSisEmpty_r(this.v, geom.v) == 1 { return true diff --git a/test/completedb_test.go b/test/completedb_test.go new file mode 100644 index 0000000..ecef74d --- /dev/null +++ b/test/completedb_test.go @@ -0,0 +1,412 @@ +package test + +import ( + "database/sql" + + "math" + "testing" + + "github.com/omniscale/imposm3/geom/geos" +) + +var ts importTestSuite + +func TestPrepare(t *testing.T) { + ts.dir = "/tmp/imposm3test" + ts.config = importConfig{ + connection: "postgis://", + cacheDir: ts.dir, + osmFileName: "build/complete_db.pbf", + mappingFileName: "complete_db_mapping.json", + } + ts.g = geos.NewGeos() + + var err error + ts.db, err = sql.Open("postgres", "sslmode=disable") + if err != nil { + t.Fatal(err) + } + ts.dropSchemas() +} + +func TestImport(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_roads") != false { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + ts.importOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_roads") != true { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } +} + +func TestDeploy(t *testing.T) { + ts.deployOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_roads") != false { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if ts.tableExists(t, dbschemaProduction, "osm_roads") != true { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } +} + +type checkElem struct { + table string + id int64 + osmType string + tags map[string]string +} + +func assertRecordsMissing(t *testing.T, elems []checkElem) { + for _, e := range elems { + if ts.queryExists(t, e.table, e.id) { + t.Errorf("found %d in %d", e.id, e.table) + } + } +} + +func assertRecords(t *testing.T, elems []checkElem) { + for _, e := range elems { + keys := make([]string, 0, len(e.tags)) + for k, _ := range e.tags { + keys = append(keys, k) + } + r := ts.query(t, e.table, e.id, keys) + if e.osmType == "" { + if r.missing { + continue + } + t.Errorf("got unexpected record %d", r.id) + } + if r.osmType != e.osmType { + t.Errorf("got unexpected type %s != %s", r.osmType, e.osmType) + } + for k, v := range e.tags { + if r.tags[k] != v { + t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) + } + } + } +} + +func assertValid(t *testing.T, e checkElem) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } +} + +func assertArea(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Area() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func assertLength(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Length() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func TestLandusageToWaterarea1(t *testing.T) { + // Parks inserted into landusages + // t.assert_cached_way(11001) + // t.assert_cached_way(12001) + // t.assert_cached_way(13001) + + assertRecords(t, []checkElem{ + {"osm_waterareas", 11001, "", nil}, + {"osm_waterareas", -12001, "", nil}, + {"osm_waterareas", -13001, "", nil}, + + {"osm_waterareas_gen0", 11001, "", nil}, + {"osm_waterareas_gen0", -12001, "", nil}, + {"osm_waterareas_gen0", -13001, "", nil}, + + {"osm_waterareas_gen1", 11001, "", nil}, + {"osm_waterareas_gen1", -12001, "", nil}, + {"osm_waterareas_gen1", -13001, "", nil}, + + {"osm_landusages", 11001, "park", nil}, + {"osm_landusages", -12001, "park", nil}, + {"osm_landusages", -13001, "park", nil}, + + {"osm_landusages_gen0", 11001, "park", nil}, + {"osm_landusages_gen0", -12001, "park", nil}, + {"osm_landusages_gen0", -13001, "park", nil}, + + {"osm_landusages_gen1", 11001, "park", nil}, + {"osm_landusages_gen1", -12001, "park", nil}, + {"osm_landusages_gen1", -13001, "park", nil}, + }) +} + +func TestChangedHoleTags1(t *testing.T) { + // Multipolygon relation with untagged hole + // t.assert_cached_way(14001) + // t.assert_cached_way(14011) + + assertRecords(t, []checkElem{ + {"osm_waterareas", 14011, "", nil}, + {"osm_waterareas", -14011, "", nil}, + {"osm_landusages", -14001, "park", nil}, + }) +} + +func TestSplitOuterMultipolygonWay1(t *testing.T) { + // Single outer way of multipolygon was inserted. + assertRecords(t, []checkElem{ + {"osm_roads", 15002, "", nil}, + {"osm_landusages", -15001, "park", nil}, + }) + assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452) +} + +func TestMergeOuterMultipolygonWay1(t *testing.T) { + // Splitted outer way of multipolygon was inserted. + assertRecords(t, []checkElem{ + {"osm_landusages", -16001, "park", nil}, + {"osm_roads", 16002, "residential", nil}, + }) + assertArea(t, checkElem{"osm_landusages", -16001, "park", nil}, 12779350582) +} + +func TestBrokenMultipolygonWays(t *testing.T) { + // MultiPolygons with broken outer ways are handled. + // outer way does not merge (17002 has one node) + + assertRecords(t, []checkElem{ + {"osm_landusages", -17001, "", nil}, + {"osm_roads", 17001, "residential", nil}, + {"osm_roads", 17002, "", nil}, + }) + + // outer way does not merge (17102 has no nodes) + assertRecords(t, []checkElem{ + {"osm_landusages", -17101, "", nil}, + {"osm_roads", 17101, "residential", nil}, + {"osm_roads", 17102, "", nil}, + }) +} + +func TestNodeWayInsertedTwice(t *testing.T) { + // Way with multiple mappings is inserted twice in same table + // rows = t.query_row(t.db_conf, 'osm_roads', 18001) + // rows.sort(key=lambda x: x['type']) + rows := ts.queryRows(t, "osm_roads", 18001) + if len(rows) != 2 || rows[0].osmType != "residential" || rows[1].osmType != "tram" { + t.Errorf("unexpected roads: %v", rows) + } +} + +func TestOuterWayNotInserted(t *testing.T) { + // Outer way with different tag is not inserted twice into same table + + assertRecords(t, []checkElem{ + {"osm_landusages", -19001, "farmland", nil}, + {"osm_landusages", 19002, "farmyard", nil}, + {"osm_landusages", 19001, "", nil}, + }) +} + +func TestOuterWayInserted(t *testing.T) { + // Outer way with different tag is inserted twice into different table + + assertRecords(t, []checkElem{ + {"osm_landusages", 19101, "farm", nil}, + {"osm_landusages", 19102, "farmyard", nil}, + {"osm_admin", -19101, "administrative", nil}, + }) +} + +func TestNodeWayRefAfterDelete1(t *testing.T) { + // Nodes refereces way + // data = t.cache_query(nodes=[20001, 20002], deps=True) + // assert '20001' in data['nodes']['20001']['ways'] + // assert '20001' in data['nodes']['20002']['ways'] + assertRecords(t, []checkElem{ + {"osm_roads", 20001, "residential", nil}, + {"osm_barrierpoints", 20001, "block", nil}, + }) +} + +func TestWayRelRefAfterDelete1(t *testing.T) { + // Ways references relation + // data = t.cache_query(ways=[21001], deps=True) + // assert data['ways']['21001']['relations'].keys() == ['21001'] + assertRecords(t, []checkElem{ + {"osm_roads", 21001, "residential", nil}, + {"osm_landusages", -21001, "park", nil}, + }) +} + +func TestRelationWayNotInserted(t *testing.T) { + // Part of relation was inserted only once. + + assertRecords(t, []checkElem{ + {"osm_landusages", -9001, "park", map[string]string{"name": "rel 9001"}}, + {"osm_landusages", 9009, "", nil}, + {"osm_landusages", -9101, "park", map[string]string{"name": "rel 9101"}}, + {"osm_landusages", 9109, "", nil}, + {"osm_landusages", 9110, "scrub", nil}, + }) +} + +func TestRelationWaysInserted(t *testing.T) { + // Outer ways of multipolygon are inserted. + + assertRecords(t, []checkElem{ + {"osm_landusages", -9201, "park", map[string]string{"name": "9209"}}, + {"osm_landusages", 9201, "", nil}, + // outer ways of multipolygon stand for their own + {"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}}, + {"osm_roads", 9210, "residential", map[string]string{"name": "9210"}}, + + // no name on relation + {"osm_landusages", -9301, "park", map[string]string{"name": ""}}, + // outer ways of multipolygon stand for their own + {"osm_roads", 9309, "secondary", map[string]string{"name": "9309"}}, + {"osm_roads", 9310, "residential", map[string]string{"name": "9310"}}, + }) + +} + +func TestRelationWayInserted(t *testing.T) { + // Part of relation was inserted twice. + + assertRecords(t, []checkElem{ + {"osm_landusages", -8001, "park", map[string]string{"name": "rel 8001"}}, + {"osm_roads", 8009, "residential", nil}, + }) +} + +func TestSingleNodeWaysNotInserted(t *testing.T) { + // Ways with single/duplicate nodes are not inserted. + + assertRecords(t, []checkElem{ + {"osm_landusages", 30001, "", nil}, + {"osm_landusages", 30002, "", nil}, + {"osm_landusages", 30003, "", nil}, + }) +} + +func TestPolygonWithDuplicateNodesIsValid(t *testing.T) { + // Polygon with duplicate nodes is valid. + + assertValid(t, checkElem{"osm_landusages", 30005, "park", nil}) +} + +func TestIncompletePolygons(t *testing.T) { + // Non-closed/incomplete polygons are not inserted. + + assertRecords(t, []checkElem{ + {"osm_landusages", 30004, "", nil}, + {"osm_landusages", 30006, "", nil}, + }) +} + +func TestResidentialToSecondary(t *testing.T) { + // Residential road is not in roads_gen0/1. + + assertRecords(t, []checkElem{ + {"osm_roads", 40001, "residential", nil}, + {"osm_roads_gen0", 40001, "", nil}, + {"osm_roads_gen1", 40002, "", nil}, + }) +} + +func TestRelationBeforeRemove(t *testing.T) { + // Relation and way is inserted. + + assertRecords(t, []checkElem{ + {"osm_buildings", 50011, "yes", nil}, + {"osm_landusages", -50021, "park", nil}, + }) +} + +func TestRelationWithoutTags(t *testing.T) { + // Relation without tags is inserted. + + assertRecords(t, []checkElem{ + {"osm_buildings", 50111, "", nil}, + {"osm_buildings", -50121, "yes", nil}, + }) +} + +func TestDuplicateIds(t *testing.T) { + // Relation/way with same ID is inserted. + + assertRecords(t, []checkElem{ + {"osm_buildings", 51001, "way", nil}, + {"osm_buildings", -51001, "mp", nil}, + {"osm_buildings", 51011, "way", nil}, + {"osm_buildings", -51011, "mp", nil}, + }) +} + +func TestGeneralizedBananaPolygonIsValid(t *testing.T) { + // Generalized polygons are valid. + + assertValid(t, checkElem{"osm_landusages", 7101, "", nil}) + // simplified geometies are valid too + assertValid(t, checkElem{"osm_landusages_gen0", 7101, "", nil}) + assertValid(t, checkElem{"osm_landusages_gen1", 7101, "", nil}) +} + +func TestGeneralizedLinestringIsValid(t *testing.T) { + // Generalized linestring is valid. + + // geometry is not simple, but valid + assertLength(t, checkElem{"osm_roads", 7201, "primary", nil}, 1243660.044819) + if ts.g.IsSimple(ts.queryGeom(t, "osm_roads", 7201)) { + t.Errorf("expected non-simple geometry for 7201") + } + // check that geometry 'survives' simplification + assertLength(t, checkElem{"osm_roads_gen0", 7201, "primary", nil}, 1243660.044819) + assertLength(t, checkElem{"osm_roads_gen1", 7201, "primary", nil}, 1243660.044819) +} + +func TestRingWithGap(t *testing.T) { + // Multipolygon and way with gap (overlapping but different endpoints) gets closed + assertValid(t, checkElem{"osm_landusages", -7301, "", nil}) + assertValid(t, checkElem{"osm_landusages", 7311, "", nil}) +} + +func TestMultipolygonWithOpenRing(t *testing.T) { + // Multipolygon is inserted even if there is an open ring/member + assertValid(t, checkElem{"osm_landusages", -7401, "", nil}) +} + +func TestUpdatedNodes1(t *testing.T) { + // Zig-Zag line is inserted. + assertLength(t, checkElem{"osm_roads", 60000, "", nil}, 14035.61150207768) +} + +func TestUpdateNodeToCoord1(t *testing.T) { + // Node is inserted with tag. + // coords = t.cache_query(nodes=(70001, 70002)) + // assert coords['nodes']["70001"]["tags"] == {"amenity": "police"} + // assert "tags" not in coords['nodes']["70002"] + assertRecords(t, []checkElem{ + {"osm_amenities", 70001, "police", nil}, + {"osm_amenities", 70002, "", nil}, + }) +} + +func TestEnumerateKey(t *testing.T) { + // Enumerate from key. + assertRecords(t, []checkElem{ + {"osm_landusages", 100001, "park", map[string]string{"enum": "1"}}, + {"osm_landusages", 100002, "park", map[string]string{"enum": "0"}}, + {"osm_landusages", 100003, "wood", map[string]string{"enum": "15"}}, + }) +} diff --git a/test/helper_test.go b/test/helper_test.go new file mode 100644 index 0000000..c8ffe7c --- /dev/null +++ b/test/helper_test.go @@ -0,0 +1,179 @@ +package test + +import ( + "database/sql" + "fmt" + "log" + "strings" + "testing" + + "github.com/lib/pq/hstore" + + "github.com/omniscale/imposm3/geom/geos" + + "github.com/omniscale/imposm3/config" + "github.com/omniscale/imposm3/import_" +) + +const ( + dbschemaImport = "imposm3testimport" + dbschemaProduction = "imposm3testproduction" + dbschemaBackup = "imposm3testbackup" +) + +type importConfig struct { + connection string + osmFileName string + mappingFileName string + cacheDir string + verbose bool +} + +type importTestSuite struct { + dir string + config importConfig + db *sql.DB + g *geos.Geos +} + +func (s *importTestSuite) importOsm(t *testing.T) { + importArgs := []string{ + "-connection", s.config.connection, + "-read", s.config.osmFileName, + "-write", + "-cachedir", s.config.cacheDir, + "-diff", + "-overwritecache", + "-dbschema-import", dbschemaImport, + // "-optimize", + "-mapping", s.config.mappingFileName, + "-quiet", + "-deployproduction=false", + } + + config.ParseImport(importArgs) + import_.Import() +} + +func (s *importTestSuite) deployOsm(t *testing.T) { + importArgs := []string{ + "-read=", // overwrite previous options + "-write=false", + "-optimize=false", + "-connection", s.config.connection, + "-dbschema-import", dbschemaImport, + "-dbschema-production", dbschemaProduction, + "-dbschema-backup", dbschemaBackup, + "-deployproduction", + "-mapping", s.config.mappingFileName, + "-quiet", + } + + config.ParseImport(importArgs) + import_.Import() +} + +func (s *importTestSuite) dropSchemas() { + var err error + _, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaImport)) + if err != nil { + log.Fatal(err) + } + _, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaProduction)) + if err != nil { + log.Fatal(err) + } + _, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaBackup)) + if err != nil { + log.Fatal(err) + } +} + +func (s *importTestSuite) tableExists(t *testing.T, schema, table string) bool { + row := s.db.QueryRow(fmt.Sprintf(`SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s' AND table_schema='%s')`, table, schema)) + var exists bool + if err := row.Scan(&exists); err != nil { + t.Error(err) + return false + } + return exists +} + +type record struct { + id int + name string + osmType string + wkt string + missing bool + tags map[string]string +} + +func (s *importTestSuite) queryExists(t *testing.T, table string, id int64) bool { + row := s.db.QueryRow(fmt.Sprintf(`SELECT EXISTS(SELECT * FROM "%s"."%s" WHERE osm_id=$1)`, dbschemaProduction, table), id) + var exists bool + if err := row.Scan(&exists); err != nil { + t.Error(err) + return false + } + return exists +} + +func (s *importTestSuite) query(t *testing.T, table string, id int64, keys []string) record { + kv := make([]string, len(keys)) + for i, k := range keys { + kv[i] = "'" + k + "', " + k + "::varchar" + } + columns := strings.Join(kv, ", ") + if columns == "" { + columns = "''::hstore" + } else { + columns = "hstore(ARRAY[" + columns + "])" + } + stmt := fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry), %s FROM "%s"."%s" WHERE osm_id=$1`, columns, dbschemaProduction, table) + row := s.db.QueryRow(stmt, id) + r := record{} + h := hstore.Hstore{} + if err := row.Scan(&r.id, &r.name, &r.osmType, &r.wkt, &h); err != nil { + if err == sql.ErrNoRows { + r.missing = true + } else { + t.Fatal(err) + } + } + if len(h.Map) > 0 { + r.tags = make(map[string]string) + } + for k, v := range h.Map { + if v.Valid { + r.tags[k] = v.String + } + } + return r +} + +func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []record { + rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name`, dbschemaProduction, table), id) + if err != nil { + t.Fatal(err) + } + rs := []record{} + for rows.Next() { + var r record + if err := rows.Scan(&r.id, &r.name, &r.osmType, &r.wkt); err != nil { + t.Fatal(err) + } + rs = append(rs, r) + } + return rs +} + +func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos.Geom { + r := s.query(t, table, id, nil) + g := geos.NewGeos() + defer g.Finish() + geom := g.FromWkt(r.wkt) + if geom == nil { + t.Fatalf("unable to read WKT for %s", id) + } + return geom +} From 15111cab7a21c5debe14f2841d7c9b66031dd8b7 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Sun, 22 Nov 2015 12:25:35 +0100 Subject: [PATCH 15/36] add diff tests --- cmd/main.go | 47 +------- diff/process.go | 45 ++++++++ test/completedb_test.go | 245 +++++++++++++++++++++++++++++++++++++++- test/helper_test.go | 14 +++ 4 files changed, 304 insertions(+), 47 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index e7f8cfe..9f7a27b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,12 +6,9 @@ import ( "os" "runtime" - "github.com/omniscale/imposm3/cache" - "github.com/omniscale/imposm3/cache/query" "github.com/omniscale/imposm3/config" "github.com/omniscale/imposm3/diff" - "github.com/omniscale/imposm3/geom/limit" "github.com/omniscale/imposm3/import_" "github.com/omniscale/imposm3/logging" "github.com/omniscale/imposm3/stats" @@ -53,49 +50,7 @@ func Main(usage func()) { if config.BaseOptions.Httpprofile != "" { stats.StartHttpPProf(config.BaseOptions.Httpprofile) } - - if config.BaseOptions.Quiet { - logging.SetQuiet(true) - } - - var geometryLimiter *limit.Limiter - if config.BaseOptions.LimitTo != "" { - var err error - step := log.StartStep("Reading limitto geometries") - geometryLimiter, err = limit.NewFromGeoJSON( - config.BaseOptions.LimitTo, - config.BaseOptions.LimitToCacheBuffer, - config.BaseOptions.Srid, - ) - if err != nil { - log.Fatal(err) - } - log.StopStep(step) - } - osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir) - err := osmCache.Open() - if err != nil { - log.Fatal("osm cache: ", err) - } - defer osmCache.Close() - - diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir) - err = diffCache.Open() - if err != nil { - log.Fatal("diff cache: ", err) - } - - for _, oscFile := range config.DiffFlags.Args() { - err := diff.Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false) - if err != nil { - osmCache.Close() - diffCache.Close() - log.Fatalf("unable to process %s: %v", oscFile, err) - } - } - // explicitly Close since os.Exit prevents defers - osmCache.Close() - diffCache.Close() + diff.Diff() case "query-cache": query.Query(os.Args[2:]) diff --git a/diff/process.go b/diff/process.go index a77f664..ca378fa 100644 --- a/diff/process.go +++ b/diff/process.go @@ -24,6 +24,51 @@ import ( var log = logging.NewLogger("diff") +func Diff() { + if config.BaseOptions.Quiet { + logging.SetQuiet(true) + } + + var geometryLimiter *limit.Limiter + if config.BaseOptions.LimitTo != "" { + var err error + step := log.StartStep("Reading limitto geometries") + geometryLimiter, err = limit.NewFromGeoJSON( + config.BaseOptions.LimitTo, + config.BaseOptions.LimitToCacheBuffer, + config.BaseOptions.Srid, + ) + if err != nil { + log.Fatal(err) + } + log.StopStep(step) + } + osmCache := cache.NewOSMCache(config.BaseOptions.CacheDir) + err := osmCache.Open() + if err != nil { + log.Fatal("osm cache: ", err) + } + defer osmCache.Close() + + diffCache := cache.NewDiffCache(config.BaseOptions.CacheDir) + err = diffCache.Open() + if err != nil { + log.Fatal("diff cache: ", err) + } + + for _, oscFile := range config.DiffFlags.Args() { + err := Update(oscFile, geometryLimiter, nil, osmCache, diffCache, false) + if err != nil { + osmCache.Close() + diffCache.Close() + log.Fatalf("unable to process %s: %v", oscFile, err) + } + } + // explicitly Close since os.Exit prevents defers + osmCache.Close() + diffCache.Close() +} + func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expireor, osmCache *cache.OSMCache, diffCache *cache.DiffCache, force bool) error { state, err := diffstate.ParseFromOsc(oscFile) if err != nil { diff --git a/test/completedb_test.go b/test/completedb_test.go index ecef74d..fbc0bd7 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -3,6 +3,10 @@ package test import ( "database/sql" + "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/geom" + "github.com/omniscale/imposm3/proj" + "math" "testing" @@ -78,7 +82,7 @@ func assertRecords(t *testing.T, elems []checkElem) { t.Errorf("got unexpected record %d", r.id) } if r.osmType != e.osmType { - t.Errorf("got unexpected type %s != %s", r.osmType, e.osmType) + t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id) } for k, v := range e.tags { if r.tags[k] != v { @@ -410,3 +414,242 @@ func TestEnumerateKey(t *testing.T) { {"osm_landusages", 100003, "wood", map[string]string{"enum": "15"}}, }) } + +func TestUpdate(t *testing.T) { + ts.updateOsm(t, "./build/complete_db.osc.gz") +} + +func TestNoDuplicates(t *testing.T) { + // Relations/ways are only inserted once Checks #66 + + // highways = t.query_duplicates(t.db_conf, 'osm_roads') + // # one duplicate for test_node_way_inserted_twice is expected + // assert highways == [[18001, 2]], highways + // landusages = t.query_duplicates(t.db_conf, 'osm_landusages') + // assert not landusages, landusages +} + +func TestUpdatedLandusage(t *testing.T) { + // Multipolygon relation was modified + + // t.assert_cached_node(1001, (13.5, 47.5)) + nd := element.Node{Long: 13.4, Lat: 47.5} + proj.NodeToMerc(&nd) + point, err := geom.Point(ts.g, nd) + if err != nil { + t.Fatal(err) + } + poly := ts.queryGeom(t, "osm_landusages", -1001) + // point not in polygon after update + if ts.g.Intersects(point, poly) { + t.Error("point intersects polygon") + } +} + +func TestPartialDelete(t *testing.T) { + // Deleted relation but nodes are still cached + + // t.assert_cached_node(2001) + // t.assert_cached_way(2001) + // t.assert_cached_way(2002) + + assertRecords(t, []checkElem{ + {"osm_landusages", -2001, "", nil}, + {"osm_landusages", 2001, "", nil}, + }) +} + +func TestUpdatedNodes(t *testing.T) { + // Nodes were added, modified or deleted + + // t.assert_missing_node(10000) + // t.assert_cached_node(10001, (10.0, 40.0)) + // t.assert_cached_node(10002, (10.1, 40.0)) + + assertRecords(t, []checkElem{ + {"osm_places", 10001, "village", map[string]string{"name": "Bar"}}, + {"osm_places", 10002, "city", map[string]string{"name": "Baz"}}, + }) +} + +func TestLandusageToWaterarea2(t *testing.T) { + // Parks converted to water moved from landusages to waterareas + + assertRecords(t, []checkElem{ + + {"osm_waterareas", 11001, "water", nil}, + {"osm_waterareas", -12001, "water", nil}, + {"osm_waterareas", -13001, "water", nil}, + + {"osm_waterareas_gen0", 11001, "water", nil}, + {"osm_waterareas_gen0", -12001, "water", nil}, + {"osm_waterareas_gen0", -13001, "water", nil}, + + {"osm_waterareas_gen1", 11001, "water", nil}, + {"osm_waterareas_gen1", -12001, "water", nil}, + {"osm_waterareas_gen1", -13001, "water", nil}, + + {"osm_landusages", 11001, "", nil}, + {"osm_landusages", -12001, "", nil}, + {"osm_landusages", -13001, "", nil}, + + {"osm_landusages_gen0", 11001, "", nil}, + {"osm_landusages_gen0", -12001, "", nil}, + {"osm_landusages_gen0", -13001, "", nil}, + + {"osm_landusages_gen1", 11001, "", nil}, + {"osm_landusages_gen1", -12001, "", nil}, + {"osm_landusages_gen1", -13001, "", nil}, + }) +} + +func TestChangedHoleTags2(t *testing.T) { + // Newly tagged hole is inserted + + // t.assert_cached_way(14001) + // t.assert_cached_way(14011) + + assertArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779) + assertArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182) +} + +func TestSplitOuterMultipolygonWay2(t *testing.T) { + // Splitted outer way of multipolygon was inserted + + // data = t.cache_query(ways=[15001, 15002], deps=True) + // assert data['ways']['15001']['relations'].keys() == ['15001'] + // assert data['ways']['15002']['relations'].keys() == ['15001'] + + assertRecords(t, []checkElem{ + {"osm_landusages", 15001, "", nil}, + {"osm_roads", 15002, "residential", nil}, + }) + assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452) +} + +func TestMergeOuterMultipolygonWay2(t *testing.T) { + // Merged outer way of multipolygon was inserted + + // data = t.cache_query(ways=[16001, 16002], deps=True) + // assert data['ways']['16001']['relations'].keys() == ['16001'] + // assert data['ways']['16002'] == None + + // data = t.cache_query(relations=[16001], full=True) + // assert sorted(data['relations']['16001']['ways'].keys()) == ['16001', '16011'] + + assertRecords(t, []checkElem{ + {"osm_landusages", 16001, "", nil}, + {"osm_roads", 16002, "", nil}, + }) + assertArea(t, checkElem{"osm_landusages", -16001, "park", nil}, 12779350582) +} + +func TestNodeWayRefAfterDelete2(t *testing.T) { + // Node does not referece deleted way + + // data = t.cache_query(nodes=[20001, 20002], deps=True) + // assert 'ways' not in data['nodes']['20001'] + // assert data['nodes']['20002'] == None + assertRecords(t, []checkElem{ + {"osm_roads", 20001, "", nil}, + {"osm_barrierpoints", 20001, "block", nil}, + }) +} + +func TestWayRelRefAfterDelete2(t *testing.T) { + // Way does not referece deleted relation + + // data = t.cache_query(ways=[21001], deps=True) + // assert 'relations' not in data['ways']['21001'] + assertRecords(t, []checkElem{ + {"osm_roads", 21001, "residential", nil}, + {"osm_landusages", 21001, "", nil}, + {"osm_landusages", -21001, "", nil}, + }) +} + +func TestResidentialToSecondary2(t *testing.T) { + // New secondary (from residential) is now in roads_gen0/1. + + assertRecords(t, []checkElem{ + {"osm_roads", 40001, "secondary", nil}, + {"osm_roads_gen0", 40001, "secondary", nil}, + {"osm_roads_gen1", 40001, "secondary", nil}, + }) +} + +func TestRelationAfterRemove(t *testing.T) { + // Relation is deleted and way is still present. + assertRecords(t, []checkElem{ + {"osm_buildings", 50011, "yes", nil}, + {"osm_landusages", 50021, "", nil}, + {"osm_landusages", -50021, "", nil}, + }) +} + +func TestRelationWithoutTags2(t *testing.T) { + // Relation without tags is removed. + + // t.cache_query(ways=[50111], deps=True) + // assert t.cache_query(relations=[50121], deps=True)['relations']["50121"] == None + + assertRecords(t, []checkElem{ + {"osm_buildings", 50111, "yes", nil}, + {"osm_buildings", 50121, "", nil}, + {"osm_buildings", -50121, "", nil}, + }) +} + +func TestDuplicateIds2(t *testing.T) { + // Only relation/way with same ID was deleted. + + assertRecords(t, []checkElem{ + {"osm_buildings", 51001, "way", nil}, + {"osm_buildings", -51001, "", nil}, + {"osm_buildings", 51011, "", nil}, + {"osm_buildings", -51011, "mp", nil}, + }) +} + +func TestUpdatedWay2(t *testing.T) { + // All nodes of straightened way are updated. + + // new length 0.1 degree + assertLength(t, checkElem{"osm_roads", 60000, "park", nil}, 20037508.342789244/180.0/10.0) +} + +func TestUpdateNodeToCoord2(t *testing.T) { + // Node is becomes coord after tags are removed. + + // coords = t.cache_query(nodes=(70001, 70002)) + + // assert "tags" not in coords['nodes']["70001"] + // assert coords['nodes']["70002"]["tags"] == {"amenity": "police"} + + // assert not t.query_row(t.db_conf, 'osm_amenities', 70001) + // assert t.query_row(t.db_conf, 'osm_amenities', 70002) + assertRecords(t, []checkElem{ + {"osm_amenities", 70001, "", nil}, + {"osm_amenities", 70002, "police", nil}, + }) +} + +func TestNoDuplicateInsert(t *testing.T) { + // Relation is not inserted again if a nother relation with the same way was modified + // Checks #65 + + assertRecords(t, []checkElem{ + {"osm_landusages", -201191, "park", nil}, + {"osm_landusages", -201192, "forest", nil}, + {"osm_roads", 201151, "residential", nil}, + }) +} + +func TestUnsupportedRelation(t *testing.T) { + // Unsupported relation type is not inserted with update + + assertRecords(t, []checkElem{ + {"osm_landusages", -201291, "", nil}, + {"osm_landusages", 201251, "park", nil}, + }) +} diff --git a/test/helper_test.go b/test/helper_test.go index c8ffe7c..fc5b55f 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -9,6 +9,7 @@ import ( "github.com/lib/pq/hstore" + "github.com/omniscale/imposm3/diff" "github.com/omniscale/imposm3/geom/geos" "github.com/omniscale/imposm3/config" @@ -73,6 +74,19 @@ func (s *importTestSuite) deployOsm(t *testing.T) { import_.Import() } +func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) { + args := []string{ + "-connection", s.config.connection, + "-cachedir", s.config.cacheDir, + "-limitto", "clipping.geojson", + "-dbschema-production", dbschemaProduction, + "-mapping", s.config.mappingFileName, + diffFile, + } + config.ParseDiffImport(args) + diff.Diff() +} + func (s *importTestSuite) dropSchemas() { var err error _, err = s.db.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, dbschemaImport)) From 3f083fb75350fac5299273f342905ecf048d3ed4 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 23 Nov 2015 09:05:19 +0100 Subject: [PATCH 16/36] add single table tests --- test/completedb_test.go | 155 ++++++++++++++++++-------------- test/helper_test.go | 180 ++++++++++++++++++++++++++++++++++++- test/single_table_test.go | 184 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 448 insertions(+), 71 deletions(-) create mode 100644 test/single_table_test.go diff --git a/test/completedb_test.go b/test/completedb_test.go index fbc0bd7..ee7b31f 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -7,7 +7,6 @@ import ( "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/proj" - "math" "testing" "github.com/omniscale/imposm3/geom/geos" @@ -53,74 +52,6 @@ func TestDeploy(t *testing.T) { } } -type checkElem struct { - table string - id int64 - osmType string - tags map[string]string -} - -func assertRecordsMissing(t *testing.T, elems []checkElem) { - for _, e := range elems { - if ts.queryExists(t, e.table, e.id) { - t.Errorf("found %d in %d", e.id, e.table) - } - } -} - -func assertRecords(t *testing.T, elems []checkElem) { - for _, e := range elems { - keys := make([]string, 0, len(e.tags)) - for k, _ := range e.tags { - keys = append(keys, k) - } - r := ts.query(t, e.table, e.id, keys) - if e.osmType == "" { - if r.missing { - continue - } - t.Errorf("got unexpected record %d", r.id) - } - if r.osmType != e.osmType { - t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id) - } - for k, v := range e.tags { - if r.tags[k] != v { - t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) - } - } - } -} - -func assertValid(t *testing.T, e checkElem) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } -} - -func assertArea(t *testing.T, e checkElem, expect float64) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } - actual := geom.Area() - if math.Abs(expect-actual) > 1 { - t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) - } -} - -func assertLength(t *testing.T, e checkElem, expect float64) { - geom := ts.queryGeom(t, e.table, e.id) - if !ts.g.IsValid(geom) { - t.Fatalf("geometry of %d is invalid", e.id) - } - actual := geom.Length() - if math.Abs(expect-actual) > 1 { - t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) - } -} - func TestLandusageToWaterarea1(t *testing.T) { // Parks inserted into landusages // t.assert_cached_way(11001) @@ -653,3 +584,89 @@ func TestUnsupportedRelation(t *testing.T) { {"osm_landusages", 201251, "park", nil}, }) } + +// ####################################################################### + +func TestDeployRevert(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.importOsm(t) + + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.deployOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if !ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup) + } + + ts.revertDeployOsm(t) + + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } +} + +func TestRemoveBackup(t *testing.T) { + if !ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } + + ts.deployOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if !ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads does exists in schema %s", dbschemaBackup) + } + + ts.removeBackupOsm(t) + + if ts.tableExists(t, dbschemaImport, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaImport) + } + if !ts.tableExists(t, dbschemaProduction, "osm_roads") { + t.Fatalf("table osm_roads does not exists in schema %s", dbschemaProduction) + } + if ts.tableExists(t, dbschemaBackup, "osm_roads") { + t.Fatalf("table osm_roads exists in schema %s", dbschemaBackup) + } +} diff --git a/test/helper_test.go b/test/helper_test.go index fc5b55f..e526063 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "log" + "math" "strings" "testing" @@ -49,7 +50,9 @@ func (s *importTestSuite) importOsm(t *testing.T) { // "-optimize", "-mapping", s.config.mappingFileName, "-quiet", + "-revertdeploy=false", "-deployproduction=false", + "-removebackup=false", } config.ParseImport(importArgs) @@ -61,6 +64,9 @@ func (s *importTestSuite) deployOsm(t *testing.T) { "-read=", // overwrite previous options "-write=false", "-optimize=false", + "-revertdeploy=false", + "-deployproduction", + "-removebackup=false", "-connection", s.config.connection, "-dbschema-import", dbschemaImport, "-dbschema-production", dbschemaProduction, @@ -74,6 +80,49 @@ func (s *importTestSuite) deployOsm(t *testing.T) { import_.Import() } +func (s *importTestSuite) revertDeployOsm(t *testing.T) { + importArgs := []string{ + "-read=", // overwrite previous options + "-write=false", + "-optimize=false", + "-revertdeploy", + "-deployproduction=false", + "-removebackup=false", + "-connection", s.config.connection, + "-dbschema-import", dbschemaImport, + "-dbschema-production", dbschemaProduction, + "-dbschema-backup", dbschemaBackup, + "-revertdeploy", + "-deployproduction=false", + "-removebackup=false", + "-mapping", s.config.mappingFileName, + "-quiet", + } + + config.ParseImport(importArgs) + import_.Import() +} + +func (s *importTestSuite) removeBackupOsm(t *testing.T) { + importArgs := []string{ + "-read=", // overwrite previous options + "-write=false", + "-optimize=false", + "-revertdeploy=false", + "-deployproduction=false", + "-removebackup", + "-connection", s.config.connection, + "-dbschema-import", dbschemaImport, + "-dbschema-production", dbschemaProduction, + "-dbschema-backup", dbschemaBackup, + "-mapping", s.config.mappingFileName, + "-quiet", + } + + config.ParseImport(importArgs) + import_.Import() +} + func (s *importTestSuite) updateOsm(t *testing.T, diffFile string) { args := []string{ "-connection", s.config.connection, @@ -165,8 +214,31 @@ func (s *importTestSuite) query(t *testing.T, table string, id int64, keys []str return r } +func (s *importTestSuite) queryTags(t *testing.T, table string, id int64) record { + stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) + row := s.db.QueryRow(stmt, id) + r := record{} + h := hstore.Hstore{} + if err := row.Scan(&r.id, &r.wkt, &h); err != nil { + if err == sql.ErrNoRows { + r.missing = true + } else { + t.Fatal(err) + } + } + if len(h.Map) > 0 { + r.tags = make(map[string]string) + } + for k, v := range h.Map { + if v.Valid { + r.tags[k] = v.String + } + } + return r +} + func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []record { - rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name`, dbschemaProduction, table), id) + rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, name, type, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1 ORDER BY type, name, ST_GeometryType(geometry)`, dbschemaProduction, table), id) if err != nil { t.Fatal(err) } @@ -182,7 +254,16 @@ func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []reco } func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos.Geom { - r := s.query(t, table, id, nil) + stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) + row := s.db.QueryRow(stmt, id) + r := record{} + if err := row.Scan(&r.id, &r.wkt); err != nil { + if err == sql.ErrNoRows { + r.missing = true + } else { + t.Fatal(err) + } + } g := geos.NewGeos() defer g.Finish() geom := g.FromWkt(r.wkt) @@ -191,3 +272,98 @@ func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos. } return geom } + +type checkElem struct { + table string + id int64 + osmType string + tags map[string]string +} + +func assertRecordsMissing(t *testing.T, elems []checkElem) { + for _, e := range elems { + if ts.queryExists(t, e.table, e.id) { + t.Errorf("found %d in %d", e.id, e.table) + } + } +} + +func assertRecords(t *testing.T, elems []checkElem) { + for _, e := range elems { + keys := make([]string, 0, len(e.tags)) + for k, _ := range e.tags { + keys = append(keys, k) + } + r := ts.query(t, e.table, e.id, keys) + if e.osmType == "" { + if r.missing { + continue + } + t.Errorf("got unexpected record %d", r.id) + } + if r.osmType != e.osmType { + t.Errorf("got unexpected type %s != %s for %d", r.osmType, e.osmType, e.id) + } + for k, v := range e.tags { + if r.tags[k] != v { + t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) + } + } + } +} + +func assertHstore(t *testing.T, elems []checkElem) { + for _, e := range elems { + r := ts.queryTags(t, e.table, e.id) + if e.osmType == "" { + if r.missing { + continue + } + t.Errorf("got unexpected record %d", r.id) + } + if len(e.tags) != len(r.tags) { + t.Errorf("tags for %d differ %v != %v", e.id, r.tags, e.tags) + } + for k, v := range e.tags { + if r.tags[k] != v { + t.Errorf("%s does not match for %d %s != %s", k, e.id, r.tags[k], v) + } + } + } +} + +func assertValid(t *testing.T, e checkElem) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } +} + +func assertArea(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Area() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func assertLength(t *testing.T, e checkElem, expect float64) { + geom := ts.queryGeom(t, e.table, e.id) + if !ts.g.IsValid(geom) { + t.Fatalf("geometry of %d is invalid", e.id) + } + actual := geom.Length() + if math.Abs(expect-actual) > 1 { + t.Errorf("unexpected size of %d %f!=%f", e.id, actual, expect) + } +} + +func assertGeomType(t *testing.T, e checkElem, expect string) { + actual := ts.g.Type(ts.queryGeom(t, e.table, e.id)) + if actual != expect { + t.Errorf("expected %s geometry for %d, got %s", expect, e.id, actual) + } +} diff --git a/test/single_table_test.go b/test/single_table_test.go new file mode 100644 index 0000000..4d6554f --- /dev/null +++ b/test/single_table_test.go @@ -0,0 +1,184 @@ +package test + +import ( + "database/sql" + + "testing" + + "github.com/omniscale/imposm3/geom/geos" +) + +const RelOffset = -1e17 + +func TestSingleTable_Prepare(t *testing.T) { + ts.dir = "/tmp/imposm3test" + ts.config = importConfig{ + connection: "postgis://", + cacheDir: ts.dir, + osmFileName: "build/single_table.pbf", + mappingFileName: "single_table_mapping.json", + } + ts.g = geos.NewGeos() + + var err error + ts.db, err = sql.Open("postgres", "sslmode=disable") + if err != nil { + t.Fatal(err) + } + ts.dropSchemas() +} + +func TestSingleTable_Import(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_all") != false { + t.Fatalf("table osm_all exists in schema %s", dbschemaImport) + } + ts.importOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_all") != true { + t.Fatalf("table osm_all does not exists in schema %s", dbschemaImport) + } +} + +func TestSingleTable_Deploy(t *testing.T) { + ts.deployOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_all") != false { + t.Fatalf("table osm_all exists in schema %s", dbschemaImport) + } + if ts.tableExists(t, dbschemaProduction, "osm_all") != true { + t.Fatalf("table osm_all does not exists in schema %s", dbschemaProduction) + } +} + +func TestSingleTable_NonMappedNodeIsMissing(t *testing.T) { + // Node without mapped tags is missing. + // t.assert_cached_node(10001, (10, 42)) + + assertHstore(t, []checkElem{ + {"osm_all", 10001, "", nil}, + }) +} + +func TestSingleTable_MappedNode(t *testing.T) { + // Node is stored with all tags. + // t.assert_cached_node(10002, (11, 42)) + + assertHstore(t, []checkElem{ + {"osm_all", 10002, "*", map[string]string{"random": "tag", "but": "mapped", "poi": "unicorn"}}, + }) +} + +func TestSingleTable_NonMappedWayIsMissing(t *testing.T) { + // Way without mapped tags is missing. + // t.assert_cached_way(20101) + // t.assert_cached_way(20102) + // t.assert_cached_way(20103) + assertHstore(t, []checkElem{ + {"osm_all", 20101, "", nil}, + {"osm_all", 20102, "", nil}, + {"osm_all", 20103, "", nil}, + }) +} + +func TestSingleTable_MappedWay(t *testing.T) { + // Way is stored with all tags. + // t.assert_cached_way(20201) + assertHstore(t, []checkElem{ + {"osm_all", -20201, "*", map[string]string{"random": "tag", "highway": "yes"}}, + }) +} + +func TestSingleTable_NonMappedClosedWayIsMissing(t *testing.T) { + // Closed way without mapped tags is missing. + // t.assert_cached_way(20301) + assertHstore(t, []checkElem{ + {"osm_all", -20301, "", nil}, + }) +} + +func TestSingleTable_MappedClosedWay(t *testing.T) { + // Closed way is stored with all tags. + // t.assert_cached_way(20401) + assertHstore(t, []checkElem{ + {"osm_all", -20401, "*", map[string]string{"random": "tag", "building": "yes"}}, + }) +} + +func TestSingleTable_MappedClosedWayAreaYes(t *testing.T) { + // Closed way with area=yes is not stored as linestring. + // t.assert_cached_way(20501) + assertHstore(t, []checkElem{ + {"osm_all", -20501, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", -20501, "*", nil}, "Polygon") +} + +func TestSingleTable_MappedClosedWayAreaNo(t *testing.T) { + // Closed way with area=no is not stored as polygon. + // t.assert_cached_way(20502) + assertHstore(t, []checkElem{ + {"osm_all", -20502, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "no"}}, + }) + assertGeomType(t, checkElem{"osm_all", -20502, "*", nil}, "LineString") +} + +func TestSingleTable_MappedClosedWayWithoutArea(t *testing.T) { + // Closed way without area is stored as mapped (linestring and polygon). + // t.assert_cached_way(20601) + // elems = t.query_row(t.db_conf, 'osm_all', -20601) + // assert len(elems) == 2 + // elems.sort(key=lambda x: x['geometry'].type) + + // assert elems[0]['geometry'].type == 'LineString', elems[0]['geometry'].type + // assert elems[0]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} + // assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type + // assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} +} + +func TestSingleTable_DuplicateIds1(t *testing.T) { + // Points/lines/polygons with same ID are inserted. + // node = t.query_row(t.db_conf, 'osm_all', 31101) + // assert node['geometry'].type == 'Point', node['geometry'].type + // assert node['tags'] == {'amenity': 'cafe'} + // assert node['geometry'].distance(t.merc_point(80, 47)) < 1 + + // ways = t.query_row(t.db_conf, 'osm_all', -31101) + // ways.sort(key=lambda x: x['geometry'].type) + // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type + // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type + // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + + assertHstore(t, []checkElem{ + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") +} + +// ####################################################################### + +func TestSingleTable_Update(t *testing.T) { + ts.updateOsm(t, "./build/single_table.osc.gz") +} + +// ####################################################################### + +func TestSingleTable_DuplicateIds2(t *testing.T) { + // Node moved and ways/rels with same ID are still present. + + // node = t.query_row(t.db_conf, 'osm_all', 31101) + // assert node['geometry'].type == 'Point', node['geometry'].type + // assert node['tags'] == {'amenity': 'cafe'} + // assert node['geometry'].distance(t.merc_point(81, 47)) < 1 + + // ways = t.query_row(t.db_conf, 'osm_all', -31101) + // ways.sort(key=lambda x: x['geometry'].type) + + // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type + // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type + // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + + assertHstore(t, []checkElem{ + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + }) + assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") +} From af4aff7d6bfebae413024769a8918a2fa037fbdc Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 27 Nov 2015 09:03:14 +0100 Subject: [PATCH 17/36] document prefix connection option --- docs/tutorial.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 2a54101..9226ba7 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -86,6 +86,11 @@ You can combine reading and writing:: imposm3 import -mapping mapping.yml -read hamburg.osm.pbf -write -connection postgis://osm:osm@localhost/osm +All tables are prefixed with ``osm_``, e.g. ``roads`` will create the table ``osm_roads``. You can change the prefix by appending ``?prefix=myprefix`` to the connection URL. Use ``NONE`` to disable prefixing:: + + imposm3 import -mapping mapping.yml -write -connection postgis://osm:osm@localhost/osm?prefix=NONE + + Limit to ~~~~~~~~ From 6722369ab101ca1ee5f4834bf5960cb95a554792 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 21 Dec 2015 10:41:27 +0100 Subject: [PATCH 18/36] add cache checks to go system tests --- test/completedb_test.go | 300 ++++++++++++++++++++++---------------- test/helper_test.go | 77 ++++++++++ test/single_table_test.go | 89 +++++------ 3 files changed, 300 insertions(+), 166 deletions(-) diff --git a/test/completedb_test.go b/test/completedb_test.go index ee7b31f..e28aded 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -2,6 +2,9 @@ package test import ( "database/sql" + "fmt" + + "github.com/omniscale/imposm3/cache" "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" @@ -54,22 +57,24 @@ func TestDeploy(t *testing.T) { func TestLandusageToWaterarea1(t *testing.T) { // Parks inserted into landusages - // t.assert_cached_way(11001) - // t.assert_cached_way(12001) - // t.assert_cached_way(13001) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 11001) + assertCachedWay(t, cache, 12001) + assertCachedWay(t, cache, 13001) assertRecords(t, []checkElem{ - {"osm_waterareas", 11001, "", nil}, - {"osm_waterareas", -12001, "", nil}, - {"osm_waterareas", -13001, "", nil}, + {"osm_waterareas", 11001, Missing, nil}, + {"osm_waterareas", -12001, Missing, nil}, + {"osm_waterareas", -13001, Missing, nil}, - {"osm_waterareas_gen0", 11001, "", nil}, - {"osm_waterareas_gen0", -12001, "", nil}, - {"osm_waterareas_gen0", -13001, "", nil}, + {"osm_waterareas_gen0", 11001, Missing, nil}, + {"osm_waterareas_gen0", -12001, Missing, nil}, + {"osm_waterareas_gen0", -13001, Missing, nil}, - {"osm_waterareas_gen1", 11001, "", nil}, - {"osm_waterareas_gen1", -12001, "", nil}, - {"osm_waterareas_gen1", -13001, "", nil}, + {"osm_waterareas_gen1", 11001, Missing, nil}, + {"osm_waterareas_gen1", -12001, Missing, nil}, + {"osm_waterareas_gen1", -13001, Missing, nil}, {"osm_landusages", 11001, "park", nil}, {"osm_landusages", -12001, "park", nil}, @@ -87,12 +92,14 @@ func TestLandusageToWaterarea1(t *testing.T) { func TestChangedHoleTags1(t *testing.T) { // Multipolygon relation with untagged hole - // t.assert_cached_way(14001) - // t.assert_cached_way(14011) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 14001) + assertCachedWay(t, cache, 14011) assertRecords(t, []checkElem{ - {"osm_waterareas", 14011, "", nil}, - {"osm_waterareas", -14011, "", nil}, + {"osm_waterareas", 14011, Missing, nil}, + {"osm_waterareas", -14011, Missing, nil}, {"osm_landusages", -14001, "park", nil}, }) } @@ -100,7 +107,7 @@ func TestChangedHoleTags1(t *testing.T) { func TestSplitOuterMultipolygonWay1(t *testing.T) { // Single outer way of multipolygon was inserted. assertRecords(t, []checkElem{ - {"osm_roads", 15002, "", nil}, + {"osm_roads", 15002, Missing, nil}, {"osm_landusages", -15001, "park", nil}, }) assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452) @@ -118,25 +125,22 @@ func TestMergeOuterMultipolygonWay1(t *testing.T) { func TestBrokenMultipolygonWays(t *testing.T) { // MultiPolygons with broken outer ways are handled. // outer way does not merge (17002 has one node) - assertRecords(t, []checkElem{ - {"osm_landusages", -17001, "", nil}, + {"osm_landusages", -17001, Missing, nil}, {"osm_roads", 17001, "residential", nil}, - {"osm_roads", 17002, "", nil}, + {"osm_roads", 17002, Missing, nil}, }) // outer way does not merge (17102 has no nodes) assertRecords(t, []checkElem{ - {"osm_landusages", -17101, "", nil}, + {"osm_landusages", -17101, Missing, nil}, {"osm_roads", 17101, "residential", nil}, - {"osm_roads", 17102, "", nil}, + {"osm_roads", 17102, Missing, nil}, }) } func TestNodeWayInsertedTwice(t *testing.T) { // Way with multiple mappings is inserted twice in same table - // rows = t.query_row(t.db_conf, 'osm_roads', 18001) - // rows.sort(key=lambda x: x['type']) rows := ts.queryRows(t, "osm_roads", 18001) if len(rows) != 2 || rows[0].osmType != "residential" || rows[1].osmType != "tram" { t.Errorf("unexpected roads: %v", rows) @@ -145,17 +149,15 @@ func TestNodeWayInsertedTwice(t *testing.T) { func TestOuterWayNotInserted(t *testing.T) { // Outer way with different tag is not inserted twice into same table - assertRecords(t, []checkElem{ {"osm_landusages", -19001, "farmland", nil}, {"osm_landusages", 19002, "farmyard", nil}, - {"osm_landusages", 19001, "", nil}, + {"osm_landusages", 19001, Missing, nil}, }) } func TestOuterWayInserted(t *testing.T) { // Outer way with different tag is inserted twice into different table - assertRecords(t, []checkElem{ {"osm_landusages", 19101, "farm", nil}, {"osm_landusages", 19102, "farmyard", nil}, @@ -164,10 +166,17 @@ func TestOuterWayInserted(t *testing.T) { } func TestNodeWayRefAfterDelete1(t *testing.T) { - // Nodes refereces way - // data = t.cache_query(nodes=[20001, 20002], deps=True) - // assert '20001' in data['nodes']['20001']['ways'] - // assert '20001' in data['nodes']['20002']['ways'] + // Nodes references way + + cache := ts.diffCache(t) + defer cache.Close() + if ids := cache.Coords.Get(20001); len(ids) != 1 || ids[0] != 20001 { + t.Error("node does not references way") + } + if ids := cache.Coords.Get(20002); len(ids) != 1 || ids[0] != 20001 { + t.Error("node does not references way") + } + assertRecords(t, []checkElem{ {"osm_roads", 20001, "residential", nil}, {"osm_barrierpoints", 20001, "block", nil}, @@ -176,8 +185,13 @@ func TestNodeWayRefAfterDelete1(t *testing.T) { func TestWayRelRefAfterDelete1(t *testing.T) { // Ways references relation - // data = t.cache_query(ways=[21001], deps=True) - // assert data['ways']['21001']['relations'].keys() == ['21001'] + + cache := ts.diffCache(t) + defer cache.Close() + if ids := cache.Ways.Get(21001); len(ids) != 1 || ids[0] != 21001 { + t.Error("way does not references relation") + } + assertRecords(t, []checkElem{ {"osm_roads", 21001, "residential", nil}, {"osm_landusages", -21001, "park", nil}, @@ -186,22 +200,20 @@ func TestWayRelRefAfterDelete1(t *testing.T) { func TestRelationWayNotInserted(t *testing.T) { // Part of relation was inserted only once. - assertRecords(t, []checkElem{ {"osm_landusages", -9001, "park", map[string]string{"name": "rel 9001"}}, - {"osm_landusages", 9009, "", nil}, + {"osm_landusages", 9009, Missing, nil}, {"osm_landusages", -9101, "park", map[string]string{"name": "rel 9101"}}, - {"osm_landusages", 9109, "", nil}, + {"osm_landusages", 9109, Missing, nil}, {"osm_landusages", 9110, "scrub", nil}, }) } func TestRelationWaysInserted(t *testing.T) { // Outer ways of multipolygon are inserted. - assertRecords(t, []checkElem{ {"osm_landusages", -9201, "park", map[string]string{"name": "9209"}}, - {"osm_landusages", 9201, "", nil}, + {"osm_landusages", 9201, Missing, nil}, // outer ways of multipolygon stand for their own {"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}}, {"osm_roads", 9210, "residential", map[string]string{"name": "9210"}}, @@ -217,7 +229,6 @@ func TestRelationWaysInserted(t *testing.T) { func TestRelationWayInserted(t *testing.T) { // Part of relation was inserted twice. - assertRecords(t, []checkElem{ {"osm_landusages", -8001, "park", map[string]string{"name": "rel 8001"}}, {"osm_roads", 8009, "residential", nil}, @@ -226,17 +237,15 @@ func TestRelationWayInserted(t *testing.T) { func TestSingleNodeWaysNotInserted(t *testing.T) { // Ways with single/duplicate nodes are not inserted. - assertRecords(t, []checkElem{ - {"osm_landusages", 30001, "", nil}, - {"osm_landusages", 30002, "", nil}, - {"osm_landusages", 30003, "", nil}, + {"osm_landusages", 30001, Missing, nil}, + {"osm_landusages", 30002, Missing, nil}, + {"osm_landusages", 30003, Missing, nil}, }) } func TestPolygonWithDuplicateNodesIsValid(t *testing.T) { // Polygon with duplicate nodes is valid. - assertValid(t, checkElem{"osm_landusages", 30005, "park", nil}) } @@ -244,8 +253,8 @@ func TestIncompletePolygons(t *testing.T) { // Non-closed/incomplete polygons are not inserted. assertRecords(t, []checkElem{ - {"osm_landusages", 30004, "", nil}, - {"osm_landusages", 30006, "", nil}, + {"osm_landusages", 30004, Missing, nil}, + {"osm_landusages", 30006, Missing, nil}, }) } @@ -254,8 +263,8 @@ func TestResidentialToSecondary(t *testing.T) { assertRecords(t, []checkElem{ {"osm_roads", 40001, "residential", nil}, - {"osm_roads_gen0", 40001, "", nil}, - {"osm_roads_gen1", 40002, "", nil}, + {"osm_roads_gen0", 40001, Missing, nil}, + {"osm_roads_gen1", 40002, Missing, nil}, }) } @@ -272,7 +281,7 @@ func TestRelationWithoutTags(t *testing.T) { // Relation without tags is inserted. assertRecords(t, []checkElem{ - {"osm_buildings", 50111, "", nil}, + {"osm_buildings", 50111, Missing, nil}, {"osm_buildings", -50121, "yes", nil}, }) } @@ -291,10 +300,10 @@ func TestDuplicateIds(t *testing.T) { func TestGeneralizedBananaPolygonIsValid(t *testing.T) { // Generalized polygons are valid. - assertValid(t, checkElem{"osm_landusages", 7101, "", nil}) + assertValid(t, checkElem{"osm_landusages", 7101, Missing, nil}) // simplified geometies are valid too - assertValid(t, checkElem{"osm_landusages_gen0", 7101, "", nil}) - assertValid(t, checkElem{"osm_landusages_gen1", 7101, "", nil}) + assertValid(t, checkElem{"osm_landusages_gen0", 7101, Missing, nil}) + assertValid(t, checkElem{"osm_landusages_gen1", 7101, Missing, nil}) } func TestGeneralizedLinestringIsValid(t *testing.T) { @@ -312,28 +321,25 @@ func TestGeneralizedLinestringIsValid(t *testing.T) { func TestRingWithGap(t *testing.T) { // Multipolygon and way with gap (overlapping but different endpoints) gets closed - assertValid(t, checkElem{"osm_landusages", -7301, "", nil}) - assertValid(t, checkElem{"osm_landusages", 7311, "", nil}) + assertValid(t, checkElem{"osm_landusages", -7301, Missing, nil}) + assertValid(t, checkElem{"osm_landusages", 7311, Missing, nil}) } func TestMultipolygonWithOpenRing(t *testing.T) { // Multipolygon is inserted even if there is an open ring/member - assertValid(t, checkElem{"osm_landusages", -7401, "", nil}) + assertValid(t, checkElem{"osm_landusages", -7401, Missing, nil}) } func TestUpdatedNodes1(t *testing.T) { // Zig-Zag line is inserted. - assertLength(t, checkElem{"osm_roads", 60000, "", nil}, 14035.61150207768) + assertLength(t, checkElem{"osm_roads", 60000, Missing, nil}, 14035.61150207768) } func TestUpdateNodeToCoord1(t *testing.T) { // Node is inserted with tag. - // coords = t.cache_query(nodes=(70001, 70002)) - // assert coords['nodes']["70001"]["tags"] == {"amenity": "police"} - // assert "tags" not in coords['nodes']["70002"] assertRecords(t, []checkElem{ {"osm_amenities", 70001, "police", nil}, - {"osm_amenities", 70002, "", nil}, + {"osm_amenities", 70002, Missing, nil}, }) } @@ -353,17 +359,33 @@ func TestUpdate(t *testing.T) { func TestNoDuplicates(t *testing.T) { // Relations/ways are only inserted once Checks #66 - // highways = t.query_duplicates(t.db_conf, 'osm_roads') - // # one duplicate for test_node_way_inserted_twice is expected - // assert highways == [[18001, 2]], highways - // landusages = t.query_duplicates(t.db_conf, 'osm_landusages') - // assert not landusages, landusages + for _, table := range []string{"osm_roads", "osm_landusages"} { + rows, err := ts.db.Query( + fmt.Sprintf(`SELECT osm_id, count(osm_id) FROM "%s"."%s" GROUP BY osm_id HAVING count(osm_id) > 1`, + dbschemaProduction, table)) + if err != nil { + t.Fatal(err) + } + var osmId, count int64 + for rows.Next() { + if err := rows.Scan(&osmId, &count); err != nil { + t.Fatal(err) + } + if table == "osm_roads" && osmId == 18001 { + // # duplicate for TestNodeWayInsertedTwice is expected + if count != 2 { + t.Error("highway not inserted twice", osmId, count) + } + } else { + t.Error("found duplicate way in osm_roads", osmId, count) + } + } + } } func TestUpdatedLandusage(t *testing.T) { // Multipolygon relation was modified - // t.assert_cached_node(1001, (13.5, 47.5)) nd := element.Node{Long: 13.4, Lat: 47.5} proj.NodeToMerc(&nd) point, err := geom.Point(ts.g, nd) @@ -380,22 +402,26 @@ func TestUpdatedLandusage(t *testing.T) { func TestPartialDelete(t *testing.T) { // Deleted relation but nodes are still cached - // t.assert_cached_node(2001) - // t.assert_cached_way(2001) - // t.assert_cached_way(2002) + cache := ts.cache(t) + defer cache.Close() + assertCachedNode(t, cache, 2001) + assertCachedWay(t, cache, 2001) + assertCachedWay(t, cache, 2002) assertRecords(t, []checkElem{ - {"osm_landusages", -2001, "", nil}, - {"osm_landusages", 2001, "", nil}, + {"osm_landusages", -2001, Missing, nil}, + {"osm_landusages", 2001, Missing, nil}, }) } func TestUpdatedNodes(t *testing.T) { // Nodes were added, modified or deleted - // t.assert_missing_node(10000) - // t.assert_cached_node(10001, (10.0, 40.0)) - // t.assert_cached_node(10002, (10.1, 40.0)) + c := ts.cache(t) + defer c.Close() + if _, err := c.Coords.GetCoord(10000); err != cache.NotFound { + t.Fatal("coord not missing") + } assertRecords(t, []checkElem{ {"osm_places", 10001, "village", map[string]string{"name": "Bar"}}, @@ -407,7 +433,6 @@ func TestLandusageToWaterarea2(t *testing.T) { // Parks converted to water moved from landusages to waterareas assertRecords(t, []checkElem{ - {"osm_waterareas", 11001, "water", nil}, {"osm_waterareas", -12001, "water", nil}, {"osm_waterareas", -13001, "water", nil}, @@ -420,25 +445,27 @@ func TestLandusageToWaterarea2(t *testing.T) { {"osm_waterareas_gen1", -12001, "water", nil}, {"osm_waterareas_gen1", -13001, "water", nil}, - {"osm_landusages", 11001, "", nil}, - {"osm_landusages", -12001, "", nil}, - {"osm_landusages", -13001, "", nil}, + {"osm_landusages", 11001, Missing, nil}, + {"osm_landusages", -12001, Missing, nil}, + {"osm_landusages", -13001, Missing, nil}, - {"osm_landusages_gen0", 11001, "", nil}, - {"osm_landusages_gen0", -12001, "", nil}, - {"osm_landusages_gen0", -13001, "", nil}, + {"osm_landusages_gen0", 11001, Missing, nil}, + {"osm_landusages_gen0", -12001, Missing, nil}, + {"osm_landusages_gen0", -13001, Missing, nil}, - {"osm_landusages_gen1", 11001, "", nil}, - {"osm_landusages_gen1", -12001, "", nil}, - {"osm_landusages_gen1", -13001, "", nil}, + {"osm_landusages_gen1", 11001, Missing, nil}, + {"osm_landusages_gen1", -12001, Missing, nil}, + {"osm_landusages_gen1", -13001, Missing, nil}, }) } func TestChangedHoleTags2(t *testing.T) { // Newly tagged hole is inserted - // t.assert_cached_way(14001) - // t.assert_cached_way(14011) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 14001) + assertCachedWay(t, cache, 14011) assertArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779) assertArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182) @@ -447,12 +474,17 @@ func TestChangedHoleTags2(t *testing.T) { func TestSplitOuterMultipolygonWay2(t *testing.T) { // Splitted outer way of multipolygon was inserted - // data = t.cache_query(ways=[15001, 15002], deps=True) - // assert data['ways']['15001']['relations'].keys() == ['15001'] - // assert data['ways']['15002']['relations'].keys() == ['15001'] + diffCache := ts.diffCache(t) + defer diffCache.Close() + if ids := diffCache.Ways.Get(15001); len(ids) != 1 || ids[0] != 15001 { + t.Error("way does not references relation") + } + if ids := diffCache.Ways.Get(15002); len(ids) != 1 || ids[0] != 15001 { + t.Error("way does not references relation") + } assertRecords(t, []checkElem{ - {"osm_landusages", 15001, "", nil}, + {"osm_landusages", 15001, Missing, nil}, {"osm_roads", 15002, "residential", nil}, }) assertArea(t, checkElem{"osm_landusages", -15001, "park", nil}, 9816216452) @@ -461,16 +493,28 @@ func TestSplitOuterMultipolygonWay2(t *testing.T) { func TestMergeOuterMultipolygonWay2(t *testing.T) { // Merged outer way of multipolygon was inserted - // data = t.cache_query(ways=[16001, 16002], deps=True) - // assert data['ways']['16001']['relations'].keys() == ['16001'] - // assert data['ways']['16002'] == None + diffCache := ts.diffCache(t) + defer diffCache.Close() + if ids := diffCache.Ways.Get(16001); len(ids) != 1 || ids[0] != 16001 { + t.Error("way does not references relation") + } + if ids := diffCache.Ways.Get(16002); len(ids) != 0 { + t.Error("way references relation") + } - // data = t.cache_query(relations=[16001], full=True) - // assert sorted(data['relations']['16001']['ways'].keys()) == ['16001', '16011'] + cache := ts.cache(t) + defer cache.Close() + rel, err := cache.Relations.GetRelation(16001) + if err != nil { + t.Fatal(err) + } + if len(rel.Members) != 2 || rel.Members[0].Id != 16001 || rel.Members[1].Id != 16011 { + t.Error("unexpected relation members", rel) + } assertRecords(t, []checkElem{ - {"osm_landusages", 16001, "", nil}, - {"osm_roads", 16002, "", nil}, + {"osm_landusages", 16001, Missing, nil}, + {"osm_roads", 16002, Missing, nil}, }) assertArea(t, checkElem{"osm_landusages", -16001, "park", nil}, 12779350582) } @@ -478,11 +522,20 @@ func TestMergeOuterMultipolygonWay2(t *testing.T) { func TestNodeWayRefAfterDelete2(t *testing.T) { // Node does not referece deleted way - // data = t.cache_query(nodes=[20001, 20002], deps=True) - // assert 'ways' not in data['nodes']['20001'] - // assert data['nodes']['20002'] == None + diffCache := ts.diffCache(t) + defer diffCache.Close() + if ids := diffCache.Coords.Get(20001); len(ids) != 0 { + t.Error("node references way") + } + c := ts.cache(t) + defer c.Close() + _, err := c.Coords.GetCoord(20002) + if err != cache.NotFound { + t.Error("found deleted node") + } + assertRecords(t, []checkElem{ - {"osm_roads", 20001, "", nil}, + {"osm_roads", 20001, Missing, nil}, {"osm_barrierpoints", 20001, "block", nil}, }) } @@ -490,12 +543,16 @@ func TestNodeWayRefAfterDelete2(t *testing.T) { func TestWayRelRefAfterDelete2(t *testing.T) { // Way does not referece deleted relation - // data = t.cache_query(ways=[21001], deps=True) - // assert 'relations' not in data['ways']['21001'] + diffCache := ts.diffCache(t) + defer diffCache.Close() + if ids := diffCache.Ways.Get(21001); len(ids) != 0 { + t.Error("way references relation") + } + assertRecords(t, []checkElem{ {"osm_roads", 21001, "residential", nil}, - {"osm_landusages", 21001, "", nil}, - {"osm_landusages", -21001, "", nil}, + {"osm_landusages", 21001, Missing, nil}, + {"osm_landusages", -21001, Missing, nil}, }) } @@ -513,21 +570,27 @@ func TestRelationAfterRemove(t *testing.T) { // Relation is deleted and way is still present. assertRecords(t, []checkElem{ {"osm_buildings", 50011, "yes", nil}, - {"osm_landusages", 50021, "", nil}, - {"osm_landusages", -50021, "", nil}, + {"osm_landusages", 50021, Missing, nil}, + {"osm_landusages", -50021, Missing, nil}, }) } func TestRelationWithoutTags2(t *testing.T) { // Relation without tags is removed. - // t.cache_query(ways=[50111], deps=True) - // assert t.cache_query(relations=[50121], deps=True)['relations']["50121"] == None + c := ts.cache(t) + defer c.Close() + assertCachedWay(t, c, 50111) + + _, err := c.Ways.GetWay(20002) + if err != cache.NotFound { + t.Error("found deleted node") + } assertRecords(t, []checkElem{ {"osm_buildings", 50111, "yes", nil}, - {"osm_buildings", 50121, "", nil}, - {"osm_buildings", -50121, "", nil}, + {"osm_buildings", 50121, Missing, nil}, + {"osm_buildings", -50121, Missing, nil}, }) } @@ -536,8 +599,8 @@ func TestDuplicateIds2(t *testing.T) { assertRecords(t, []checkElem{ {"osm_buildings", 51001, "way", nil}, - {"osm_buildings", -51001, "", nil}, - {"osm_buildings", 51011, "", nil}, + {"osm_buildings", -51001, Missing, nil}, + {"osm_buildings", 51011, Missing, nil}, {"osm_buildings", -51011, "mp", nil}, }) } @@ -552,15 +615,8 @@ func TestUpdatedWay2(t *testing.T) { func TestUpdateNodeToCoord2(t *testing.T) { // Node is becomes coord after tags are removed. - // coords = t.cache_query(nodes=(70001, 70002)) - - // assert "tags" not in coords['nodes']["70001"] - // assert coords['nodes']["70002"]["tags"] == {"amenity": "police"} - - // assert not t.query_row(t.db_conf, 'osm_amenities', 70001) - // assert t.query_row(t.db_conf, 'osm_amenities', 70002) assertRecords(t, []checkElem{ - {"osm_amenities", 70001, "", nil}, + {"osm_amenities", 70001, Missing, nil}, {"osm_amenities", 70002, "police", nil}, }) } @@ -580,7 +636,7 @@ func TestUnsupportedRelation(t *testing.T) { // Unsupported relation type is not inserted with update assertRecords(t, []checkElem{ - {"osm_landusages", -201291, "", nil}, + {"osm_landusages", -201291, Missing, nil}, {"osm_landusages", 201251, "park", nil}, }) } diff --git a/test/helper_test.go b/test/helper_test.go index e526063..d3a43f9 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -8,6 +8,10 @@ import ( "strings" "testing" + "github.com/omniscale/imposm3/element" + + "github.com/omniscale/imposm3/cache" + "github.com/lib/pq/hstore" "github.com/omniscale/imposm3/diff" @@ -38,6 +42,8 @@ type importTestSuite struct { g *geos.Geos } +const Missing = "" + func (s *importTestSuite) importOsm(t *testing.T) { importArgs := []string{ "-connection", s.config.connection, @@ -103,6 +109,22 @@ func (s *importTestSuite) revertDeployOsm(t *testing.T) { import_.Import() } +func (s *importTestSuite) cache(t *testing.T) *cache.OSMCache { + c := cache.NewOSMCache(s.config.cacheDir) + if err := c.Open(); err != nil { + t.Fatal(err) + } + return c +} + +func (s *importTestSuite) diffCache(t *testing.T) *cache.DiffCache { + c := cache.NewDiffCache(s.config.cacheDir) + if err := c.Open(); err != nil { + t.Fatal(err) + } + return c +} + func (s *importTestSuite) removeBackupOsm(t *testing.T) { importArgs := []string{ "-read=", // overwrite previous options @@ -253,6 +275,31 @@ func (s *importTestSuite) queryRows(t *testing.T, table string, id int64) []reco return rs } +func (s *importTestSuite) queryRowsTags(t *testing.T, table string, id int64) []record { + rows, err := s.db.Query(fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1 ORDER BY ST_GeometryType(geometry)`, dbschemaProduction, table), id) + if err != nil { + t.Fatal(err) + } + rs := []record{} + for rows.Next() { + var r record + h := hstore.Hstore{} + if err := rows.Scan(&r.id, &r.wkt, &h); err != nil { + t.Fatal(err) + } + if len(h.Map) > 0 { + r.tags = make(map[string]string) + } + for k, v := range h.Map { + if v.Valid { + r.tags[k] = v.String + } + } + rs = append(rs, r) + } + return rs +} + func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos.Geom { stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry) FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) row := s.db.QueryRow(stmt, id) @@ -367,3 +414,33 @@ func assertGeomType(t *testing.T, e checkElem, expect string) { t.Errorf("expected %s geometry for %d, got %s", expect, e.id, actual) } } + +func assertCachedWay(t *testing.T, c *cache.OSMCache, id int64) *element.Way { + way, err := c.Ways.GetWay(id) + if err == cache.NotFound { + t.Errorf("missing way %d", id) + } else if err != nil { + t.Fatal(err) + } + if way.Id != id { + t.Errorf("cached way contains invalid id, %d != %d", way.Id, id) + } + return way +} + +func assertCachedNode(t *testing.T, c *cache.OSMCache, id int64) *element.Node { + node, err := c.Nodes.GetNode(id) + if err == cache.NotFound { + node, err = c.Coords.GetCoord(id) + if err == cache.NotFound { + t.Errorf("missing node %d", id) + return nil + } + } else if err != nil { + t.Fatal(err) + } + if node.Id != id { + t.Errorf("cached node contains invalid id, %d != %d", node.Id, id) + } + return node +} diff --git a/test/single_table_test.go b/test/single_table_test.go index 4d6554f..cba5900 100644 --- a/test/single_table_test.go +++ b/test/single_table_test.go @@ -2,6 +2,7 @@ package test import ( "database/sql" + "strings" "testing" @@ -50,16 +51,20 @@ func TestSingleTable_Deploy(t *testing.T) { func TestSingleTable_NonMappedNodeIsMissing(t *testing.T) { // Node without mapped tags is missing. - // t.assert_cached_node(10001, (10, 42)) + cache := ts.cache(t) + defer cache.Close() + assertCachedNode(t, cache, 10001) assertHstore(t, []checkElem{ - {"osm_all", 10001, "", nil}, + {"osm_all", 10001, Missing, nil}, }) } func TestSingleTable_MappedNode(t *testing.T) { // Node is stored with all tags. - // t.assert_cached_node(10002, (11, 42)) + cache := ts.cache(t) + defer cache.Close() + assertCachedNode(t, cache, 10002) assertHstore(t, []checkElem{ {"osm_all", 10002, "*", map[string]string{"random": "tag", "but": "mapped", "poi": "unicorn"}}, @@ -68,19 +73,25 @@ func TestSingleTable_MappedNode(t *testing.T) { func TestSingleTable_NonMappedWayIsMissing(t *testing.T) { // Way without mapped tags is missing. - // t.assert_cached_way(20101) - // t.assert_cached_way(20102) - // t.assert_cached_way(20103) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 20101) + assertCachedWay(t, cache, 20102) + assertCachedWay(t, cache, 20103) + assertHstore(t, []checkElem{ - {"osm_all", 20101, "", nil}, - {"osm_all", 20102, "", nil}, - {"osm_all", 20103, "", nil}, + {"osm_all", 20101, Missing, nil}, + {"osm_all", 20102, Missing, nil}, + {"osm_all", 20103, Missing, nil}, }) } func TestSingleTable_MappedWay(t *testing.T) { // Way is stored with all tags. - // t.assert_cached_way(20201) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 20201) + assertHstore(t, []checkElem{ {"osm_all", -20201, "*", map[string]string{"random": "tag", "highway": "yes"}}, }) @@ -88,15 +99,17 @@ func TestSingleTable_MappedWay(t *testing.T) { func TestSingleTable_NonMappedClosedWayIsMissing(t *testing.T) { // Closed way without mapped tags is missing. - // t.assert_cached_way(20301) + cache := ts.cache(t) + defer cache.Close() + assertCachedWay(t, cache, 20301) assertHstore(t, []checkElem{ - {"osm_all", -20301, "", nil}, + {"osm_all", 20301, Missing, nil}, + {"osm_all", -20301, Missing, nil}, }) } func TestSingleTable_MappedClosedWay(t *testing.T) { // Closed way is stored with all tags. - // t.assert_cached_way(20401) assertHstore(t, []checkElem{ {"osm_all", -20401, "*", map[string]string{"random": "tag", "building": "yes"}}, }) @@ -104,7 +117,6 @@ func TestSingleTable_MappedClosedWay(t *testing.T) { func TestSingleTable_MappedClosedWayAreaYes(t *testing.T) { // Closed way with area=yes is not stored as linestring. - // t.assert_cached_way(20501) assertHstore(t, []checkElem{ {"osm_all", -20501, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "yes"}}, }) @@ -113,7 +125,6 @@ func TestSingleTable_MappedClosedWayAreaYes(t *testing.T) { func TestSingleTable_MappedClosedWayAreaNo(t *testing.T) { // Closed way with area=no is not stored as polygon. - // t.assert_cached_way(20502) assertHstore(t, []checkElem{ {"osm_all", -20502, "*", map[string]string{"random": "tag", "landuse": "grass", "highway": "pedestrian", "area": "no"}}, }) @@ -122,30 +133,24 @@ func TestSingleTable_MappedClosedWayAreaNo(t *testing.T) { func TestSingleTable_MappedClosedWayWithoutArea(t *testing.T) { // Closed way without area is stored as mapped (linestring and polygon). - // t.assert_cached_way(20601) - // elems = t.query_row(t.db_conf, 'osm_all', -20601) - // assert len(elems) == 2 - // elems.sort(key=lambda x: x['geometry'].type) - // assert elems[0]['geometry'].type == 'LineString', elems[0]['geometry'].type - // assert elems[0]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} - // assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type - // assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} + rows := ts.queryRowsTags(t, "osm_all", -20601) + if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") { + t.Errorf("unexpected geometries: %v", rows) + } } func TestSingleTable_DuplicateIds1(t *testing.T) { // Points/lines/polygons with same ID are inserted. - // node = t.query_row(t.db_conf, 'osm_all', 31101) - // assert node['geometry'].type == 'Point', node['geometry'].type - // assert node['tags'] == {'amenity': 'cafe'} - // assert node['geometry'].distance(t.merc_point(80, 47)) < 1 - // ways = t.query_row(t.db_conf, 'osm_all', -31101) - // ways.sort(key=lambda x: x['geometry'].type) - // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type - // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type - // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + assertHstore(t, []checkElem{ + {"osm_all", 31101, "*", map[string]string{"amenity": "cafe"}}, + }) + + rows := ts.queryRowsTags(t, "osm_all", -31101) + if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") { + t.Errorf("unexpected geometries: %v", rows) + } assertHstore(t, []checkElem{ {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, @@ -164,18 +169,14 @@ func TestSingleTable_Update(t *testing.T) { func TestSingleTable_DuplicateIds2(t *testing.T) { // Node moved and ways/rels with same ID are still present. - // node = t.query_row(t.db_conf, 'osm_all', 31101) - // assert node['geometry'].type == 'Point', node['geometry'].type - // assert node['tags'] == {'amenity': 'cafe'} - // assert node['geometry'].distance(t.merc_point(81, 47)) < 1 + assertHstore(t, []checkElem{ + {"osm_all", 31101, "*", map[string]string{"amenity": "cafe"}}, + }) - // ways = t.query_row(t.db_conf, 'osm_all', -31101) - // ways.sort(key=lambda x: x['geometry'].type) - - // assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type - // assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - // assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type - // assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} + rows := ts.queryRowsTags(t, "osm_all", -31101) + if len(rows) != 2 || strings.HasPrefix(rows[0].wkt, "LineString") || strings.HasPrefix(rows[1].wkt, "Polygon") { + t.Errorf("unexpected geometries: %v", rows) + } assertHstore(t, []checkElem{ {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, From 7bf29b60f7a814686dce099d43528c98a0685561 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 21 Dec 2015 12:54:36 +0100 Subject: [PATCH 19/36] remove python tests --- test/Makefile | 28 +- test/complete_db_test.py | 537 -------------------------------------- test/helper.py | 265 ------------------- test/single_table_test.py | 184 ------------- 4 files changed, 9 insertions(+), 1005 deletions(-) delete mode 100644 test/complete_db_test.py delete mode 100644 test/helper.py delete mode 100644 test/single_table_test.py diff --git a/test/Makefile b/test/Makefile index 433405f..516e27e 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,23 +1,19 @@ -.PHONY: build all test clean - -IMPOSM_BIN=../imposm3 +.PHONY: all test clean ifdef VERBOSE -NOSEOPTS = -vs +TESTOPTS = -v else -NOSEOPTS = -v +TESTOPTS = endif -all: build test - -build: - cd ..; make build - -$(IMPOSM_BIN): build +all: test clean: rm -rf build +PBF_FILES=$(addprefix build/,$(patsubst %.osm,%.pbf,$(wildcard *.osm))) +OSCGZ_FILES=$(addprefix build/,$(patsubst %.osc,%.osc.gz,$(wildcard *.osc))) + build/%.pbf: %.osm @mkdir -p build osmosis --read-xml $< --sort TypeThenId --write-pbf $@ omitmetadata=true @@ -26,12 +22,6 @@ build/%.osc.gz: %.osc @mkdir -p build gzip --stdout $< > $@ -test: .lasttestrun_complete_db .lasttestrun_single_table +test: $(PBF_FILES) $(OSCGZ_FILES) + (cd .. && godep go test ./test $(TESTOPTS)) -.lasttestrun_complete_db: $(IMPOSM_BIN) complete_db_test.py build/complete_db.osc.gz build/complete_db.pbf - nosetests complete_db_test.py $(NOSEOPTS) - @touch .lasttestrun_complete_db - -.lasttestrun_single_table: $(IMPOSM_BIN) single_table_test.py build/single_table.osc.gz build/single_table.pbf - nosetests single_table_test.py $(NOSEOPTS) - @touch .lasttestrun_single_table \ No newline at end of file diff --git a/test/complete_db_test.py b/test/complete_db_test.py deleted file mode 100644 index ecde4f8..0000000 --- a/test/complete_db_test.py +++ /dev/null @@ -1,537 +0,0 @@ -import unittest - -import helper as t - -mapping_file = 'complete_db_mapping.json' - -def setup(): - t.setup() - -def teardown(): - t.teardown() - -####################################################################### -def test_import(): - """Import succeeds""" - t.drop_schemas() - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - t.imposm3_import(t.db_conf, './build/complete_db.pbf', mapping_file) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - -def test_deploy(): - """Deploy succeeds""" - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - t.imposm3_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - -####################################################################### - -def test_imported_landusage(): - """Multipolygon relation is inserted""" - t.assert_cached_node(1001, (13, 47.5)) - landusage_1001 = t.query_row(t.db_conf, 'osm_landusages', -1001) - # point in polygon - assert landusage_1001['geometry'].intersects(t.merc_point(13.4, 47.5)) - # hole in multipolygon relation - assert not landusage_1001['geometry'].intersects(t.merc_point(14.75, 47.75)) - -def test_missing_nodes(): - """Cache does not contain nodes from previous imports""" - t.assert_missing_node(10001) - t.assert_missing_node(10002) - place_10000 = t.query_row(t.db_conf, 'osm_places', 10000) - assert place_10000['name'] == 'Foo', place_10000 - -def test_name_tags(): - """Road contains multiple names""" - road = t.query_row(t.db_conf, 'osm_roads', 1101) - assert road['name'] == 'name', road - assert road['name:de'] == 'name:de', road - assert road['name_en'] == 'name:en', road - -def test_landusage_to_waterarea_1(): - """Parks inserted into landusages""" - t.assert_cached_way(11001) - t.assert_cached_way(12001) - t.assert_cached_way(13001) - - assert not t.query_row(t.db_conf, 'osm_waterareas', 11001) - assert not t.query_row(t.db_conf, 'osm_waterareas', -12001) - assert not t.query_row(t.db_conf, 'osm_waterareas', -13001) - - assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', 11001) - assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', -12001) - assert not t.query_row(t.db_conf, 'osm_waterareas_gen0', -13001) - - assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', 11001) - assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', -12001) - assert not t.query_row(t.db_conf, 'osm_waterareas_gen1', -13001) - - assert t.query_row(t.db_conf, 'osm_landusages', 11001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages', -12001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages', -13001)['type'] == 'park' - - assert t.query_row(t.db_conf, 'osm_landusages_gen0', 11001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages_gen0', -12001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages_gen0', -13001)['type'] == 'park' - - assert t.query_row(t.db_conf, 'osm_landusages_gen1', 11001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages_gen1', -12001)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages_gen1', -13001)['type'] == 'park' - - -def test_changed_hole_tags_1(): - """Multipolygon relation with untagged hole""" - t.assert_cached_way(14001) - t.assert_cached_way(14011) - - assert not t.query_row(t.db_conf, 'osm_waterareas', 14011) - assert not t.query_row(t.db_conf, 'osm_waterareas', -14011) - assert t.query_row(t.db_conf, 'osm_landusages', -14001)['type'] == 'park' - -def test_split_outer_multipolygon_way_1(): - """Single outer way of multipolygon was inserted.""" - park_15001 = t.query_row(t.db_conf, 'osm_landusages', -15001) - assert park_15001['type'] == 'park' - t.assert_almost_equal(park_15001['geometry'].area, 9816216452, -1) - assert t.query_row(t.db_conf, 'osm_roads', 15002) == None - -def test_merge_outer_multipolygon_way_1(): - """Splitted outer way of multipolygon was inserted.""" - park_16001 = t.query_row(t.db_conf, 'osm_landusages', -16001) - assert park_16001['type'] == 'park' - t.assert_almost_equal(park_16001['geometry'].area, 12779350582, -1) - assert t.query_row(t.db_conf, 'osm_roads', 16002)['type'] == 'residential' - -def test_broken_multipolygon_ways(): - """MultiPolygons with broken outer ways are handled.""" - # outer way does not merge (17002 has one node) - assert t.query_row(t.db_conf, 'osm_landusages', -17001) == None - assert t.query_row(t.db_conf, 'osm_roads', 17001)['type'] == 'residential' - assert t.query_row(t.db_conf, 'osm_roads', 17002) == None - - # outer way does not merge (17102 has no nodes) - assert t.query_row(t.db_conf, 'osm_landusages', -17101) == None - assert t.query_row(t.db_conf, 'osm_roads', 17101)['type'] == 'residential' - assert t.query_row(t.db_conf, 'osm_roads', 17102) == None - -def test_node_way_inserted_twice(): - """Way with multiple mappings is inserted twice in same table""" - rows = t.query_row(t.db_conf, 'osm_roads', 18001) - rows.sort(key=lambda x: x['type']) - - assert rows[0]['type'] == 'residential' - assert rows[1]['type'] == 'tram' - -def test_outer_way_not_inserted(): - """Outer way with different tag is not inserted twice into same table""" - farm = t.query_row(t.db_conf, 'osm_landusages', -19001) - assert farm['type'] == 'farmland' - assert not t.query_row(t.db_conf, 'osm_landusages', 19001) - - farmyard = t.query_row(t.db_conf, 'osm_landusages', 19002) - assert farmyard['type'] == 'farmyard' - -def test_outer_way_inserted(): - """Outer way with different tag is inserted twice into different table""" - farm = t.query_row(t.db_conf, 'osm_landusages', 19101) - assert farm['type'] == 'farm' - assert not t.query_row(t.db_conf, 'osm_landusages', -19101) - - farmyard = t.query_row(t.db_conf, 'osm_landusages', 19102) - assert farmyard['type'] == 'farmyard' - - admin = t.query_row(t.db_conf, 'osm_admin', -19101) - assert admin['type'] == 'administrative' - -def test_node_way_ref_after_delete_1(): - """Nodes refereces way""" - data = t.cache_query(nodes=[20001, 20002], deps=True) - assert '20001' in data['nodes']['20001']['ways'] - assert '20001' in data['nodes']['20002']['ways'] - assert t.query_row(t.db_conf, 'osm_roads', 20001)['type'] == 'residential' - assert t.query_row(t.db_conf, 'osm_barrierpoints', 20001)['type'] == 'block' - -def test_way_rel_ref_after_delete_1(): - """Ways references relation""" - data = t.cache_query(ways=[21001], deps=True) - assert data['ways']['21001']['relations'].keys() == ['21001'] - assert t.query_row(t.db_conf, 'osm_roads', 21001)['type'] == 'residential' - assert t.query_row(t.db_conf, 'osm_landusages', -21001)['type'] == 'park' - -def test_relation_way_not_inserted(): - """Part of relation was inserted only once.""" - park = t.query_row(t.db_conf, 'osm_landusages', -9001) - assert park['type'] == 'park' - assert park['name'] == 'rel 9001' - assert t.query_row(t.db_conf, 'osm_landusages', 9009) == None - - park = t.query_row(t.db_conf, 'osm_landusages', -9101) - assert park['type'] == 'park' - assert park['name'] == 'rel 9101' - assert t.query_row(t.db_conf, 'osm_landusages', 9109) == None - - scrub = t.query_row(t.db_conf, 'osm_landusages', 9110) - assert scrub['type'] == 'scrub' - -def test_relation_ways_inserted(): - """Outer ways of multipolygon are inserted. """ - park = t.query_row(t.db_conf, 'osm_landusages', -9201) - assert park['type'] == 'park' - assert park['name'] == '9209' - assert not t.query_row(t.db_conf, 'osm_landusages', 9201) - - # outer ways of multipolygon stand for their own - road = t.query_row(t.db_conf, 'osm_roads', 9209) - assert road['type'] == 'secondary' - assert road['name'] == '9209' - road = t.query_row(t.db_conf, 'osm_roads', 9210) - assert road['type'] == 'residential' - assert road['name'] == '9210' - - park = t.query_row(t.db_conf, 'osm_landusages', -9301) - assert park['type'] == 'park' - assert park['name'] == '' # no name on relation - - # outer ways of multipolygon stand for their own - road = t.query_row(t.db_conf, 'osm_roads', 9309) - assert road['type'] == 'secondary' - assert road['name'] == '9309' - road = t.query_row(t.db_conf, 'osm_roads', 9310) - assert road['type'] == 'residential' - assert road['name'] == '9310' - -def test_relation_way_inserted(): - """Part of relation was inserted twice.""" - park = t.query_row(t.db_conf, 'osm_landusages', -8001) - assert park['type'] == 'park' - assert park['name'] == 'rel 8001' - assert t.query_row(t.db_conf, 'osm_roads', 8009)["type"] == 'residential' - -def test_single_node_ways_not_inserted(): - """Ways with single/duplicate nodes are not inserted.""" - assert not t.query_row(t.db_conf, 'osm_roads', 30001) - assert not t.query_row(t.db_conf, 'osm_roads', 30002) - assert not t.query_row(t.db_conf, 'osm_roads', 30003) - -def test_polygon_with_duplicate_nodes_is_valid(): - """Polygon with duplicate nodes is valid.""" - geom = t.query_row(t.db_conf, 'osm_landusages', 30005)['geometry'] - assert geom.is_valid - assert len(geom.exterior.coords) == 4 - -def test_incomplete_polygons(): - """Non-closed/incomplete polygons are not inserted.""" - assert not t.query_row(t.db_conf, 'osm_landusages', 30004) - assert not t.query_row(t.db_conf, 'osm_landusages', 30006) - -def test_residential_to_secondary(): - """Residential road is not in roads_gen0/1.""" - assert t.query_row(t.db_conf, 'osm_roads', 40001)['type'] == 'residential' - assert not t.query_row(t.db_conf, 'osm_roads_gen0', 40001) - assert not t.query_row(t.db_conf, 'osm_roads_gen1', 40001) - -def test_relation_before_remove(): - """Relation and way is inserted.""" - assert t.query_row(t.db_conf, 'osm_buildings', 50011)['type'] == 'yes' - assert t.query_row(t.db_conf, 'osm_landusages', -50021)['type'] == 'park' - -def test_relation_without_tags(): - """Relation without tags is inserted.""" - assert t.query_row(t.db_conf, 'osm_buildings', 50111) == None - assert t.query_row(t.db_conf, 'osm_buildings', -50121)['type'] == 'yes' - -def test_duplicate_ids(): - """Relation/way with same ID is inserted.""" - assert t.query_row(t.db_conf, 'osm_buildings', 51001)['type'] == 'way' - assert t.query_row(t.db_conf, 'osm_buildings', -51001)['type'] == 'mp' - assert t.query_row(t.db_conf, 'osm_buildings', 51011)['type'] == 'way' - assert t.query_row(t.db_conf, 'osm_buildings', -51011)['type'] == 'mp' - -def test_generalized_banana_polygon_is_valid(): - """Generalized polygons are valid.""" - park = t.query_row(t.db_conf, 'osm_landusages', 7101) - # geometry is valid - assert park['geometry'].is_valid, park - park = t.query_row(t.db_conf, 'osm_landusages_gen0', 7101) - # simplified geometies are valid too - assert park['geometry'].is_valid, park - park = t.query_row(t.db_conf, 'osm_landusages_gen1', 7101) - assert park['geometry'].is_valid, park - -def test_generalized_linestring_is_valid(): - """Generalized linestring is valid.""" - road = t.query_row(t.db_conf, 'osm_roads', 7201) - # geometry is not simple, but valid - # check that geometry 'survives' simplification - assert not road['geometry'].is_simple, road['geometry'].wkt - assert road['geometry'].is_valid, road['geometry'].wkt - assert road['geometry'].length > 1000000 - road = t.query_row(t.db_conf, 'osm_roads_gen0', 7201) - # but simplified geometies are simple - assert road['geometry'].is_valid, road['geometry'].wkt - assert road['geometry'].length > 1000000 - road = t.query_row(t.db_conf, 'osm_roads_gen1', 7201) - assert road['geometry'].is_valid, road['geometry'].wkt - assert road['geometry'].length > 1000000 - -def test_ring_with_gap(): - """Multipolygon and way with gap (overlapping but different endpoints) gets closed""" - park = t.query_row(t.db_conf, 'osm_landusages', -7301) - assert park['geometry'].is_valid, park - - park = t.query_row(t.db_conf, 'osm_landusages', 7311) - assert park['geometry'].is_valid, park - -def test_multipolygon_with_open_ring(): - """Multipolygon is inserted even if there is an open ring/member""" - park = t.query_row(t.db_conf, 'osm_landusages', -7401) - assert park['geometry'].is_valid, park - -def test_updated_nodes1(): - """Zig-Zag line is inserted.""" - road = t.query_row(t.db_conf, 'osm_roads', 60000) - t.assert_almost_equal(road['geometry'].length, 14035.61150207768) - -def test_update_node_to_coord_1(): - """Node is inserted with tag.""" - coords = t.cache_query(nodes=(70001, 70002)) - assert coords['nodes']["70001"]["tags"] == {"amenity": "police"} - assert "tags" not in coords['nodes']["70002"] - - assert t.query_row(t.db_conf, 'osm_amenities', 70001) - assert not t.query_row(t.db_conf, 'osm_amenities', 70002) - -def test_enumerate_key(): - """Enumerate from key.""" - assert t.query_row(t.db_conf, 'osm_landusages', 100001)['enum'] == 1 - assert t.query_row(t.db_conf, 'osm_landusages', 100002)['enum'] == 0 - assert t.query_row(t.db_conf, 'osm_landusages', 100003)['enum'] == 15 - - -####################################################################### -def test_update(): - """Diff import applies""" - t.imposm3_update(t.db_conf, './build/complete_db.osc.gz', mapping_file) -####################################################################### - - -def test_no_duplicates(): - """ - Relations/ways are only inserted once - Checks #66 - """ - highways = t.query_duplicates(t.db_conf, 'osm_roads') - # one duplicate for test_node_way_inserted_twice is expected - assert highways == [[18001, 2]], highways - landusages = t.query_duplicates(t.db_conf, 'osm_landusages') - assert not landusages, landusages - -def test_updated_landusage(): - """Multipolygon relation was modified""" - t.assert_cached_node(1001, (13.5, 47.5)) - landusage_1001 = t.query_row(t.db_conf, 'osm_landusages', -1001) - # point not in polygon after update - assert not landusage_1001['geometry'].intersects(t.merc_point(13.4, 47.5)) - -def test_partial_delete(): - """Deleted relation but nodes are still cached""" - t.assert_cached_node(2001) - t.assert_cached_way(2001) - t.assert_cached_way(2002) - assert not t.query_row(t.db_conf, 'osm_landusages', -2001) - assert not t.query_row(t.db_conf, 'osm_landusages', 2001) - -def test_updated_nodes(): - """Nodes were added, modified or deleted""" - t.assert_missing_node(10000) - t.assert_cached_node(10001, (10.0, 40.0)) - t.assert_cached_node(10002, (10.1, 40.0)) - place_10001 = t.query_row(t.db_conf, 'osm_places', 10001) - assert place_10001['name'] == 'Bar', place_10001 - place_10002 = t.query_row(t.db_conf, 'osm_places', 10002) - assert place_10002['name'] == 'Baz', place_10002 - -def test_landusage_to_waterarea_2(): - """Parks converted to water moved from landusages to waterareas""" - t.assert_cached_way(11001) - t.assert_cached_way(12001) - t.assert_cached_way(13001) - - assert not t.query_row(t.db_conf, 'osm_landusages', 11001) - assert not t.query_row(t.db_conf, 'osm_landusages', -12001) - assert not t.query_row(t.db_conf, 'osm_landusages', -13001) - - assert not t.query_row(t.db_conf, 'osm_landusages_gen0', 11001) - assert not t.query_row(t.db_conf, 'osm_landusages_gen0', -12001) - assert not t.query_row(t.db_conf, 'osm_landusages_gen0', -13001) - - assert not t.query_row(t.db_conf, 'osm_landusages_gen1', 11001) - assert not t.query_row(t.db_conf, 'osm_landusages_gen1', -12001) - assert not t.query_row(t.db_conf, 'osm_landusages_gen1', -13001) - - assert t.query_row(t.db_conf, 'osm_waterareas', 11001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas', -12001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas', -13001)['type'] == 'water' - - assert t.query_row(t.db_conf, 'osm_waterareas_gen0', 11001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas_gen0', -12001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas_gen0', -13001)['type'] == 'water' - - assert t.query_row(t.db_conf, 'osm_waterareas_gen1', 11001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas_gen1', -12001)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_waterareas_gen1', -13001)['type'] == 'water' - -def test_changed_hole_tags_2(): - """Newly tagged hole is inserted""" - t.assert_cached_way(14001) - t.assert_cached_way(14011) - - assert t.query_row(t.db_conf, 'osm_waterareas', 14011)['type'] == 'water' - assert t.query_row(t.db_conf, 'osm_landusages', -14001)['type'] == 'park' - t.assert_almost_equal(t.query_row(t.db_conf, 'osm_waterareas', 14011)['geometry'].area, 26672000000, -6) - t.assert_almost_equal(t.query_row(t.db_conf, 'osm_landusages', -14001)['geometry'].area, 10373600000, -6) - -def test_split_outer_multipolygon_way_2(): - """Splitted outer way of multipolygon was inserted""" - data = t.cache_query(ways=[15001, 15002], deps=True) - assert data['ways']['15001']['relations'].keys() == ['15001'] - assert data['ways']['15002']['relations'].keys() == ['15001'] - - assert t.query_row(t.db_conf, 'osm_landusages', 15001) == None - park_15001 = t.query_row(t.db_conf, 'osm_landusages', -15001) - assert park_15001['type'] == 'park' - t.assert_almost_equal(park_15001['geometry'].area, 9816216452, -1) - assert t.query_row(t.db_conf, 'osm_roads', 15002)['type'] == 'residential' - -def test_merge_outer_multipolygon_way_2(): - """Merged outer way of multipolygon was inserted""" - data = t.cache_query(ways=[16001, 16002], deps=True) - assert data['ways']['16001']['relations'].keys() == ['16001'] - assert data['ways']['16002'] == None - - data = t.cache_query(relations=[16001], full=True) - assert sorted(data['relations']['16001']['ways'].keys()) == ['16001', '16011'] - - assert t.query_row(t.db_conf, 'osm_landusages', 16001) == None - park_16001 = t.query_row(t.db_conf, 'osm_landusages', -16001) - assert park_16001['type'] == 'park' - t.assert_almost_equal(park_16001['geometry'].area, 12779350582, -1) - assert t.query_row(t.db_conf, 'osm_roads', 16002) == None - -def test_node_way_ref_after_delete_2(): - """Node does not referece deleted way""" - data = t.cache_query(nodes=[20001, 20002], deps=True) - assert 'ways' not in data['nodes']['20001'] - assert data['nodes']['20002'] == None - assert t.query_row(t.db_conf, 'osm_roads', 20001) == None - assert t.query_row(t.db_conf, 'osm_barrierpoints', 20001)['type'] == 'block' - -def test_way_rel_ref_after_delete_2(): - """Way does not referece deleted relation""" - data = t.cache_query(ways=[21001], deps=True) - assert 'relations' not in data['ways']['21001'] - assert t.query_row(t.db_conf, 'osm_roads', 21001)['type'] == 'residential' - assert t.query_row(t.db_conf, 'osm_landusages', 21001) == None - assert t.query_row(t.db_conf, 'osm_landusages', -21001) == None - -def test_residential_to_secondary2(): - """New secondary (from residential) is now in roads_gen0/1.""" - assert t.query_row(t.db_conf, 'osm_roads', 40001)['type'] == 'secondary' - assert t.query_row(t.db_conf, 'osm_roads_gen0', 40001)['type'] == 'secondary' - assert t.query_row(t.db_conf, 'osm_roads_gen1', 40001)['type'] == 'secondary' - -def test_relation_after_remove(): - """Relation is deleted and way is still present.""" - assert t.query_row(t.db_conf, 'osm_buildings', 50011)['type'] == 'yes' - assert t.query_row(t.db_conf, 'osm_landusages', 50021) == None - assert t.query_row(t.db_conf, 'osm_landusages', -50021) == None - -def test_relation_without_tags2(): - """Relation without tags is removed.""" - t.cache_query(ways=[50111], deps=True) - assert t.cache_query(relations=[50121], deps=True)['relations']["50121"] == None - - assert t.query_row(t.db_conf, 'osm_buildings', 50111)['type'] == 'yes' - assert t.query_row(t.db_conf, 'osm_buildings', 50121) == None - assert t.query_row(t.db_conf, 'osm_buildings', -50121) == None - -def test_duplicate_ids2(): - """Only relation/way with same ID was deleted.""" - assert t.query_row(t.db_conf, 'osm_buildings', 51001)['type'] == 'way' - assert t.query_row(t.db_conf, 'osm_buildings', -51001) == None - assert t.query_row(t.db_conf, 'osm_buildings', -51011)['type'] == 'mp' - assert t.query_row(t.db_conf, 'osm_buildings', 51011) == None - -def test_updated_way2(): - """All nodes of straightened way are updated.""" - road = t.query_row(t.db_conf, 'osm_roads', 60000) - # new length 0.1 degree - t.assert_almost_equal(road['geometry'].length, 20037508.342789244/180.0/10.0) - -def test_update_node_to_coord_2(): - """Node is becomes coord after tags are removed.""" - coords = t.cache_query(nodes=(70001, 70002)) - - assert "tags" not in coords['nodes']["70001"] - assert coords['nodes']["70002"]["tags"] == {"amenity": "police"} - - assert not t.query_row(t.db_conf, 'osm_amenities', 70001) - assert t.query_row(t.db_conf, 'osm_amenities', 70002) - -def test_no_duplicate_insert(): - """ - Relation is not inserted again if a nother relation with the same way was modified - Checks #65 - """ - assert t.query_row(t.db_conf, 'osm_landusages', -201191)['type'] == 'park' - assert t.query_row(t.db_conf, 'osm_landusages', -201192)['type'] == 'forest' - assert t.query_row(t.db_conf, 'osm_roads', 201151)['type'] == 'residential' - -def test_unsupported_relation(): - """ - Unsupported relation type is not inserted with update - """ - assert not t.query_row(t.db_conf, 'osm_landusages', -201291) - assert t.query_row(t.db_conf, 'osm_landusages', 201251)['type'] == 'park' - -####################################################################### -def test_deploy_and_revert_deploy(): - """Revert deploy succeeds""" - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - - # import again to have a new import schema - t.imposm3_import(t.db_conf, './build/complete_db.pbf', mapping_file) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - - t.imposm3_deploy(t.db_conf, mapping_file) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_revert_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - -def test_remove_backup(): - """Remove backup succeeds""" - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_deploy(t.db_conf, mapping_file) - - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_remove_backups(t.db_conf, mapping_file) - - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_roads', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_roads', schema=t.TEST_SCHEMA_BACKUP) - diff --git a/test/helper.py b/test/helper.py deleted file mode 100644 index d89492e..0000000 --- a/test/helper.py +++ /dev/null @@ -1,265 +0,0 @@ -import math -import tempfile -import shutil -import subprocess -import os -import psycopg2 -import psycopg2.extras -import json -from shapely.wkb import loads as wkb_loads -from shapely.geometry import Point -import binascii - -import unittest - -__all__ = [ - "assert_almost_equal", - "query_row", - "cache_query", - "merc_point", - "imposm3_import", - "imposm3_deploy", - "imposm3_update", - "imposm3_revert_deploy", - "imposm3_remove_backups", - "table_exists", - "drop_schemas", - "TEST_SCHEMA_IMPORT", - "TEST_SCHEMA_PRODUCTION", - "TEST_SCHEMA_BACKUP", - "db_conf", - "assert_missing_node", - "assert_cached_node", - "assert_cached_way", -] - -class Dummy(unittest.TestCase): - def nop(): - pass -_t = Dummy('nop') -assert_almost_equal = _t.assertAlmostEqual - -tmpdir = None - -def setup(): - global tmpdir - tmpdir = tempfile.mkdtemp() - -def teardown(): - shutil.rmtree(tmpdir) - drop_schemas() - _close_test_connection(db_conf) - - -db_conf = { - 'host': 'localhost', -} - -TEST_SCHEMA_IMPORT = "imposm3testimport" -TEST_SCHEMA_PRODUCTION = "imposm3testpublic" -TEST_SCHEMA_BACKUP = "imposm3testbackup" - -def merc_point(lon, lat): - pole = 6378137 * math.pi # 20037508.342789244 - - x = lon * pole / 180.0 - y = math.log(math.tan((90.0+lat)*math.pi/360.0)) / math.pi * pole - return Point(x, y) - - -def pg_db_url(db_conf): - return 'postgis://%(host)s' % db_conf - -def create_geom_in_row(rowdict): - if rowdict: - rowdict['geometry'] = wkb_loads(binascii.unhexlify(rowdict['geometry'])) - return rowdict - -def query_row(db_conf, table, osmid): - conn = _test_connection(db_conf) - cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - cur.execute('select * from %s.%s where osm_id = %%s' % (TEST_SCHEMA_PRODUCTION, table), [osmid]) - results = [] - for row in cur.fetchall(): - create_geom_in_row(row) - results.append(row) - cur.close() - - if not results: - return None - if len(results) == 1: - return results[0] - return results - -def query_duplicates(db_conf, table): - conn = _test_connection(db_conf) - cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - cur.execute('select osm_id, count(osm_id) from %s.%s group by osm_id having count(osm_id) > 1' % (TEST_SCHEMA_PRODUCTION, table)) - results = [] - for row in cur.fetchall(): - results.append(row) - cur.close() - return results - -def imposm3_import(db_conf, pbf, mapping_file): - _close_test_connection(db_conf) - conn = pg_db_url(db_conf) - - try: - print subprocess.check_output(( - os.path.join("..", "imposm3") + - " import -connection %s -read %s" - " -write" - " -cachedir %s" - " -diff" - " -overwritecache" - " -dbschema-import " + TEST_SCHEMA_IMPORT + - " -optimize" - " -mapping %s ") % ( - conn, pbf, tmpdir, mapping_file - ), shell=True) - except subprocess.CalledProcessError, ex: - print ex.output - raise - -def imposm3_deploy(db_conf, mapping_file): - _close_test_connection(db_conf) - conn = pg_db_url(db_conf) - - try: - print subprocess.check_output(( - os.path.join("..", "imposm3") + - " import -connection %s" - " -dbschema-import " + TEST_SCHEMA_IMPORT + - " -dbschema-production " + TEST_SCHEMA_PRODUCTION + - " -dbschema-backup " + TEST_SCHEMA_BACKUP + - " -deployproduction" - " -mapping %s ") % ( - conn, mapping_file, - ), shell=True) - except subprocess.CalledProcessError, ex: - print ex.output - raise - -def imposm3_revert_deploy(db_conf, mapping_file): - _close_test_connection(db_conf) - conn = pg_db_url(db_conf) - - try: - print subprocess.check_output(( - os.path.join("..", "imposm3") + - " import -connection %s" - " -dbschema-import " + TEST_SCHEMA_IMPORT + - " -dbschema-production " + TEST_SCHEMA_PRODUCTION + - " -dbschema-backup " + TEST_SCHEMA_BACKUP + - " -revertdeploy" - " -mapping %s ") % ( - conn, mapping_file, - ), shell=True) - except subprocess.CalledProcessError, ex: - print ex.output - raise - -def imposm3_remove_backups(db_conf, mapping_file): - _close_test_connection(db_conf) - conn = pg_db_url(db_conf) - - try: - print subprocess.check_output(( - os.path.join("..", "imposm3") + - " import -connection %s" - " -dbschema-backup " + TEST_SCHEMA_BACKUP + - " -removebackup" - " -mapping %s ") % ( - conn, mapping_file, - ), shell=True) - except subprocess.CalledProcessError, ex: - print ex.output - raise - -def imposm3_update(db_conf, osc, mapping_file): - _close_test_connection(db_conf) - conn = pg_db_url(db_conf) - - try: - print subprocess.check_output(( - os.path.join("..", "imposm3") + - " diff -connection %s" - " -cachedir %s" - " -limitto clipping.geojson" - " -dbschema-production " + TEST_SCHEMA_PRODUCTION + - " -mapping %s %s") % ( - conn, tmpdir, mapping_file, osc, - ), shell=True) - except subprocess.CalledProcessError, ex: - print ex.output - raise - -def cache_query(nodes='', ways='', relations='', deps='', full=''): - if nodes: - nodes = '-node ' + ','.join(map(str, nodes)) - if ways: - ways = '-way ' + ','.join(map(str, ways)) - if relations: - relations = '-rel ' + ','.join(map(str, relations)) - if deps: - deps = '-deps' - if full: - full = '-full' - out = subprocess.check_output( - os.path.join("..", "imposm3") + - " query-cache -cachedir %s %s %s %s %s %s" % ( - tmpdir, nodes, ways, relations, deps, full), - shell=True) - print out - return json.loads(out) - -def _test_connection(db_conf): - if '_connection' in db_conf: - return db_conf['_connection'] - db_conf['_connection'] = psycopg2.connect(**db_conf) - return db_conf['_connection'] - -def _close_test_connection(db_conf): - if '_connection' in db_conf: - db_conf['_connection'].close() - del db_conf['_connection'] - -def table_exists(table, schema=TEST_SCHEMA_IMPORT): - conn = _test_connection(db_conf) - cur = conn.cursor() - cur.execute("SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name='%s' AND table_schema='%s')" - % (table, schema)) - - exists = cur.fetchone()[0] - cur.close() - return exists - -def assert_missing_node(id): - data = cache_query(nodes=[id]) - if data['nodes'][str(id)]: - raise AssertionError('node %d found' % id) - -def assert_cached_node(id, (lon, lat)=(None, None)): - data = cache_query(nodes=[id]) - node = data['nodes'][str(id)] - if not node: - raise AssertionError('node %d not found' % id) - - if lon and lat: - assert_almost_equal(lon, node['lon'], 6) - assert_almost_equal(lat, node['lat'], 6) - -def assert_cached_way(id): - data = cache_query(ways=[id]) - if not data['ways'][str(id)]: - raise AssertionError('way %d not found' % id) - -def drop_schemas(): - conn = _test_connection(db_conf) - cur = conn.cursor() - cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_IMPORT) - cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_PRODUCTION) - cur.execute("DROP SCHEMA IF EXISTS %s CASCADE" % TEST_SCHEMA_BACKUP) - conn.commit() - diff --git a/test/single_table_test.py b/test/single_table_test.py deleted file mode 100644 index fb82adf..0000000 --- a/test/single_table_test.py +++ /dev/null @@ -1,184 +0,0 @@ -import psycopg2 -import psycopg2.extras - -import helper as t - -psycopg2.extras.register_hstore(psycopg2.connect(**t.db_conf), globally=True) - -mapping_file = 'single_table_mapping.json' - -def setup(): - t.setup() - -def teardown(): - t.teardown() - -RELOFFSET = int(-1e17) - -####################################################################### -def test_import(): - """Import succeeds""" - t.drop_schemas() - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - t.imposm3_import(t.db_conf, './build/single_table.pbf', mapping_file) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - -def test_deploy(): - """Deploy succeeds""" - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - t.imposm3_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - -####################################################################### - -def test_non_mapped_node_is_missing(): - """Node without mapped tags is missing.""" - t.assert_cached_node(10001, (10, 42)) - assert not t.query_row(t.db_conf, 'osm_all', 10001) - -def test_mapped_node(): - """Node is stored with all tags.""" - t.assert_cached_node(10002, (11, 42)) - poi = t.query_row(t.db_conf, 'osm_all', 10002) - assert poi['tags'] == {'random': 'tag', 'but': 'mapped', 'poi': 'unicorn'} - - -def test_non_mapped_way_is_missing(): - """Way without mapped tags is missing.""" - t.assert_cached_way(20101) - assert not t.query_row(t.db_conf, 'osm_all', 20101) - t.assert_cached_way(20102) - assert not t.query_row(t.db_conf, 'osm_all', 20102) - t.assert_cached_way(20103) - assert not t.query_row(t.db_conf, 'osm_all', 20103) - -def test_mapped_way(): - """Way is stored with all tags.""" - t.assert_cached_way(20201) - highway = t.query_row(t.db_conf, 'osm_all', -20201) - assert highway['tags'] == {'random': 'tag', 'highway': 'yes'} - -def test_non_mapped_closed_way_is_missing(): - """Closed way without mapped tags is missing.""" - t.assert_cached_way(20301) - assert not t.query_row(t.db_conf, 'osm_all', -20301) - -def test_mapped_closed_way(): - """Closed way is stored with all tags.""" - t.assert_cached_way(20401) - building = t.query_row(t.db_conf, 'osm_all', -20401) - assert building['tags'] == {'random': 'tag', 'building': 'yes'} - -def test_mapped_closed_way_area_yes(): - """Closed way with area=yes is not stored as linestring.""" - t.assert_cached_way(20501) - elem = t.query_row(t.db_conf, 'osm_all', -20501) - assert elem['geometry'].type == 'Polygon', elem['geometry'].type - assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'yes'} - -def test_mapped_closed_way_area_no(): - """Closed way with area=no is not stored as polygon.""" - t.assert_cached_way(20502) - elem = t.query_row(t.db_conf, 'osm_all', -20502) - assert elem['geometry'].type == 'LineString', elem['geometry'].type - assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'no'} - -def test_mapped_closed_way_without_area(): - """Closed way without area is stored as mapped (linestring and polygon).""" - t.assert_cached_way(20601) - elems = t.query_row(t.db_conf, 'osm_all', -20601) - assert len(elems) == 2 - elems.sort(key=lambda x: x['geometry'].type) - - assert elems[0]['geometry'].type == 'LineString', elems[0]['geometry'].type - assert elems[0]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} - assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type - assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'} - -def test_duplicate_ids_1(): - """Points/lines/polygons with same ID are inserted.""" - node = t.query_row(t.db_conf, 'osm_all', 31101) - assert node['geometry'].type == 'Point', node['geometry'].type - assert node['tags'] == {'amenity': 'cafe'} - assert node['geometry'].distance(t.merc_point(80, 47)) < 1 - - ways = t.query_row(t.db_conf, 'osm_all', -31101) - ways.sort(key=lambda x: x['geometry'].type) - assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type - assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type - assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - - rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L) - assert rel['geometry'].type == 'Polygon', rel['geometry'].type - assert rel['tags'] == {'building': 'yes'} - - -####################################################################### - -def test_update(): - """Diff import applies""" - t.imposm3_update(t.db_conf, './build/single_table.osc.gz', mapping_file) - -####################################################################### - -def test_duplicate_ids_2(): - """Node moved and ways/rels with same ID are still present.""" - node = t.query_row(t.db_conf, 'osm_all', 31101) - assert node['geometry'].type == 'Point', node['geometry'].type - assert node['tags'] == {'amenity': 'cafe'} - assert node['geometry'].distance(t.merc_point(81, 47)) < 1 - - ways = t.query_row(t.db_conf, 'osm_all', -31101) - ways.sort(key=lambda x: x['geometry'].type) - - assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type - assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type - assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'} - - rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L) - - assert rel['geometry'].type == 'Polygon', rel['geometry'].type - assert rel['tags'] == {'building': 'yes'} - -####################################################################### -def test_deploy_and_revert_deploy(): - """Revert deploy succeeds""" - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - - # import again to have a new import schema - t.imposm3_import(t.db_conf, './build/single_table.pbf', mapping_file) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - - t.imposm3_deploy(t.db_conf, mapping_file) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_revert_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - -def test_remove_backup(): - """Remove backup succeeds""" - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_deploy(t.db_conf, mapping_file) - - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_remove_backups(t.db_conf, mapping_file) - - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_all', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_all', schema=t.TEST_SCHEMA_BACKUP) - From 36bf5f14b3a19bc06c9c3892cb2b73a49d315b1f Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 21 Dec 2015 13:09:46 +0100 Subject: [PATCH 20/36] update Makefile/README for new go system tests --- Makefile | 6 ++++-- README.md | 26 ++++++++++---------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 3e50c9d..f1311c4 100644 --- a/Makefile +++ b/Makefile @@ -34,11 +34,13 @@ clean: rm -f imposm3 (cd test && make clean) -test: test-unit test-system +test: imposm3 + $(GO) test ./... -i + $(GO) test ./... test-unit: imposm3 $(GO) test ./... -i - $(GO) test ./... + $(GO) test `$(GO) list ./... | grep -v 'imposm3/test'` test-system: imposm3 (cd test && make test) diff --git a/README.md b/README.md index 633ae2e..03a3dd2 100644 --- a/README.md +++ b/README.md @@ -250,29 +250,23 @@ The GEOS package is released as LGPL3 and is linked dynamically. See LICENSE.bin #### Unit tests #### - go test imposm3/... +To run all unit tests: + + make test-unit + +Or: + + godep go test ./... #### System tests #### There are system test that import and update OSM data and verify the database content. - -##### Dependencies ##### - -These tests are written in Python and requires `nose`, `shapely` and `psycopg2`. - -On a recent Ubuntu can install the following packages for that: `python-nose python-shapely python-psycopg2` -Or you can [install a Python virtualenv](https://virtualenv.pypa.io/en/latest/installation.html): - - virtualenv imposm3test - source imposm3test/bin/activate - pip install nose shapely psycopg2 - -You also need `osmosis` to create test PBF files. -There is a Makefile that (re)builds `imposm3` and creates all test files if necessary and then runs the test itself. +You need `osmosis` to create the test PBF files. +There is a Makefile that creates all test files if necessary and then runs the test itself. make test Call `make test-system` to skip the unit tests. -WARNING: It uses your local PostgeSQL database (`import` schema). Change the database with the standard `PGDATABASE`, `PGHOST`, etc. environment variables. +WARNING: It uses your local PostgeSQL database (`imposm3testimport`, `imposm3testproduction` and `imposm3testbackup` schema). Change the database with the standard `PGDATABASE`, `PGHOST`, etc. environment variables. From 7a49cdff6e3eae6267edd3ad777bf0dec727de48 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 21 Dec 2015 13:27:59 +0100 Subject: [PATCH 21/36] remove link to freebsd build instructions as they only build a fixed version --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 03a3dd2..01a46f1 100644 --- a/README.md +++ b/README.md @@ -173,9 +173,6 @@ Imposm contains a fixed set of the dependencies that are known to work. You need cd src/github.com/omniscale/imposm3 godep go install ./... -### FreeBSD - -On FreeBSD you can use the ports system: Simply fetch https://github.com/thomersch/imposm3-freebsd and run `make install`. Usage ----- From 902dee364f7d81d61e2570ad44a77d0f7dac803c Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 22 Dec 2015 10:09:54 +0100 Subject: [PATCH 22/36] fix go install command (closes #78) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01a46f1..9168292 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Imposm contains a fixed set of the dependencies that are known to work. You need git clone https://github.com/omniscale/imposm3 src/github.com/omniscale/imposm3 cd src/github.com/omniscale/imposm3 - godep go install ./... + godep go install ./ Usage From 901b40bfd7e9d0eace5c74903d64731fdae2fafa Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 22 Dec 2015 10:10:35 +0100 Subject: [PATCH 23/36] diff: document last.state.txt (closes #79) --- docs/tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 9226ba7..2dc5b33 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -209,6 +209,8 @@ To update an existing database with three change files:: imposm3 diff -config config.json changes-1.osc.gz changes-2.osc.gz changes-3.osc.gz +Imposm 3 stores the sequence number of the last imported changeset in `${cachedir}/last.state.txt`, if it finds a matching state file (`123.state.txt` for `123.osc.gz`). Imposm refuses to import the same diff files a second time if these state files are present. + Remember that you have to make the initial import with the ``-diff`` option. See above. .. note:: You should not make changes to the mapping file after the initial import. Changes are not detected and this can result aborted updates or incomplete data. From 1aff01cc9e958295221fd130c3e2bb560c84b8ef Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 8 Jan 2016 09:35:41 +0100 Subject: [PATCH 24/36] use column name in index name instead of 'osm_id' --- database/postgis/postgis.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index bbc6f67..3fec077 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -203,8 +203,8 @@ func createIndex(pg *PostGIS, tableName string, columns []ColumnSpec) error { } } if col.FieldType.Name == "id" { - sql := fmt.Sprintf(`CREATE INDEX "%s_osm_id_idx" ON "%s"."%s" USING BTREE ("%s")`, - tableName, pg.Config.ImportSchema, tableName, col.Name) + sql := fmt.Sprintf(`CREATE INDEX "%s_%s_idx" ON "%s"."%s" USING BTREE ("%s")`, + tableName, col.Name, pg.Config.ImportSchema, tableName, col.Name) step := log.StartStep(fmt.Sprintf("Creating OSM id index on %s", tableName)) _, err := pg.Db.Exec(sql) log.StopStep(step) From 423390ea71fc40f659aac649b1fe0807473f7def Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Jan 2016 11:19:28 +0100 Subject: [PATCH 25/36] Add new relation_members table type --- database/database.go | 4 + database/postgis/postgis.go | 16 +++- database/postgis/spec.go | 10 ++- diff/process.go | 2 + element/element.go | 2 + import_/import.go | 2 + mapping/config.go | 25 +++--- mapping/fields.go | 95 ++++++++++++++++++----- mapping/matcher.go | 34 ++++++-- test/Makefile | 10 ++- test/route_relation.osc | 3 + test/route_relation.osm | 113 +++++++++++++++++++++++++++ test/route_relation_mapping.json | 128 +++++++++++++++++++++++++++++++ test/route_relation_test.go | 53 +++++++++++++ test/route_relation_test.py | 83 ++++++++++++++++++++ writer/relations.go | 98 ++++++++++++++++++++--- 16 files changed, 626 insertions(+), 52 deletions(-) create mode 100644 test/route_relation.osc create mode 100644 test/route_relation.osm create mode 100644 test/route_relation_mapping.json create mode 100644 test/route_relation_test.go create mode 100644 test/route_relation_test.py diff --git a/database/database.go b/database/database.go index c265fbc..50a8fdf 100644 --- a/database/database.go +++ b/database/database.go @@ -36,6 +36,7 @@ type Inserter interface { InsertPoint(element.OSMElem, geom.Geometry, []mapping.Match) error InsertLineString(element.OSMElem, geom.Geometry, []mapping.Match) error InsertPolygon(element.OSMElem, geom.Geometry, []mapping.Match) error + InsertRelationMember(element.Relation, element.Member, geom.Geometry, []mapping.Match) error } type Deployer interface { @@ -103,6 +104,9 @@ func (n *nullDb) Abort() error func (n *nullDb) InsertPoint(element.OSMElem, geom.Geometry, []mapping.Match) error { return nil } func (n *nullDb) InsertLineString(element.OSMElem, geom.Geometry, []mapping.Match) error { return nil } func (n *nullDb) InsertPolygon(element.OSMElem, geom.Geometry, []mapping.Match) error { return nil } +func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geometry, []mapping.Match) error { + return nil +} func newNullDb(conf Config, m *mapping.Mapping) (DB, error) { return &nullDb{}, nil diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 3fec077..8c710ec 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -58,7 +58,7 @@ func createTable(tx *sql.Tx, spec TableSpec) error { } func addGeometryColumn(tx *sql.Tx, tableName string, spec TableSpec) error { - colName := "geometry" + colName := "" for _, col := range spec.Columns { if col.Type.Name() == "GEOMETRY" { colName = col.Name @@ -66,6 +66,10 @@ func addGeometryColumn(tx *sql.Tx, tableName string, spec TableSpec) error { } } + if colName == "" { + return nil + } + geomType := strings.ToUpper(spec.GeometryType) if geomType == "POLYGON" { geomType = "GEOMETRY" // for multipolygon support @@ -480,6 +484,16 @@ func (pg *PostGIS) InsertPolygon(elem element.OSMElem, geom geom.Geometry, match return nil } +func (pg *PostGIS) InsertRelationMember(rel element.Relation, m element.Member, geom geom.Geometry, matches []mapping.Match) error { + for _, match := range matches { + row := match.MemberRow(&rel, &m, &geom) + if err := pg.txRouter.Insert(match.Table.Name, row); err != nil { + return err + } + } + return nil +} + func (pg *PostGIS) Delete(id int64, matches interface{}) error { if matches, ok := matches.([]mapping.Match); ok { for _, match := range matches { diff --git a/database/postgis/spec.go b/database/postgis/spec.go index e6fa7f0..8c50428 100644 --- a/database/postgis/spec.go +++ b/database/postgis/spec.go @@ -125,11 +125,19 @@ func (spec *TableSpec) DeleteSQL() string { } func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { + var geomType string + switch t.Type { + case mapping.RelationMemberTable: + geomType = "geometry" + default: + geomType = string(t.Type) + } + spec := TableSpec{ Name: t.Name, FullName: pg.Prefix + t.Name, Schema: pg.Config.ImportSchema, - GeometryType: string(t.Type), + GeometryType: geomType, Srid: pg.Config.Srid, } for _, field := range t.Fields { diff --git a/diff/process.go b/diff/process.go index ca378fa..c15da9a 100644 --- a/diff/process.go +++ b/diff/process.go @@ -149,6 +149,8 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi relations, db, progress, tagmapping.PolygonMatcher(), + tagmapping.RelationMatcher(), + tagmapping.RelationMemberMatcher(), config.BaseOptions.Srid) relWriter.SetLimiter(geometryLimiter) relWriter.SetExpireor(expireor) diff --git a/element/element.go b/element/element.go index 0bc5fdf..c98f586 100644 --- a/element/element.go +++ b/element/element.go @@ -72,6 +72,8 @@ type Member struct { Type MemberType `json:"type"` Role string `json:"role"` Way *Way `json:"-"` + Node *Node `json:"-"` + Elem *OSMElem `json:"-"` } type Relation struct { diff --git a/import_/import.go b/import_/import.go index 584e1b5..7138c4f 100644 --- a/import_/import.go +++ b/import_/import.go @@ -182,6 +182,8 @@ func Import() { relations, db, progress, tagmapping.PolygonMatcher(), + tagmapping.RelationMatcher(), + tagmapping.RelationMemberMatcher(), config.BaseOptions.Srid) relWriter.SetLimiter(geometryLimiter) relWriter.EnableConcurrent() diff --git a/mapping/config.go b/mapping/config.go index 78de91d..430510c 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -11,11 +11,12 @@ import ( ) type Field struct { - Name string `yaml:"name"` - Key Key `yaml:"key"` - Keys []Key `yaml:"keys"` - Type string `yaml:"type"` - Args map[string]interface{} `yaml:"args"` + Name string `yaml:"name"` + Key Key `yaml:"key"` + Keys []Key `yaml:"keys"` + Type string `yaml:"type"` + Args map[string]interface{} `yaml:"args"` + FromMembers bool `yaml:"from_members"` } type Table struct { @@ -133,6 +134,10 @@ func (tt *TableType) UnmarshalJSON(data []byte) error { *tt = PolygonTable case `"geometry"`: *tt = GeometryTable + case `"relation"`: + *tt = RelationTable + case `"relation_member"`: + *tt = RelationMemberTable default: return errors.New("unknown type " + string(data)) } @@ -140,10 +145,12 @@ func (tt *TableType) UnmarshalJSON(data []byte) error { } const ( - PolygonTable TableType = "polygon" - LineStringTable TableType = "linestring" - PointTable TableType = "point" - GeometryTable TableType = "geometry" + PolygonTable TableType = "polygon" + LineStringTable TableType = "linestring" + PointTable TableType = "point" + GeometryTable TableType = "geometry" + RelationTable TableType = "relation" + RelationMemberTable TableType = "relation_member" ) func NewMapping(filename string) (*Mapping, error) { diff --git a/mapping/fields.go b/mapping/fields.go index c899ed0..15a7900 100644 --- a/mapping/fields.go +++ b/mapping/fields.go @@ -17,26 +17,31 @@ var AvailableFieldTypes map[string]FieldType func init() { AvailableFieldTypes = map[string]FieldType{ - "bool": {"bool", "bool", Bool, nil}, - "boolint": {"boolint", "int8", BoolInt, nil}, - "id": {"id", "int64", Id, nil}, - "string": {"string", "string", String, nil}, - "direction": {"direction", "int8", Direction, nil}, - "integer": {"integer", "int32", Integer, nil}, - "mapping_key": {"mapping_key", "string", KeyName, nil}, - "mapping_value": {"mapping_value", "string", ValueName, nil}, - "geometry": {"geometry", "geometry", Geometry, nil}, - "validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil}, - "hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil}, - "wayzorder": {"wayzorder", "int32", WayZOrder, nil}, - "pseudoarea": {"pseudoarea", "float32", PseudoArea, nil}, - "zorder": {"zorder", "int32", nil, MakeZOrder}, - "enumerate": {"enumerate", "int32", nil, MakeEnumerate}, - "string_suffixreplace": {"string_suffixreplace", "string", nil, MakeSuffixReplace}, + "bool": {"bool", "bool", Bool, nil, nil, false}, + "boolint": {"boolint", "int8", BoolInt, nil, nil, false}, + "id": {"id", "int64", Id, nil, nil, false}, + "string": {"string", "string", String, nil, nil, false}, + "direction": {"direction", "int8", Direction, nil, nil, false}, + "integer": {"integer", "int32", Integer, nil, nil, false}, + "mapping_key": {"mapping_key", "string", KeyName, nil, nil, false}, + "mapping_value": {"mapping_value", "string", ValueName, nil, nil, false}, + "relation_member_id": {"relation_member_id", "int64", nil, nil, RelationMemberID, true}, + "relation_member_role": {"relation_member_role", "string", nil, nil, RelationMemberRole, true}, + "relation_member_type": {"relation_member_type", "int8", nil, nil, RelationMemberType, true}, + "relation_member_index": {"relation_member_index", "int32", nil, nil, RelationMemberIndex, true}, + "geometry": {"geometry", "geometry", Geometry, nil, nil, false}, + "validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil, nil, false}, + "hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil, nil, false}, + "wayzorder": {"wayzorder", "int32", WayZOrder, nil, nil, false}, + "pseudoarea": {"pseudoarea", "float32", PseudoArea, nil, nil, false}, + "zorder": {"zorder", "int32", nil, MakeZOrder, nil, false}, + "enumerate": {"enumerate", "int32", nil, MakeEnumerate, nil, false}, + "string_suffixreplace": {"string_suffixreplace", "string", nil, MakeSuffixReplace, nil, false}, } } type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{} +type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{} type MakeMakeValue func(string, FieldType, Field) (MakeValue, error) @@ -55,6 +60,22 @@ func (f *FieldSpec) Value(elem *element.OSMElem, geom *geom.Geometry, match Matc return nil } +func (f *FieldSpec) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} { + if f.Type.Func != nil { + if f.Type.FromMembers { + if member.Elem == nil { + return nil + } + return f.Type.Func(member.Elem.Tags[string(f.Key)], member.Elem, geom, match) + } + return f.Type.Func(rel.Tags[string(f.Key)], &rel.OSMElem, geom, match) + } + if f.Type.MemberFunc != nil { + return f.Type.MemberFunc(rel, member, match) + } + return nil +} + type TableFields struct { fields []FieldSpec } @@ -67,6 +88,14 @@ func (t *TableFields) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match return row } +func (t *TableFields) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} { + var row []interface{} + for _, field := range t.fields { + row = append(row, field.MemberValue(rel, member, geom, match)) + } + return row +} + func (field *Field) FieldType() *FieldType { if fieldType, ok := AvailableFieldTypes[field.Type]; ok { if fieldType.MakeFunc != nil { @@ -75,8 +104,9 @@ func (field *Field) FieldType() *FieldType { log.Print(err) return nil } - fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil} + fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMembers} } + fieldType.FromMembers = field.FromMembers return &fieldType } return nil @@ -101,10 +131,12 @@ func (t *Table) TableFields() *TableFields { } type FieldType struct { - Name string - GoType string - Func MakeValue - MakeFunc MakeMakeValue + Name string + GoType string + Func MakeValue + MakeFunc MakeMakeValue + MemberFunc MakeMemberValue + FromMembers bool } func Bool(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { @@ -145,6 +177,27 @@ func ValueName(val string, elem *element.OSMElem, geom *geom.Geometry, match Mat return match.Value } +func RelationMemberType(rel *element.Relation, member *element.Member, match Match) interface{} { + return member.Type +} + +func RelationMemberRole(rel *element.Relation, member *element.Member, match Match) interface{} { + return member.Role +} + +func RelationMemberID(rel *element.Relation, member *element.Member, match Match) interface{} { + return member.Id +} + +func RelationMemberIndex(rel *element.Relation, member *element.Member, match Match) interface{} { + for i := range rel.Members { + if rel.Members[i].Id == member.Id { + return i + } + } + return -1 +} + func Direction(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { if val == "1" || val == "yes" || val == "true" { return 1 diff --git a/mapping/matcher.go b/mapping/matcher.go index 38926ae..ffab4ad 100644 --- a/mapping/matcher.go +++ b/mapping/matcher.go @@ -1,29 +1,47 @@ package mapping import ( + _ "log" + "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" ) func (m *Mapping) PointMatcher() NodeMatcher { mappings := make(TagTables) - m.mappings("point", mappings) + m.mappings(PointTable, mappings) filters := m.ElementFilters() - return &tagMatcher{mappings, m.tables("point"), filters, false} + return &tagMatcher{mappings, m.tables(PointTable), filters, false} } func (m *Mapping) LineStringMatcher() WayMatcher { mappings := make(TagTables) - m.mappings("linestring", mappings) + m.mappings(LineStringTable, mappings) filters := m.ElementFilters() - return &tagMatcher{mappings, m.tables("linestring"), filters, false} + return &tagMatcher{mappings, m.tables(LineStringTable), filters, false} } func (m *Mapping) PolygonMatcher() RelWayMatcher { mappings := make(TagTables) - m.mappings("polygon", mappings) + m.mappings(PolygonTable, mappings) filters := m.ElementFilters() - return &tagMatcher{mappings, m.tables("polygon"), filters, true} + return &tagMatcher{mappings, m.tables(PolygonTable), filters, true} +} + +func (m *Mapping) RelationMatcher() RelationMatcher { + mappings := make(TagTables) + m.mappings(RelationTable, mappings) + filters := m.ElementFilters() + log.Print(mappings) + return &tagMatcher{mappings, m.tables(RelationTable), filters, true} +} + +func (m *Mapping) RelationMemberMatcher() RelationMatcher { + mappings := make(TagTables) + m.mappings(RelationMemberTable, mappings) + filters := m.ElementFilters() + log.Print(mappings) + return &tagMatcher{mappings, m.tables(RelationMemberTable), filters, true} } type Match struct { @@ -61,6 +79,10 @@ func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} { return m.tableFields.MakeRow(elem, geom, *m) } +func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry) []interface{} { + return m.tableFields.MakeMemberRow(rel, member, geom, *m) +} + func (tm *tagMatcher) MatchNode(node *element.Node) []Match { return tm.match(&node.Tags) } diff --git a/test/Makefile b/test/Makefile index 516e27e..fa6e637 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,4 +1,4 @@ -.PHONY: all test clean +.PHONY: all test clean files ifdef VERBOSE TESTOPTS = -v @@ -16,12 +16,16 @@ OSCGZ_FILES=$(addprefix build/,$(patsubst %.osc,%.osc.gz,$(wildcard *.osc))) build/%.pbf: %.osm @mkdir -p build - osmosis --read-xml $< --sort TypeThenId --write-pbf $@ omitmetadata=true + osmosis --read-xml $< --sort type="TypeThenId" --write-pbf $@ omitmetadata=true build/%.osc.gz: %.osc @mkdir -p build gzip --stdout $< > $@ -test: $(PBF_FILES) $(OSCGZ_FILES) +files: $(PBF_FILES) $(OSCGZ_FILES) + +test: files (cd .. && godep go test ./test $(TESTOPTS)) +route_relation: files + (cd .. && godep go test -test.run TestRouteRelation_ ./test $(TESTOPTS)) diff --git a/test/route_relation.osc b/test/route_relation.osc new file mode 100644 index 0000000..7f21e40 --- /dev/null +++ b/test/route_relation.osc @@ -0,0 +1,3 @@ + + + diff --git a/test/route_relation.osm b/test/route_relation.osm new file mode 100644 index 0000000..b1d5682 --- /dev/null +++ b/test/route_relation.osm @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/route_relation_mapping.json b/test/route_relation_mapping.json new file mode 100644 index 0000000..c81fc92 --- /dev/null +++ b/test/route_relation_mapping.json @@ -0,0 +1,128 @@ +{ + "tags": { + "load_all": true, + "exclude": [ + "created_by", + "source" + ] + }, + "tables": { + "route_members": { + "fields": [ + { + "type": "id", + "name": "osm_id" + }, + { + "type": "string", + "name": "ref", + "key": "ref" + }, + { + "type": "relation_member_id", + "name": "member" + }, + { + "type": "relation_member_index", + "name": "index" + }, + { + "type": "relation_member_role", + "name": "role" + }, + { + "type": "relation_member_type", + "name": "type" + }, + { + "type": "geometry", + "name": "geometry" + }, + { + "type": "string", + "name": "name", + "key": "name", + "from_members": true + } + ], + "type": "relation_member", + "mapping": { + "route": [ + "bus", + "tram", + "rail" + ] + } + }, + "routes": { + "fields": [ + { + "type": "id", + "name": "osm_id" + }, + { + "type": "string", + "name": "ref", + "key": "ref" + }, + { + "type": "hstore_tags", + "name": "tags" + } + ], + "type": "relation", + "mapping": { + "route": [ + "bus", + "tram", + "rail" + ] + } + }, + "master_routes": { + "fields": [ + { + "type": "id", + "name": "osm_id" + }, + { + "type": "relation_member_id", + "name": "member" + }, + { + "type": "relation_member_index", + "name": "index" + }, + { + "type": "relation_member_role", + "name": "role" + }, + { + "type": "relation_member_type", + "name": "type" + }, + { + "type": "geometry", + "name": "geometry" + }, + { + "type": "string", + "name": "subname", + "key": "name", + "from_members": true + }, + { + "type": "string", + "name": "name", + "key": "name" + } + ], + "type": "relation_member", + "mapping": { + "route_master": [ + "bus" + ] + } + } + } +} diff --git a/test/route_relation_test.go b/test/route_relation_test.go new file mode 100644 index 0000000..f71c0af --- /dev/null +++ b/test/route_relation_test.go @@ -0,0 +1,53 @@ +package test + +import ( + "database/sql" + + "testing" + + "github.com/omniscale/imposm3/geom/geos" +) + +func TestRouteRelation_Prepare(t *testing.T) { + ts.dir = "/tmp/imposm3test" + ts.config = importConfig{ + connection: "postgis://", + cacheDir: ts.dir, + osmFileName: "build/route_relation.pbf", + mappingFileName: "route_relation_mapping.json", + } + ts.g = geos.NewGeos() + + var err error + ts.db, err = sql.Open("postgres", "sslmode=disable") + if err != nil { + t.Fatal(err) + } + ts.dropSchemas() +} + +func TestRouteRelation_Import(t *testing.T) { + if ts.tableExists(t, dbschemaImport, "osm_routes") != false { + t.Fatalf("table osm_routes exists in schema %s", dbschemaImport) + } + ts.importOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_routes") != true { + t.Fatalf("table osm_routes does not exists in schema %s", dbschemaImport) + } +} + +func TestRouteRelation_Deploy(t *testing.T) { + ts.deployOsm(t) + if ts.tableExists(t, dbschemaImport, "osm_routes") != false { + t.Fatalf("table osm_routes exists in schema %s", dbschemaImport) + } + if ts.tableExists(t, dbschemaProduction, "osm_routes") != true { + t.Fatalf("table osm_routes does not exists in schema %s", dbschemaProduction) + } +} + +// ####################################################################### + +func TestRouteRelation_Update(t *testing.T) { + ts.updateOsm(t, "./build/route_relation.osc.gz") +} diff --git a/test/route_relation_test.py b/test/route_relation_test.py new file mode 100644 index 0000000..eb20330 --- /dev/null +++ b/test/route_relation_test.py @@ -0,0 +1,83 @@ +import psycopg2 +import psycopg2.extras + +import helper as t + +psycopg2.extras.register_hstore(psycopg2.connect(**t.db_conf), globally=True) + +mapping_file = 'route_relation_mapping.json' + +def setup(): + t.setup() + +def teardown(): + t.teardown() + +RELOFFSET = int(-1e17) + +####################################################################### +def test_import(): + """Import succeeds""" + t.drop_schemas() + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + t.imposm3_import(t.db_conf, './build/route_relation.pbf', mapping_file) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + +def test_deploy(): + """Deploy succeeds""" + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + t.imposm3_deploy(t.db_conf, mapping_file) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + +####################################################################### + + +####################################################################### + +def test_update(): + """Diff import applies""" + t.imposm3_update(t.db_conf, './build/route_relation.osc.gz', mapping_file) + +####################################################################### + + +####################################################################### +def test_deploy_and_revert_deploy(): + """Revert deploy succeeds""" + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + + # import again to have a new import schema + t.imposm3_import(t.db_conf, './build/route_relation.pbf', mapping_file) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + + t.imposm3_deploy(t.db_conf, mapping_file) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + + t.imposm3_revert_deploy(t.db_conf, mapping_file) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + +def test_remove_backup(): + """Remove backup succeeds""" + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + + t.imposm3_deploy(t.db_conf, mapping_file) + + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + + t.imposm3_remove_backups(t.db_conf, mapping_file) + + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) + assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) + assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) + diff --git a/writer/relations.go b/writer/relations.go index 5fad3b7..8351456 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -9,17 +9,19 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/expire" geomp "github.com/omniscale/imposm3/geom" - "github.com/omniscale/imposm3/geom/geos" + geosp "github.com/omniscale/imposm3/geom/geos" "github.com/omniscale/imposm3/mapping" "github.com/omniscale/imposm3/stats" ) type RelationWriter struct { OsmElemWriter - singleIdSpace bool - rel chan *element.Relation - polygonMatcher mapping.RelWayMatcher - maxGap float64 + singleIdSpace bool + rel chan *element.Relation + polygonMatcher mapping.RelWayMatcher + relationMatcher mapping.RelationMatcher + relationMemberMatcher mapping.RelationMatcher + maxGap float64 } func NewRelationWriter( @@ -30,6 +32,8 @@ func NewRelationWriter( inserter database.Inserter, progress *stats.Statistics, matcher mapping.RelWayMatcher, + relMatcher mapping.RelationMatcher, + relMemberMatcher mapping.RelationMatcher, srid int, ) *OsmElemWriter { maxGap := 1e-1 // 0.1m @@ -45,10 +49,12 @@ func NewRelationWriter( inserter: inserter, srid: srid, }, - singleIdSpace: singleIdSpace, - polygonMatcher: matcher, - rel: rel, - maxGap: maxGap, + singleIdSpace: singleIdSpace, + polygonMatcher: matcher, + relationMatcher: relMatcher, + relationMemberMatcher: relMemberMatcher, + rel: rel, + maxGap: maxGap, } rw.OsmElemWriter.writer = &rw return &rw.OsmElemWriter @@ -62,7 +68,7 @@ func (rw *RelationWriter) relId(id int64) int64 { } func (rw *RelationWriter) loop() { - geos := geos.NewGeos() + geos := geosp.NewGeos() geos.SetHandleSrid(rw.srid) defer geos.Finish() @@ -76,7 +82,7 @@ NextRel: } continue NextRel } - for _, m := range r.Members { + for i, m := range r.Members { if m.Way == nil { continue } @@ -88,6 +94,76 @@ NextRel: continue NextRel } rw.NodesToSrid(m.Way.Nodes) + r.Members[i].Elem = &m.Way.OSMElem + } + + relMemberMatches := rw.relationMemberMatcher.MatchRelation(r) + if len(relMemberMatches) > 0 { + for i, m := range r.Members { + if m.Type == element.RELATION { + mrel, err := rw.osmCache.Relations.GetRelation(m.Id) + if err != nil { + if err == cache.NotFound { + log.Warn(err) + continue NextRel + } + } + r.Members[i].Elem = &mrel.OSMElem + } else if m.Type == element.NODE { + nd, err := rw.osmCache.Nodes.GetNode(m.Id) + if err != nil { + if err == cache.NotFound { + nd, err = rw.osmCache.Coords.GetCoord(m.Id) + if err != nil { + if err != cache.NotFound { + log.Warn(err) + } + continue NextRel + } + } else { + log.Warn(err) + continue NextRel + } + } + rw.NodeToSrid(nd) + r.Members[i].Node = nd + r.Members[i].Elem = &nd.OSMElem + } + } + + for _, m := range r.Members { + var g *geosp.Geom + var err error + if m.Node != nil { + g, err = geomp.Point(geos, *m.Node) + } else if m.Way != nil { + g, err = geomp.LineString(geos, m.Way.Nodes) + } + + if err != nil { + log.Warn(err) + continue + } + + var gelem geomp.Geometry + if g == nil { + g = geos.FromWkt("POLYGON EMPTY") + gelem = geomp.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} + } else { + gelem, err = geomp.AsGeomElement(geos, g) + if err != nil { + log.Warn(err) + continue + } + } + + rw.inserter.InsertRelationMember(*r, m, gelem, relMemberMatches) + } + } + + relMatches := rw.relationMatcher.MatchRelation(r) + if len(relMatches) > 0 { + rw.inserter.InsertPolygon(r.OSMElem, geomp.Geometry{}, relMatches) } // BuildRelation updates r.Members but we need all of them From e030ffcf45cf45a5e5a3fb4f3db5138e57832468 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Jan 2016 12:08:14 +0100 Subject: [PATCH 26/36] remove debug log.print --- mapping/matcher.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mapping/matcher.go b/mapping/matcher.go index ffab4ad..d5675bb 100644 --- a/mapping/matcher.go +++ b/mapping/matcher.go @@ -1,8 +1,6 @@ package mapping import ( - _ "log" - "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" ) @@ -32,7 +30,6 @@ func (m *Mapping) RelationMatcher() RelationMatcher { mappings := make(TagTables) m.mappings(RelationTable, mappings) filters := m.ElementFilters() - log.Print(mappings) return &tagMatcher{mappings, m.tables(RelationTable), filters, true} } @@ -40,7 +37,6 @@ func (m *Mapping) RelationMemberMatcher() RelationMatcher { mappings := make(TagTables) m.mappings(RelationMemberTable, mappings) filters := m.ElementFilters() - log.Print(mappings) return &tagMatcher{mappings, m.tables(RelationMemberTable), filters, true} } From 44230b50f2c0e93985f600c3bb1cc270014cde66 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Jan 2016 12:10:34 +0100 Subject: [PATCH 27/36] convert route mapping to yaml --- test/route_relation_mapping.json | 128 ------------------------------- test/route_relation_mapping.yml | 67 ++++++++++++++++ test/route_relation_test.go | 2 +- 3 files changed, 68 insertions(+), 129 deletions(-) delete mode 100644 test/route_relation_mapping.json create mode 100644 test/route_relation_mapping.yml diff --git a/test/route_relation_mapping.json b/test/route_relation_mapping.json deleted file mode 100644 index c81fc92..0000000 --- a/test/route_relation_mapping.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "tags": { - "load_all": true, - "exclude": [ - "created_by", - "source" - ] - }, - "tables": { - "route_members": { - "fields": [ - { - "type": "id", - "name": "osm_id" - }, - { - "type": "string", - "name": "ref", - "key": "ref" - }, - { - "type": "relation_member_id", - "name": "member" - }, - { - "type": "relation_member_index", - "name": "index" - }, - { - "type": "relation_member_role", - "name": "role" - }, - { - "type": "relation_member_type", - "name": "type" - }, - { - "type": "geometry", - "name": "geometry" - }, - { - "type": "string", - "name": "name", - "key": "name", - "from_members": true - } - ], - "type": "relation_member", - "mapping": { - "route": [ - "bus", - "tram", - "rail" - ] - } - }, - "routes": { - "fields": [ - { - "type": "id", - "name": "osm_id" - }, - { - "type": "string", - "name": "ref", - "key": "ref" - }, - { - "type": "hstore_tags", - "name": "tags" - } - ], - "type": "relation", - "mapping": { - "route": [ - "bus", - "tram", - "rail" - ] - } - }, - "master_routes": { - "fields": [ - { - "type": "id", - "name": "osm_id" - }, - { - "type": "relation_member_id", - "name": "member" - }, - { - "type": "relation_member_index", - "name": "index" - }, - { - "type": "relation_member_role", - "name": "role" - }, - { - "type": "relation_member_type", - "name": "type" - }, - { - "type": "geometry", - "name": "geometry" - }, - { - "type": "string", - "name": "subname", - "key": "name", - "from_members": true - }, - { - "type": "string", - "name": "name", - "key": "name" - } - ], - "type": "relation_member", - "mapping": { - "route_master": [ - "bus" - ] - } - } - } -} diff --git a/test/route_relation_mapping.yml b/test/route_relation_mapping.yml new file mode 100644 index 0000000..5a09795 --- /dev/null +++ b/test/route_relation_mapping.yml @@ -0,0 +1,67 @@ +tags: + load_all: true + exclude: + - created_by + - source + +tables: + master_routes: + type: relation_member + fields: + - name: osm_id + type: id + - name: member + type: relation_member_id + - name: index + type: relation_member_index + - name: role + type: relation_member_role + - name: type + type: relation_member_type + - name: geometry + type: geometry + - name: subname + key: name + type: string + from_members: true + - key: name + name: name + type: string + mapping: + route_master: [bus] + route_members: + type: relation_member + fields: + - name: osm_id + type: id + - key: ref + name: ref + type: string + - name: member + type: relation_member_id + - name: index + type: relation_member_index + - name: role + type: relation_member_role + - name: type + type: relation_member_type + - name: geometry + type: geometry + - name: name + key: name + type: string + from_members: true + mapping: + route: [bus, tram, rail] + routes: + type: relation + fields: + - name: osm_id + type: id + - key: ref + name: ref + type: string + - name: tags + type: hstore_tags + mapping: + route: [bus, tram, rail] diff --git a/test/route_relation_test.go b/test/route_relation_test.go index f71c0af..16b52a0 100644 --- a/test/route_relation_test.go +++ b/test/route_relation_test.go @@ -14,7 +14,7 @@ func TestRouteRelation_Prepare(t *testing.T) { connection: "postgis://", cacheDir: ts.dir, osmFileName: "build/route_relation.pbf", - mappingFileName: "route_relation_mapping.json", + mappingFileName: "route_relation_mapping.yml", } ts.g = geos.NewGeos() From c419d9b2c22e667ad7c50df81f91cdb244c6dc5e Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Jan 2016 12:58:49 +0100 Subject: [PATCH 28/36] fix test for extended elemen.Member --- cache/binary/serialize_test.go | 7 ++-- geom/multipolygon_test.go | 66 +++++++++++++++++----------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/cache/binary/serialize_test.go b/cache/binary/serialize_test.go index e7d567c..9bbf4e7 100644 --- a/cache/binary/serialize_test.go +++ b/cache/binary/serialize_test.go @@ -1,8 +1,9 @@ package binary import ( - "github.com/omniscale/imposm3/element" "testing" + + "github.com/omniscale/imposm3/element" ) func compareRefs(a []int64, b []int64) bool { @@ -74,8 +75,8 @@ func TestMarshalRelation(t *testing.T) { rel.Tags = make(element.Tags) rel.Tags["name"] = "test" rel.Tags["landusage"] = "forest" - rel.Members = append(rel.Members, element.Member{123, element.WAY, "outer", nil}) - rel.Members = append(rel.Members, element.Member{124, element.WAY, "inner", nil}) + rel.Members = append(rel.Members, element.Member{Id: 123, Type: element.WAY, Role: "outer"}) + rel.Members = append(rel.Members, element.Member{Id: 124, Type: element.WAY, Role: "inner"}) data, _ := MarshalRelation(rel) rel, _ = UnmarshalRelation(data) diff --git a/geom/multipolygon_test.go b/geom/multipolygon_test.go index bdc6cad..dc31b1c 100644 --- a/geom/multipolygon_test.go +++ b/geom/multipolygon_test.go @@ -53,8 +53,8 @@ func TestSimplePolygonWithHole(t *testing.T) { rel := element.Relation{ OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom, err := buildRelation(&rel, 3857) @@ -96,8 +96,8 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) { rel := element.Relation{ OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "rel"}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom, err := buildRelation(&rel, 3857) @@ -149,9 +149,9 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) { rel := element.Relation{ OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, - {3, element.WAY, "inner", &w3}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, + {Id: 3, Type: element.WAY, Role: "inner", Way: &w3}, } geom, err := buildRelation(&rel, 3857) @@ -216,11 +216,11 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, - {3, element.WAY, "inner", &w3}, - {4, element.WAY, "inner", &w4}, - {5, element.WAY, "inner", &w5}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, + {Id: 3, Type: element.WAY, Role: "inner", Way: &w3}, + {Id: 4, Type: element.WAY, Role: "inner", Way: &w4}, + {Id: 5, Type: element.WAY, Role: "inner", Way: &w5}, } geom, err := buildRelation(&rel, 3857) @@ -263,9 +263,9 @@ func TestPolygonFromThreeWays(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, - {3, element.WAY, "inner", &w3}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, + {Id: 3, Type: element.WAY, Role: "inner", Way: &w3}, } geom, err := buildRelation(&rel, 3857) @@ -316,9 +316,9 @@ func TestTouchingPolygonsWithHole(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"water": "riverbank"}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "outer", &w2}, - {3, element.WAY, "inner", &w3}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "outer", Way: &w2}, + {Id: 3, Type: element.WAY, Role: "inner", Way: &w3}, } geom, err := buildRelation(&rel, 3857) if err != nil { @@ -358,8 +358,8 @@ func TestInsertedWaysDifferentTags(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom, err := buildRelation(&rel, 3857) @@ -400,8 +400,8 @@ func TestInsertMultipleTags(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, // also highway=secondary - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, // also highway=secondary + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom, err := buildRelation(&rel, 3857) @@ -449,8 +449,8 @@ func TestBrokenPolygonSelfIntersect(t *testing.T) { rel1 := element.Relation{OSMElem: element.OSMElem{Id: 1}} rel1.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom1, err := buildRelation(&rel1, 3857) @@ -490,8 +490,8 @@ func TestBrokenPolygonSelfIntersect(t *testing.T) { rel2 := element.Relation{OSMElem: element.OSMElem{Id: 1}} rel2.Members = []element.Member{ - {1, element.WAY, "outer", &w3}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w3}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom2, err := buildRelation(&rel2, 3857) @@ -539,8 +539,8 @@ func TestBrokenPolygonSelfIntersectTriangle(t *testing.T) { rel := element.Relation{OSMElem: element.OSMElem{Id: 1}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "inner", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, } geom, err := buildRelation(&rel, 3857) @@ -579,8 +579,8 @@ func TestBrokenPolygonSelfIntersectTriangle(t *testing.T) { rel = element.Relation{OSMElem: element.OSMElem{Id: 1}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w3}, - {2, element.WAY, "inner", &w4}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w3}, + {Id: 2, Type: element.WAY, Role: "inner", Way: &w4}, } geom, err = buildRelation(&rel, 3857) @@ -610,7 +610,7 @@ func TestOpenRing(t *testing.T) { rel := element.Relation{ OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, } _, err := buildRelation(&rel, 3857) @@ -634,8 +634,8 @@ func TestClosedAndOpenRing(t *testing.T) { rel := element.Relation{ OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{}}} rel.Members = []element.Member{ - {1, element.WAY, "outer", &w1}, - {2, element.WAY, "outer", &w2}, + {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, + {Id: 2, Type: element.WAY, Role: "outer", Way: &w2}, } prep, err := PrepareRelation(&rel, 3857, 0.1) From c7705bbccf8bc171d46f8258ff87ed64ab3e5486 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Jan 2016 16:32:48 +0100 Subject: [PATCH 29/36] remove unsused test --- test/route_relation_test.py | 83 ------------------------------------- 1 file changed, 83 deletions(-) delete mode 100644 test/route_relation_test.py diff --git a/test/route_relation_test.py b/test/route_relation_test.py deleted file mode 100644 index eb20330..0000000 --- a/test/route_relation_test.py +++ /dev/null @@ -1,83 +0,0 @@ -import psycopg2 -import psycopg2.extras - -import helper as t - -psycopg2.extras.register_hstore(psycopg2.connect(**t.db_conf), globally=True) - -mapping_file = 'route_relation_mapping.json' - -def setup(): - t.setup() - -def teardown(): - t.teardown() - -RELOFFSET = int(-1e17) - -####################################################################### -def test_import(): - """Import succeeds""" - t.drop_schemas() - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - t.imposm3_import(t.db_conf, './build/route_relation.pbf', mapping_file) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - -def test_deploy(): - """Deploy succeeds""" - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - t.imposm3_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - -####################################################################### - - -####################################################################### - -def test_update(): - """Diff import applies""" - t.imposm3_update(t.db_conf, './build/route_relation.osc.gz', mapping_file) - -####################################################################### - - -####################################################################### -def test_deploy_and_revert_deploy(): - """Revert deploy succeeds""" - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - - # import again to have a new import schema - t.imposm3_import(t.db_conf, './build/route_relation.pbf', mapping_file) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - - t.imposm3_deploy(t.db_conf, mapping_file) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_revert_deploy(t.db_conf, mapping_file) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - -def test_remove_backup(): - """Remove backup succeeds""" - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_deploy(t.db_conf, mapping_file) - - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - - t.imposm3_remove_backups(t.db_conf, mapping_file) - - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_IMPORT) - assert t.table_exists('osm_routes', schema=t.TEST_SCHEMA_PRODUCTION) - assert not t.table_exists('osm_routes', schema=t.TEST_SCHEMA_BACKUP) - From 7a7314bfb5226a6f70b5e21f490ff855f2fc8a6c Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 5 Jan 2016 09:40:30 +0100 Subject: [PATCH 30/36] rename option to from_member --- mapping/config.go | 12 ++++++------ mapping/fields.go | 18 +++++++++--------- test/route_relation_mapping.yml | 7 +++++-- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/mapping/config.go b/mapping/config.go index 430510c..0c84d8b 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -11,12 +11,12 @@ import ( ) type Field struct { - Name string `yaml:"name"` - Key Key `yaml:"key"` - Keys []Key `yaml:"keys"` - Type string `yaml:"type"` - Args map[string]interface{} `yaml:"args"` - FromMembers bool `yaml:"from_members"` + Name string `yaml:"name"` + Key Key `yaml:"key"` + Keys []Key `yaml:"keys"` + Type string `yaml:"type"` + Args map[string]interface{} `yaml:"args"` + FromMember bool `yaml:"from_member"` } type Table struct { diff --git a/mapping/fields.go b/mapping/fields.go index 15a7900..31e0ba4 100644 --- a/mapping/fields.go +++ b/mapping/fields.go @@ -62,7 +62,7 @@ func (f *FieldSpec) Value(elem *element.OSMElem, geom *geom.Geometry, match Matc func (f *FieldSpec) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} { if f.Type.Func != nil { - if f.Type.FromMembers { + if f.Type.FromMember { if member.Elem == nil { return nil } @@ -104,9 +104,9 @@ func (field *Field) FieldType() *FieldType { log.Print(err) return nil } - fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMembers} + fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMember} } - fieldType.FromMembers = field.FromMembers + fieldType.FromMember = field.FromMember return &fieldType } return nil @@ -131,12 +131,12 @@ func (t *Table) TableFields() *TableFields { } type FieldType struct { - Name string - GoType string - Func MakeValue - MakeFunc MakeMakeValue - MemberFunc MakeMemberValue - FromMembers bool + Name string + GoType string + Func MakeValue + MakeFunc MakeMakeValue + MemberFunc MakeMemberValue + FromMember bool } func Bool(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { diff --git a/test/route_relation_mapping.yml b/test/route_relation_mapping.yml index 5a09795..d9a9773 100644 --- a/test/route_relation_mapping.yml +++ b/test/route_relation_mapping.yml @@ -23,7 +23,7 @@ tables: - name: subname key: name type: string - from_members: true + from_member: true - key: name name: name type: string @@ -47,10 +47,13 @@ tables: type: relation_member_type - name: geometry type: geometry + - name: relname + key: name + type: string - name: name key: name type: string - from_members: true + from_member: true mapping: route: [bus, tram, rail] routes: From b25774d891faf128a219c243f9d7ac8f5ea6a35a Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 6 Jan 2016 08:46:06 +0100 Subject: [PATCH 31/36] fix race condition triggered with new relation_members handling --- cache/delta.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cache/delta.go b/cache/delta.go index 569e3de..f860ce9 100644 --- a/cache/delta.go +++ b/cache/delta.go @@ -28,7 +28,8 @@ func (b *coordsBunch) GetCoord(id int64) (*element.Node, error) { return b.coords[i].Id >= id }) if idx < len(b.coords) && b.coords[idx].Id == id { - return &b.coords[idx], nil + nd := b.coords[idx] // create copy prevent to race when node gets reprojected + return &nd, nil } return nil, NotFound } From 3baf2b482ffc6cced04ab5178a742f9f28a76f5e Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 6 Jan 2016 08:46:44 +0100 Subject: [PATCH 32/36] refactor relation writing --- writer/relations.go | 304 ++++++++++++++++++++++++-------------------- 1 file changed, 165 insertions(+), 139 deletions(-) diff --git a/writer/relations.go b/writer/relations.go index 8351456..9c8b92d 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -80,7 +80,7 @@ NextRel: if err != cache.NotFound { log.Warn(err) } - continue NextRel + continue } for i, m := range r.Members { if m.Way == nil { @@ -97,132 +97,84 @@ NextRel: r.Members[i].Elem = &m.Way.OSMElem } - relMemberMatches := rw.relationMemberMatcher.MatchRelation(r) - if len(relMemberMatches) > 0 { - for i, m := range r.Members { - if m.Type == element.RELATION { - mrel, err := rw.osmCache.Relations.GetRelation(m.Id) - if err != nil { - if err == cache.NotFound { - log.Warn(err) - continue NextRel - } - } - r.Members[i].Elem = &mrel.OSMElem - } else if m.Type == element.NODE { - nd, err := rw.osmCache.Nodes.GetNode(m.Id) - if err != nil { - if err == cache.NotFound { - nd, err = rw.osmCache.Coords.GetCoord(m.Id) - if err != nil { - if err != cache.NotFound { - log.Warn(err) - } - continue NextRel - } - } else { - log.Warn(err) - continue NextRel - } - } - rw.NodeToSrid(nd) - r.Members[i].Node = nd - r.Members[i].Elem = &nd.OSMElem - } - } - - for _, m := range r.Members { - var g *geosp.Geom - var err error - if m.Node != nil { - g, err = geomp.Point(geos, *m.Node) - } else if m.Way != nil { - g, err = geomp.LineString(geos, m.Way.Nodes) - } - - if err != nil { - log.Warn(err) - continue - } - - var gelem geomp.Geometry - if g == nil { - g = geos.FromWkt("POLYGON EMPTY") - gelem = geomp.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} - } else { - gelem, err = geomp.AsGeomElement(geos, g) - if err != nil { - log.Warn(err) - continue - } - } - - rw.inserter.InsertRelationMember(*r, m, gelem, relMemberMatches) - } - } - - relMatches := rw.relationMatcher.MatchRelation(r) - if len(relMatches) > 0 { - rw.inserter.InsertPolygon(r.OSMElem, geomp.Geometry{}, relMatches) - } - - // BuildRelation updates r.Members but we need all of them + // handleRelation updates r.Members but we need all of them // for the diffCache allMembers := r.Members - // prepare relation first (build rings and compute actual - // relation tags) - prepedRel, err := geomp.PrepareRelation(r, rw.srid, rw.maxGap) - if err != nil { - if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { - log.Warn(err) - } - continue NextRel + inserted := false + + if handleRelationMembers(rw, r, geos) { + inserted = true + } + if handleRelation(rw, r, geos) { + inserted = true + } + if handleMultiPolygon(rw, r, geos) { + inserted = true } - // check for matches befor building the geometry - matches := rw.polygonMatcher.MatchRelation(r) - if len(matches) == 0 { - continue NextRel - } - - // build the multipolygon - geom, err := prepedRel.Build() - if err != nil { - if geom.Geom != nil { - geos.Destroy(geom.Geom) - } - if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { - log.Warn(err) - } - continue NextRel - } - - if rw.limiter != nil { - start := time.Now() - parts, err := rw.limiter.Clip(geom.Geom) - if err != nil { - log.Warn(err) - continue NextRel - } - if duration := time.Now().Sub(start); duration > time.Minute { - log.Warnf("clipping relation %d to -limitto took %s", r.Id, duration) - } - for _, g := range parts { - rel := element.Relation(*r) - rel.Id = rw.relId(r.Id) - geom = geomp.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} - err := rw.inserter.InsertPolygon(rel.OSMElem, geom, matches) - if err != nil { - if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { - log.Warn(err) - } - continue + if inserted && rw.diffCache != nil { + rw.diffCache.Ways.AddFromMembers(r.Id, allMembers) + for _, member := range allMembers { + if member.Way != nil { + rw.diffCache.Coords.AddFromWay(member.Way) } } - } else { + } + if inserted && rw.expireor != nil { + for _, m := range allMembers { + if m.Way != nil { + expire.ExpireNodes(rw.expireor, m.Way.Nodes) + } + } + } + } + rw.wg.Done() +} + +func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool { + // prepare relation first (build rings and compute actual + // relation tags) + prepedRel, err := geomp.PrepareRelation(r, rw.srid, rw.maxGap) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + return false + } + + // check for matches befor building the geometry + matches := rw.polygonMatcher.MatchRelation(r) + if matches == nil { + return false + } + + // build the multipolygon + geom, err := prepedRel.Build() + if geom.Geom != nil { + defer geos.Destroy(geom.Geom) + } + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + return false + } + + if rw.limiter != nil { + start := time.Now() + parts, err := rw.limiter.Clip(geom.Geom) + if err != nil { + log.Warn(err) + return false + } + if duration := time.Now().Sub(start); duration > time.Minute { + log.Warnf("clipping relation %d to -limitto took %s", r.Id, duration) + } + for _, g := range parts { rel := element.Relation(*r) rel.Id = rw.relId(r.Id) + geom = geomp.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} err := rw.inserter.InsertPolygon(rel.OSMElem, geom, matches) if err != nil { if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { @@ -231,29 +183,103 @@ NextRel: continue } } - - for _, m := range mapping.SelectRelationPolygons(rw.polygonMatcher, r) { - err = rw.osmCache.InsertedWays.PutWay(m.Way) - if err != nil { + } else { + rel := element.Relation(*r) + rel.Id = rw.relId(r.Id) + err := rw.inserter.InsertPolygon(rel.OSMElem, geom, matches) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { log.Warn(err) } + return false } - if rw.diffCache != nil { - rw.diffCache.Ways.AddFromMembers(r.Id, allMembers) - for _, member := range allMembers { - if member.Way != nil { - rw.diffCache.Coords.AddFromWay(member.Way) - } - } - } - if rw.expireor != nil { - for _, m := range allMembers { - if m.Way != nil { - expire.ExpireNodes(rw.expireor, m.Way.Nodes) - } - } - } - geos.Destroy(geom.Geom) } - rw.wg.Done() + + for _, m := range mapping.SelectRelationPolygons(rw.polygonMatcher, r) { + err = rw.osmCache.InsertedWays.PutWay(m.Way) + if err != nil { + log.Warn(err) + } + } + return true +} + +func handleRelation(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool { + relMatches := rw.relationMatcher.MatchRelation(r) + if relMatches == nil { + return false + } + rel := element.Relation(*r) + rel.Id = rw.relId(r.Id) + rw.inserter.InsertPolygon(rel.OSMElem, geomp.Geometry{}, relMatches) + return true +} + +func handleRelationMembers(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool { + relMemberMatches := rw.relationMemberMatcher.MatchRelation(r) + if relMemberMatches == nil { + return false + } + for i, m := range r.Members { + if m.Type == element.RELATION { + mrel, err := rw.osmCache.Relations.GetRelation(m.Id) + if err != nil { + if err == cache.NotFound { + log.Warn(err) + return false + } + } + r.Members[i].Elem = &mrel.OSMElem + } else if m.Type == element.NODE { + nd, err := rw.osmCache.Nodes.GetNode(m.Id) + if err != nil { + if err == cache.NotFound { + nd, err = rw.osmCache.Coords.GetCoord(m.Id) + if err != nil { + if err != cache.NotFound { + log.Warn(err) + } + return false + } + } else { + log.Warn(err) + return false + } + } + rw.NodeToSrid(nd) + r.Members[i].Node = nd + r.Members[i].Elem = &nd.OSMElem + } + } + + for _, m := range r.Members { + var g *geosp.Geom + var err error + if m.Node != nil { + g, err = geomp.Point(geos, *m.Node) + } else if m.Way != nil { + g, err = geomp.LineString(geos, m.Way.Nodes) + } + + if err != nil { + log.Warn(err) + return false + } + + var gelem geomp.Geometry + if g == nil { + g = geos.FromWkt("POLYGON EMPTY") + gelem = geomp.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} + } else { + gelem, err = geomp.AsGeomElement(geos, g) + if err != nil { + log.Warn(err) + return false + } + } + rel := element.Relation(*r) + rel.Id = rw.relId(r.Id) + rw.inserter.InsertRelationMember(rel, m, gelem, relMemberMatches) + } + return true } From 8eae74a6daf596a12fa5e5ad2797924b9b91edeb Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 6 Jan 2016 09:07:14 +0100 Subject: [PATCH 33/36] diff update relation_members --- database/postgis/postgis.go | 6 ++-- diff/deleter.go | 18 ++++------- test/helper_test.go | 27 ++++++++++++++-- test/route_relation.osc | 9 ++++++ test/route_relation.osm | 18 +++++++++++ test/route_relation_test.go | 61 +++++++++++++++++++++++++++++++++++++ 6 files changed, 122 insertions(+), 17 deletions(-) diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 8c710ec..7b28681 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -512,10 +512,10 @@ func (pg *PostGIS) DeleteElem(elem element.OSMElem) error { // handle deletes of geometries that did not match in ProbeXxx. // we have to handle multipolygon relations that took the tags of the // main-member. those tags are not avail. during delete. just try to - // delete from each polygon table. - if v, ok := elem.Tags["type"]; ok && (v == "multipolygon" || v == "boundary") { + // delete from each polygon/relation table. + if _, ok := elem.Tags["type"]; ok { for _, tableSpec := range pg.Tables { - if tableSpec.GeometryType != "polygon" { + if tableSpec.GeometryType != "polygon" && tableSpec.GeometryType != "geometry" && tableSpec.GeometryType != "relation" { continue } pg.txRouter.Delete(tableSpec.Name, elem.Id) diff --git a/diff/deleter.go b/diff/deleter.go index c872f68..99482c3 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -83,18 +83,12 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) if elem.Tags == nil { return nil } - if matches := d.tmPolygons.MatchRelation(elem); len(matches) > 0 { - if err := d.delDb.Delete(d.RelId(elem.Id), matches); err != nil { - return err - } - } else { - // handle relations with tags from members by deleting - // from all tables - e := element.OSMElem(elem.OSMElem) - e.Id = -e.Id - if err := d.delDb.DeleteElem(e); err != nil { - return err - } + // delete from all tables to handle relations with tags from members + // and relation_members + e := element.OSMElem(elem.OSMElem) + e.Id = -e.Id + if err := d.delDb.DeleteElem(e); err != nil { + return err } if deleteRefs { diff --git a/test/helper_test.go b/test/helper_test.go index d3a43f9..c29466e 100644 --- a/test/helper_test.go +++ b/test/helper_test.go @@ -237,11 +237,11 @@ func (s *importTestSuite) query(t *testing.T, table string, id int64, keys []str } func (s *importTestSuite) queryTags(t *testing.T, table string, id int64) record { - stmt := fmt.Sprintf(`SELECT osm_id, ST_AsText(geometry), tags FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) + stmt := fmt.Sprintf(`SELECT osm_id, tags FROM "%s"."%s" WHERE osm_id=$1`, dbschemaProduction, table) row := s.db.QueryRow(stmt, id) r := record{} h := hstore.Hstore{} - if err := row.Scan(&r.id, &r.wkt, &h); err != nil { + if err := row.Scan(&r.id, &h); err != nil { if err == sql.ErrNoRows { r.missing = true } else { @@ -320,6 +320,29 @@ func (s *importTestSuite) queryGeom(t *testing.T, table string, id int64) *geos. return geom } +func (s *importTestSuite) queryDynamic(t *testing.T, table, where string) []map[string]string { + stmt := fmt.Sprintf(`SELECT hstore(r) FROM (SELECT ST_AsText(geometry) AS wkt, * FROM "%s"."%s" WHERE %s) AS r`, dbschemaProduction, table, where) + rows, err := s.db.Query(stmt) + if err != nil { + t.Fatal(err) + } + results := []map[string]string{} + for rows.Next() { + h := hstore.Hstore{} + if err := rows.Scan(&h); err != nil { + t.Fatal(err) + } + r := make(map[string]string) + for k, v := range h.Map { + if v.Valid { + r[k] = v.String + } + } + results = append(results, r) + } + return results +} + type checkElem struct { table string id int64 diff --git a/test/route_relation.osc b/test/route_relation.osc index 7f21e40..ff6ad12 100644 --- a/test/route_relation.osc +++ b/test/route_relation.osc @@ -1,3 +1,12 @@ + + + + + + + + + diff --git a/test/route_relation.osm b/test/route_relation.osm index b1d5682..25c42cc 100644 --- a/test/route_relation.osm +++ b/test/route_relation.osm @@ -99,7 +99,25 @@ + + + + + + + + + + + + + + + + + + diff --git a/test/route_relation_test.go b/test/route_relation_test.go index 16b52a0..a03facc 100644 --- a/test/route_relation_test.go +++ b/test/route_relation_test.go @@ -2,6 +2,7 @@ package test import ( "database/sql" + "strconv" "testing" @@ -46,8 +47,68 @@ func TestRouteRelation_Deploy(t *testing.T) { } } +func TestRouteRelation_RelationData(t *testing.T) { + // check tags of relation + r := ts.queryTags(t, "osm_routes", -100901) + if r.tags["name"] != "Bus 301: A => B" { + t.Error(r) + } +} + +func TestRouteRelation_MemberGeomUpdated1(t *testing.T) { + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100502") + if len(rows) != 1 { + t.Fatal(rows) + } + g := ts.g.FromWkt(rows[0]["wkt"]) + if g.Length() != 111.32448543701321 { + t.Fatal(g.Length()) + } + + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100503") + if len(rows) != 1 { + t.Fatal(rows) + } + if rows[0]["name"] != "" { + t.Error(rows[0]) + } +} + // ####################################################################### func TestRouteRelation_Update(t *testing.T) { ts.updateOsm(t, "./build/route_relation.osc.gz") } + +// ####################################################################### + +func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100502") + if len(rows) != 1 { + t.Fatal(rows) + } + g := ts.g.FromWkt(rows[0]["wkt"]) + if g.Length() != 184.97560221624542 { + t.Fatal(g.Length()) + } + + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100503") + if len(rows) != 1 { + t.Fatal(rows) + } + if rows[0]["name"] != "new name" { + t.Error(rows[0]) + } +} + +func TestRouteRelation_MemberNotUpdated(t *testing.T) { + // check that member is not updated if no node/way changed + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -100903 AND member = 100501") + if len(rows) != 1 { + t.Fatal(rows) + } + if id, err := strconv.ParseInt(rows[0]["id"], 10, 32); err != nil || id > 27 { + t.Error("member was re-inserted", rows) + } + +} From 766118b324ae230d9f19dd1ff353a715e5d3e993 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 8 Jan 2016 16:08:00 +0100 Subject: [PATCH 34/36] update relations when member node changed --- cache/diff.go | 53 +++++++++++++++++++++++++++++++++---- diff/deleter.go | 9 +++++++ diff/process.go | 24 ++++++++++------- test/complete_db.osc | 7 +++++ test/complete_db.osm | 20 ++++++++++++++ test/completedb_test.go | 12 +++++++++ test/route_relation.osc | 32 ++++++++++++++++++++++ test/route_relation.osm | 12 +++++++++ test/route_relation_test.go | 29 +++++++++++++++----- writer/relations.go | 1 + 10 files changed, 178 insertions(+), 21 deletions(-) diff --git a/cache/diff.go b/cache/diff.go index 0060c85..128e260 100644 --- a/cache/diff.go +++ b/cache/diff.go @@ -1,7 +1,6 @@ package cache import ( - "github.com/jmhodges/levigo" "log" "os" "path/filepath" @@ -9,6 +8,8 @@ import ( "sort" "sync" + "github.com/jmhodges/levigo" + "github.com/omniscale/imposm3/cache/binary" "github.com/omniscale/imposm3/element" ) @@ -20,10 +21,11 @@ func (a byInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byInt64) Less(i, j int) bool { return a[i] < a[j] } type DiffCache struct { - Dir string - Coords *CoordsRefIndex - Ways *WaysRefIndex - opened bool + Dir string + Coords *CoordsRefIndex // Stores which ways a coord references + CoordsRel *CoordsRelRefIndex // Stores which relations a coord references + Ways *WaysRefIndex // Stores which relations a way references + opened bool } func NewDiffCache(dir string) *DiffCache { @@ -36,6 +38,10 @@ func (c *DiffCache) Close() { c.Coords.Close() c.Coords = nil } + if c.CoordsRel != nil { + c.CoordsRel.Close() + c.CoordsRel = nil + } if c.Ways != nil { c.Ways.Close() c.Ways = nil @@ -46,6 +52,9 @@ func (c *DiffCache) Flush() { if c.Coords != nil { c.Coords.Flush() } + if c.CoordsRel != nil { + c.CoordsRel.Flush() + } if c.Ways != nil { c.Ways.Flush() } @@ -58,6 +67,11 @@ func (c *DiffCache) Open() error { c.Close() return err } + c.CoordsRel, err = newCoordsRelRefIndex(filepath.Join(c.Dir, "coords_rel_index")) + if err != nil { + c.Close() + return err + } c.Ways, err = newWaysRefIndex(filepath.Join(c.Dir, "ways_index")) if err != nil { c.Close() @@ -74,6 +88,9 @@ func (c *DiffCache) Exists() bool { if _, err := os.Stat(filepath.Join(c.Dir, "coords_index")); !os.IsNotExist(err) { return true } + if _, err := os.Stat(filepath.Join(c.Dir, "coords_rel_index")); !os.IsNotExist(err) { + return true + } if _, err := os.Stat(filepath.Join(c.Dir, "ways_index")); !os.IsNotExist(err) { return true } @@ -87,6 +104,9 @@ func (c *DiffCache) Remove() error { if err := os.RemoveAll(filepath.Join(c.Dir, "coords_index")); err != nil { return err } + if err := os.RemoveAll(filepath.Join(c.Dir, "coords_rel_index")); err != nil { + return err + } if err := os.RemoveAll(filepath.Join(c.Dir, "ways_index")); err != nil { return err } @@ -198,6 +218,9 @@ func newRefIndex(path string, opts *cacheOptions) (*bunchRefCache, error) { type CoordsRefIndex struct { bunchRefCache } +type CoordsRelRefIndex struct { + bunchRefCache +} type WaysRefIndex struct { bunchRefCache } @@ -210,6 +233,14 @@ func newCoordsRefIndex(dir string) (*CoordsRefIndex, error) { return &CoordsRefIndex{*cache}, nil } +func newCoordsRelRefIndex(dir string) (*CoordsRelRefIndex, error) { + cache, err := newRefIndex(dir, &globalCacheOptions.CoordsIndex) + if err != nil { + return nil, err + } + return &CoordsRelRefIndex{*cache}, nil +} + func newWaysRefIndex(dir string) (*WaysRefIndex, error) { cache, err := newRefIndex(dir, &globalCacheOptions.WaysIndex) if err != nil { @@ -365,6 +396,18 @@ func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) { } } +func (index *CoordsRelRefIndex) AddFromMembers(relId int64, members []element.Member) { + for _, member := range members { + if member.Type == element.NODE { + if index.linearImport { + index.addc <- idRef{id: member.Id, ref: relId} + } else { + index.Add(member.Id, relId) + } + } + } +} + func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) { for _, member := range members { if member.Type == element.WAY { diff --git a/diff/deleter.go b/diff/deleter.go index 99482c3..bf60f1e 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -258,6 +258,15 @@ func (d *Deleter) Delete(delElem parser.DiffElem) error { } } } + dependers = d.diffCache.CoordsRel.Get(delElem.Node.Id) + for _, rel := range dependers { + if _, ok := d.deletedRelations[rel]; ok { + continue + } + if err := d.deleteRelation(rel, false, false); err != nil { + return err + } + } } if !delElem.Add { if err := d.diffCache.Coords.Delete(delElem.Node.Id); err != nil { diff --git a/diff/process.go b/diff/process.go index c15da9a..6302756 100644 --- a/diff/process.go +++ b/diff/process.go @@ -175,9 +175,9 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi nodeWriter.SetExpireor(expireor) nodeWriter.Start() - nodeIds := make(map[int64]bool) - wayIds := make(map[int64]bool) - relIds := make(map[int64]bool) + nodeIds := make(map[int64]struct{}) + wayIds := make(map[int64]struct{}) + relIds := make(map[int64]struct{}) step := log.StartStep("Parsing changes, updating cache and removing elements") @@ -251,7 +251,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put relation %v", elem.Rel) } - relIds[elem.Rel.Id] = true + relIds[elem.Rel.Id] = struct{}{} } } else if elem.Way != nil { // check if first coord is cached to avoid caching @@ -265,7 +265,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put way %v", elem.Way) } - wayIds[elem.Way.Id] = true + wayIds[elem.Way.Id] = struct{}{} } } else if elem.Node != nil { addNode := true @@ -283,7 +283,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put coord %v", elem.Node) } - nodeIds[elem.Node.Id] = true + nodeIds[elem.Node.Id] = struct{}{} } } } @@ -301,7 +301,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi // mark member ways from deleted relations for re-insert for id, _ := range deleter.DeletedMemberWays() { - wayIds[id] = true + wayIds[id] = struct{}{} } progress.Stop() @@ -314,16 +314,22 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi for nodeId, _ := range nodeIds { dependers := diffCache.Coords.Get(nodeId) for _, way := range dependers { - wayIds[way] = true + wayIds[way] = struct{}{} } } // mark depending relations for (re)insert + for nodeId, _ := range nodeIds { + dependers := diffCache.CoordsRel.Get(nodeId) + for _, rel := range dependers { + relIds[rel] = struct{}{} + } + } for wayId, _ := range wayIds { dependers := diffCache.Ways.Get(wayId) // mark depending relations for (re)insert for _, rel := range dependers { - relIds[rel] = true + relIds[rel] = struct{}{} } } diff --git a/test/complete_db.osc b/test/complete_db.osc index b3c8c1c..3999726 100644 --- a/test/complete_db.osc +++ b/test/complete_db.osc @@ -225,4 +225,11 @@ + + + + + + + diff --git a/test/complete_db.osm b/test/complete_db.osm index a6b7b8c..5d4e2cd 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -1019,6 +1019,26 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test/completedb_test.go b/test/completedb_test.go index e28aded..6b6b794 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -297,6 +297,12 @@ func TestDuplicateIds(t *testing.T) { }) } +func TestRelationUpdatedByNode(t *testing.T) { + // Relations was updated after modified node. + + assertArea(t, checkElem{"osm_buildings", -52121, "yes", nil}, 13653930440.868315) +} + func TestGeneralizedBananaPolygonIsValid(t *testing.T) { // Generalized polygons are valid. @@ -605,6 +611,12 @@ func TestDuplicateIds2(t *testing.T) { }) } +func TestRelationUpdatedByNode2(t *testing.T) { + // Relations was updated after modified node. + + assertArea(t, checkElem{"osm_buildings", -52121, "yes", nil}, 16276875196.653734) +} + func TestUpdatedWay2(t *testing.T) { // All nodes of straightened way are updated. diff --git a/test/route_relation.osc b/test/route_relation.osc index ff6ad12..66d3456 100644 --- a/test/route_relation.osc +++ b/test/route_relation.osc @@ -9,4 +9,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/route_relation.osm b/test/route_relation.osm index 25c42cc..608a5af 100644 --- a/test/route_relation.osm +++ b/test/route_relation.osm @@ -128,4 +128,16 @@ + + + + + + + + + + + + diff --git a/test/route_relation_test.go b/test/route_relation_test.go index a03facc..89b796c 100644 --- a/test/route_relation_test.go +++ b/test/route_relation_test.go @@ -2,7 +2,6 @@ package test import ( "database/sql" - "strconv" "testing" @@ -92,6 +91,7 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { t.Fatal(g.Length()) } + // tag from member is updated rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100503") if len(rows) != 1 { t.Fatal(rows) @@ -99,16 +99,31 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { if rows[0]["name"] != "new name" { t.Error(rows[0]) } -} -func TestRouteRelation_MemberNotUpdated(t *testing.T) { - // check that member is not updated if no node/way changed - rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -100903 AND member = 100501") + // member is removed + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100512") + if len(rows) != 0 { + t.Fatal(rows) + } + + // role from member is updated + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100102") if len(rows) != 1 { t.Fatal(rows) } - if id, err := strconv.ParseInt(rows[0]["id"], 10, 32); err != nil || id > 27 { - t.Error("member was re-inserted", rows) + if rows[0]["role"] != "halt" { + t.Error(rows[0]) } } + +func TestRouteRelation_MemberUpdatedByNode(t *testing.T) { + // check that member is updated after node was modified + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101") + if len(rows) != 1 { + t.Fatal(rows) + } + if rows[0]["name"] != "Stop2" { + t.Error(rows[0]) + } +} diff --git a/writer/relations.go b/writer/relations.go index 9c8b92d..1925c87 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -115,6 +115,7 @@ NextRel: if inserted && rw.diffCache != nil { rw.diffCache.Ways.AddFromMembers(r.Id, allMembers) + rw.diffCache.CoordsRel.AddFromMembers(r.Id, allMembers) for _, member := range allMembers { if member.Way != nil { rw.diffCache.Coords.AddFromWay(member.Way) From d17e296815038aca6488a1082dfd795e0d474ba1 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 15 Jan 2016 14:41:25 +0100 Subject: [PATCH 35/36] document relation and relation_member --- README.md | 8 +- docs/index.rst | 1 + docs/mapping.rst | 37 +++++++++- docs/relations.rst | 180 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 docs/relations.rst diff --git a/README.md b/README.md index 9168292..b65566f 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,13 @@ Features Automatically creates tables with lower spatial resolutions, perfect for rendering large road networks in low resolutions. - Limit to polygons: - Limit imported geometries to polygons from Shapefiles or GeoJSON, for city/state/country imports. + Limit imported geometries to polygons from GeoJSON, for city/state/country imports. - Easy deployment: - Single binary with only runtime dependencies to common libs (GEOS, SQLite and LevelDB) + Single binary with only runtime dependencies to common libs (GEOS, ProtoBuf and LevelDB) + +- Route relations: + Import all relation types including routes. - Support for table namespace (PostgreSQL schema) @@ -88,7 +91,6 @@ There are a few features we like to see in Imposm 3: * Automatic download and import of differential files * Support for other projections than EPSG:3857 or EPSG:4326 -* Support for route relations * Improved integration with tile servers (expiration of updated tiles) * Custom field/filter functions * Official releases with binaries for more platforms diff --git a/docs/index.rst b/docs/index.rst index 9859b48..7b950b9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -58,6 +58,7 @@ Contents install tutorial mapping + relations .. Indices and tables diff --git a/docs/mapping.rst b/docs/mapping.rst index b1715b6..449152c 100644 --- a/docs/mapping.rst +++ b/docs/mapping.rst @@ -15,7 +15,7 @@ The most important part is the ``tables`` definition. Each table is a YAML objec ``type`` ~~~~~~~~ -``type`` can be ``point``, ``linestring``, ``polygon`` or ``geometry``. ``geometry`` requires a special ``mapping``. +``type`` can be ``point``, ``linestring``, ``polygon``, ``geometry``, ``relation`` and ``relation_member``. ``geometry`` requires a special ``mapping``. :doc:`Relations are described in more detail here `. ``mapping`` @@ -41,7 +41,7 @@ To import all polygons with `tourism=zoo`, `natural=wood` or `natural=land` into ``columns`` ~~~~~~~~~~~ -``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionaly ``key`` and ``args``. +``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionaly ``key``, ``args`` and ``from_member``. ``name`` ^^^^^^^^^ @@ -67,6 +67,11 @@ See :ref:`column_types` for documentation of all types. Some column types require additional arguments. Refer to the documentation of the type. +``from_member`` +^^^^^^^^^^^^^^^ + +``from_member`` is only valid for tables of the type ``relation_member``. If this is set to ``true``, then tags will be used from the member instead of the relation. + Example ~~~~~~~ @@ -241,6 +246,34 @@ Stores all tags in a HStore column. Requires the PostGIS HStore extension. This .. "string_suffixreplace": {"string_suffixreplace", "string", nil, MakeSuffixReplace}, +Element types for ``relation_member`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following types are only valid for tables of the type ``relation_member``. + +``relation_member_id`` +^^^^^^^^^^^^^^^^^^^^^^ + +The OSM ID of the relation member. + +``relation_member_type`` +^^^^^^^^^^^^^^^^^^^^^^^^ + +The type of the relation member. 0 for nodes, 1 for ways and 2 for relations. + + +``relation_member_role`` +^^^^^^^^^^^^^^^^^^^^^^^^ + +The role of the relation member as a string, e.g. `outer`, `stop`, etc. + + +``relation_member_index`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The index of the member in the relation, starting from 0. E.g. the first member is 0, second member is 1, etc. +This can be used to query bus stops of a route relation in the right order. + Generalized Tables ------------------ diff --git a/docs/relations.rst b/docs/relations.rst new file mode 100644 index 0000000..7f61f84 --- /dev/null +++ b/docs/relations.rst @@ -0,0 +1,180 @@ +Relations +========= + +In `OpenStreetMap, relations `_ define logical or geographic relationships between other nodes, ways and relations. + +The most common relation type is a multipolygon, but all other relations can be imported as well. + +Multipolygons +------------- + +`Multipolygon relations `_ are used to represent complex polygon geometries. They are also the only way to represent holes in polygons. + + +Multipolygon relations are automatically handled by Imposm for all ``polygon`` tables. + +The following mapping:: + + tables: + buildings: + type: polygon + mapping: + building: [__any__] + + +Inserts closed ways if they have a ``building`` tag:: + + + + ... + + + + +It will also insert relations of the type ``multipolygon`` with a ``building`` tag:: + + + + + + + + +The roles are ignored by Imposm as not all holes are correctly tagged as ``inner``. Imposm uses geometry operations to verify if a member of a multipolygon is a hole, or if it is a separate polygon. + + +For compatibility, multipolygon relations without tags will use the tags from the (longest) outer way. Imposm will insert the following relation as well:: + + + + ... + + + + + + + + + + + +Other relations +--------------- + +OpenStreetMap also uses relations to map more complex features. Some examples: + + - `Administrative areas `_ with boundaries, capitals and label positions. + - `Bus/tram/train routes `_ with the route itself, stops and platforms. + - `3D buildings `_ with multiple parts that should not be computed as holes. + +These relations can not be mapped to `simple` linestrings or polygons as they can contain a mix of different geometry types, or would result in invalid geometries (overlapping polygons). + +The Imposm table types ``relation`` and ``relation_member`` allow you to import all relevant data for these relations. + +``relation_member`` +^^^^^^^^^^^^^^^^^^^ + +The ``relation_member`` table type inserts each member of the relation as a separate row. The ``relation_member`` has access to the `role` and `type` value of each member. You can also import tags from the relation `and` from the member node, way or relation. + +Example +~~~~~~~ + +You can use the following mapping:: + + route_members: + type: relation_member + columns: + - name: osm_id + type: id + - name: member + type: relation_member_id + - name: index + type: relation_member_index + - name: role + type: relation_member_role + - name: type + type: relation_member_type + - name: geometry + type: geometry + - name: relname + key: name + type: string + - name: name + key: name + type: string + from_member: true + - key: ref + name: ref + type: string + mapping: + route: [bus] + + +to import a bus relation with stops, a platform and the route itself:: + + + + + + + + + + + + + + + + +This will result in seven rows with the following columns: + +======== ====================================================================================================================================================== +Column Description +======== ====================================================================================================================================================== +osm_id The ID of the relation. 100901 for all members. +member The ID of the member. 100101, 100102, etc. +index The index of the member. From 1 for 100101 to 7 for 100503. This can be used to query the bus stops in the correct order. +role The role of the member. ``stop``, ``platform``, etc. +type 0 for nodes, 1 for ways and 2 for other relations. +geometry The geometry of the member. Point for nodes and linestring for ways. +relname The value of the ``name`` tag of the relation. ``Bus 301: A => B`` in this case. +name The value of the ``name`` tag of the member element, if it has one. Note that the mapping contains ``from_member: true`` for this column. +ref The value of the ``ref`` tag of the relation. ``301`` in this case. +======== ====================================================================================================================================================== + + +You can insert the tags of the relation in a separate ``relation`` table to avoid duplication and then use `joins` when querying the data. +Both ``osm_id`` and ``relation_member_id`` columns are indexed in PostgreSQL by default to speed up these joins. + +``relation`` +^^^^^^^^^^^^ + +The ``relation`` table type inserts the mapped element regardless of the resulting geometry. For example, this allows you to create a table with the metadata (name, reference, operator, etc.) of all available route relations. The actual geometries need to be `joined` form the members. + +Example +~~~~~~~ + +The following mapping imports the bus route relation from above:: + + routes: + type: relation + columns: + - name: osm_id + type: id + - key: ref + name: ref + type: string + - name: network + key: network + type: string + mapping: + route: [bus] + + +This will create a single row with the mapped columns. + +.. note:: ``relation`` tables do not support geometry columns. Use the geometries of the members, or use a ``polygon`` table if your relations contain multipolygons. + + From bfb50549aa84f2639267c34d76691a7239b686af Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 15 Jan 2016 14:53:47 +0100 Subject: [PATCH 36/36] rename relation_member_ fiels types to member_ --- docs/mapping.rst | 16 ++++++------- docs/relations.rst | 10 ++++----- mapping/fields.go | 40 ++++++++++++++++----------------- test/route_relation_mapping.yml | 16 ++++++------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/mapping.rst b/docs/mapping.rst index 449152c..0ff0e64 100644 --- a/docs/mapping.rst +++ b/docs/mapping.rst @@ -251,25 +251,25 @@ Element types for ``relation_member`` The following types are only valid for tables of the type ``relation_member``. -``relation_member_id`` -^^^^^^^^^^^^^^^^^^^^^^ +``member_id`` +^^^^^^^^^^^^^ The OSM ID of the relation member. -``relation_member_type`` -^^^^^^^^^^^^^^^^^^^^^^^^ +``member_type`` +^^^^^^^^^^^^^^^ The type of the relation member. 0 for nodes, 1 for ways and 2 for relations. -``relation_member_role`` -^^^^^^^^^^^^^^^^^^^^^^^^ +``member_role`` +^^^^^^^^^^^^^^^ The role of the relation member as a string, e.g. `outer`, `stop`, etc. -``relation_member_index`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +``member_index`` +^^^^^^^^^^^^^^^^ The index of the member in the relation, starting from 0. E.g. the first member is 0, second member is 1, etc. This can be used to query bus stops of a route relation in the right order. diff --git a/docs/relations.rst b/docs/relations.rst index 7f61f84..55b3f38 100644 --- a/docs/relations.rst +++ b/docs/relations.rst @@ -88,13 +88,13 @@ You can use the following mapping:: - name: osm_id type: id - name: member - type: relation_member_id + type: member_id - name: index - type: relation_member_index + type: member_index - name: role - type: relation_member_role + type: member_role - name: type - type: relation_member_type + type: member_type - name: geometry type: geometry - name: relname @@ -146,7 +146,7 @@ ref The value of the ``ref`` tag of the relation. ``301`` in this case. You can insert the tags of the relation in a separate ``relation`` table to avoid duplication and then use `joins` when querying the data. -Both ``osm_id`` and ``relation_member_id`` columns are indexed in PostgreSQL by default to speed up these joins. +Both ``osm_id`` and ``member_id`` columns are indexed in PostgreSQL by default to speed up these joins. ``relation`` ^^^^^^^^^^^^ diff --git a/mapping/fields.go b/mapping/fields.go index 31e0ba4..b2ca2a6 100644 --- a/mapping/fields.go +++ b/mapping/fields.go @@ -17,26 +17,26 @@ var AvailableFieldTypes map[string]FieldType func init() { AvailableFieldTypes = map[string]FieldType{ - "bool": {"bool", "bool", Bool, nil, nil, false}, - "boolint": {"boolint", "int8", BoolInt, nil, nil, false}, - "id": {"id", "int64", Id, nil, nil, false}, - "string": {"string", "string", String, nil, nil, false}, - "direction": {"direction", "int8", Direction, nil, nil, false}, - "integer": {"integer", "int32", Integer, nil, nil, false}, - "mapping_key": {"mapping_key", "string", KeyName, nil, nil, false}, - "mapping_value": {"mapping_value", "string", ValueName, nil, nil, false}, - "relation_member_id": {"relation_member_id", "int64", nil, nil, RelationMemberID, true}, - "relation_member_role": {"relation_member_role", "string", nil, nil, RelationMemberRole, true}, - "relation_member_type": {"relation_member_type", "int8", nil, nil, RelationMemberType, true}, - "relation_member_index": {"relation_member_index", "int32", nil, nil, RelationMemberIndex, true}, - "geometry": {"geometry", "geometry", Geometry, nil, nil, false}, - "validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil, nil, false}, - "hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil, nil, false}, - "wayzorder": {"wayzorder", "int32", WayZOrder, nil, nil, false}, - "pseudoarea": {"pseudoarea", "float32", PseudoArea, nil, nil, false}, - "zorder": {"zorder", "int32", nil, MakeZOrder, nil, false}, - "enumerate": {"enumerate", "int32", nil, MakeEnumerate, nil, false}, - "string_suffixreplace": {"string_suffixreplace", "string", nil, MakeSuffixReplace, nil, false}, + "bool": {"bool", "bool", Bool, nil, nil, false}, + "boolint": {"boolint", "int8", BoolInt, nil, nil, false}, + "id": {"id", "int64", Id, nil, nil, false}, + "string": {"string", "string", String, nil, nil, false}, + "direction": {"direction", "int8", Direction, nil, nil, false}, + "integer": {"integer", "int32", Integer, nil, nil, false}, + "mapping_key": {"mapping_key", "string", KeyName, nil, nil, false}, + "mapping_value": {"mapping_value", "string", ValueName, nil, nil, false}, + "member_id": {"member_id", "int64", nil, nil, RelationMemberID, true}, + "member_role": {"member_role", "string", nil, nil, RelationMemberRole, true}, + "member_type": {"member_type", "int8", nil, nil, RelationMemberType, true}, + "member_index": {"member_index", "int32", nil, nil, RelationMemberIndex, true}, + "geometry": {"geometry", "geometry", Geometry, nil, nil, false}, + "validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil, nil, false}, + "hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil, nil, false}, + "wayzorder": {"wayzorder", "int32", WayZOrder, nil, nil, false}, + "pseudoarea": {"pseudoarea", "float32", PseudoArea, nil, nil, false}, + "zorder": {"zorder", "int32", nil, MakeZOrder, nil, false}, + "enumerate": {"enumerate", "int32", nil, MakeEnumerate, nil, false}, + "string_suffixreplace": {"string_suffixreplace", "string", nil, MakeSuffixReplace, nil, false}, } } diff --git a/test/route_relation_mapping.yml b/test/route_relation_mapping.yml index d9a9773..aa17e6d 100644 --- a/test/route_relation_mapping.yml +++ b/test/route_relation_mapping.yml @@ -11,13 +11,13 @@ tables: - name: osm_id type: id - name: member - type: relation_member_id + type: member_id - name: index - type: relation_member_index + type: member_index - name: role - type: relation_member_role + type: member_role - name: type - type: relation_member_type + type: member_type - name: geometry type: geometry - name: subname @@ -38,13 +38,13 @@ tables: name: ref type: string - name: member - type: relation_member_id + type: member_id - name: index - type: relation_member_index + type: member_index - name: role - type: relation_member_role + type: member_role - name: type - type: relation_member_type + type: member_type - name: geometry type: geometry - name: relname