vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add use_single_id_space mapping option 

Mangle way and relation IDs so that they don't colide with
node IDs for imports into a single table.
Way IDs are negated, relation IDs are negated and shifted by -1e17.
master
Oliver Tonnhofer 2014-08-11 10:18:08 +02:00
parent aaa9181134
commit 3f3c12ece1
12 changed files with 197 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
diff/deleter.go
View File

@ -18,27 +18,29 @@ type Deleter struct {
tmLineStrings mapping.WayMatcher
tmPolygons mapping.RelWayMatcher
expireor expire.Expireor
singleIdSpace bool
deletedRelations map[int64]struct{}
deletedWays map[int64]struct{}
deletedMembers map[int64]struct{}
}
func NewDeleter(db database.Deleter, osmCache *cache.OSMCache, diffCache *cache.DiffCache,
singleIdSpace bool,
tmPoints mapping.NodeMatcher,
tmLineStrings mapping.WayMatcher,
tmPolygons mapping.RelWayMatcher,
) *Deleter {
return &Deleter{
db,
osmCache,
diffCache,
tmPoints,
tmLineStrings,
tmPolygons,
nil,
make(map[int64]struct{}),
make(map[int64]struct{}),
make(map[int64]struct{}),
delDb: db,
osmCache: osmCache,
diffCache: diffCache,
tmPoints: tmPoints,
tmLineStrings: tmLineStrings,
tmPolygons: tmPolygons,
singleIdSpace: singleIdSpace,
deletedRelations: make(map[int64]struct{}),
deletedWays: make(map[int64]struct{}),
deletedMembers: make(map[int64]struct{}),
}
}
@ -50,6 +52,24 @@ func (d *Deleter) DeletedMemberWays() map[int64]struct{} {
return d.deletedMembers
}
func (d *Deleter) nodeId(id int64) int64 {
return id
}
func (d *Deleter) WayId(id int64) int64 {
if !d.singleIdSpace {
return id
}
return -id
}
func (d *Deleter) RelId(id int64) int64 {
if !d.singleIdSpace {
return -id
}
return element.RelIdOffset - id
}
func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) error {
d.deletedRelations[id] = struct{}{}
@ -64,7 +84,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
return nil
}
if matches := d.tmPolygons.MatchRelation(elem); len(matches) > 0 {
if err := d.delDb.Delete(-elem.Id, matches); err != nil {
if err := d.delDb.Delete(d.RelId(elem.Id), matches); err != nil {
return err
}
} else {
@ -137,13 +157,13 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error {
}
deleted := false
if matches := d.tmPolygons.MatchWay(elem); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil {
return err
}
deleted = true
}
if matches := d.tmLineStrings.MatchWay(elem); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
if err := d.delDb.Delete(d.WayId(elem.Id), matches); err != nil {
return err
}
deleted = true
@ -179,7 +199,7 @@ func (d *Deleter) deleteNode(id int64) error {
deleted := false
if matches := d.tmPoints.MatchNode(elem); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
if err := d.delDb.Delete(d.nodeId(elem.Id), matches); err != nil {
return err
}
deleted = true

9
diff/process.go
View File

@ -83,6 +83,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
delDb,
osmCache,
diffCache,
tagmapping.SingleIdSpace,
tagmapping.PointMatcher(),
tagmapping.LineStringMatcher(),
tagmapping.PolygonMatcher(),
@ -98,7 +99,9 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
ways := make(chan *element.Way)
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
config.BaseOptions.Srid)
@ -106,7 +109,9 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
relWriter.SetExpireor(expireor)
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(),
tagmapping.LineStringMatcher(),

14
element/element.go
View File

@ -89,3 +89,17 @@ func (idRefs *IdRefs) Delete(ref int64) {
idRefs.Refs = append(idRefs.Refs[:i], idRefs.Refs[i+1:]...)
}
}
// RelIdOffset is a constant we subtract from relation IDs
// to avoid conflicts with way and node IDs.
// Nodes, ways and relations have separate ID spaces in OSM, but
// we need unique IDs for updating and removing elements in diff mode.
// In a normal diff import relation IDs are negated to distinguish them
// from way IDs, because ways and relations can both be imported in the
// same polygon table.
// Nodes are only imported together with ways and relations in single table
// imports (see `type_mappings`). In this case we negate the way and
// relation IDs and aditionaly subtract RelIdOffset from the relation IDs.
// Ways will go from -0 to -100,000,000,000,000,000, relations from
// -100,000,000,000,000,000 down wards.
const RelIdOffset = -1e17

8
import_/import.go
View File

@ -174,7 +174,9 @@ func Import() {
osmCache.Coords.SetReadOnly(true)
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
config.BaseOptions.Srid)
@ -185,7 +187,9 @@ func Import() {
osmCache.Relations.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),
config.BaseOptions.Srid)

6
mapping/config.go
View File

@ -3,8 +3,9 @@ package mapping
import (
"encoding/json"
"errors"
"github.com/omniscale/imposm3/element"
"os"
"github.com/omniscale/imposm3/element"
)
type Field struct {
@ -44,6 +45,9 @@ type Mapping struct {
Tables Tables `json:"tables"`
GeneralizedTables GeneralizedTables `json:"generalized_tables"`
Tags Tags `json:"tags"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `json:"use_single_id_space"`
}
type Tags struct {

2
test/Makefile
View File

@ -32,6 +32,6 @@ test: .lasttestrun_complete_db .lasttestrun_single_table
nosetests complete_db_test.py $(NOSEOPTS)
@touch .lasttestrun_complete_db
.lasttestrun_single_table: $(IMPOSM_BIN) single_table_test.py build/single_table.pbf
.lasttestrun_single_table: $(IMPOSM_BIN) single_table_test.py build/single_table.osc.gz build/single_table.pbf
nosetests single_table_test.py $(NOSEOPTS)
@touch .lasttestrun_single_table

8
test/single_table.osc Normal file
View File

@ -0,0 +1,8 @@
<?xml version='1.0' encoding='UTF-8'?>
<osmChange version="0.6" generator="Osmosis 0.41">
<modify>
<node id="31101" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="81">
<tag k="amenity" v="cafe"/>
</node>
</modify>
</osmChange>

38
test/single_table.osm
View File

@ -106,4 +106,42 @@
</way>
<!-- source nodes/ways for tests below -->
<node id="31001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
<node id="31002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
<node id="31003" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="82"/>
<node id="31004" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="80"/>
<way id="31002" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="31001"/>
<nd ref="31002"/>
<tag k="barrier" v="fence"/>
</way>
<way id="31003" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="31002"/>
<nd ref="31003"/>
<nd ref="31004"/>
<nd ref="31001"/>
</way>
<!-- modify duplicate node -->
<node id="31101" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80">
<tag k="amenity" v="cafe"/>
</node>
<way id="31101" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="31001"/>
<nd ref="31002"/>
<nd ref="31003"/>
<nd ref="31004"/>
<nd ref="31001"/>
<tag k="highway" v="secondary"/>
<tag k="landuse" v="park"/>
</way>
<relation id="31101" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="31002" role="outer"/>
<member type="way" ref="31003" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="building" v="yes"/>
</relation>
</osm>

1
test/single_table_mapping.json
View File

@ -6,6 +6,7 @@
"source"
]
},
"use_single_id_space": true,
"tables": {
"all": {
"fields": [

60
test/single_table_test.py
View File

@ -13,6 +13,8 @@ def setup():
def teardown():
t.teardown()
RELOFFSET = int(-1e17)
#######################################################################
def test_import():
"""Import succeeds"""
@ -54,38 +56,38 @@ def test_non_mapped_way_is_missing():
def test_mapped_way():
"""Way is stored with all tags."""
t.assert_cached_way(20201)
highway = t.query_row(t.db_conf, 'osm_all', 20201)
highway = t.query_row(t.db_conf, 'osm_all', -20201)
assert highway['tags'] == {'random': 'tag', 'highway': 'yes'}
def test_non_mapped_closed_way_is_missing():
"""Closed way without mapped tags is missing."""
t.assert_cached_way(20301)
assert not t.query_row(t.db_conf, 'osm_all', 20301)
assert not t.query_row(t.db_conf, 'osm_all', -20301)
def test_mapped_closed_way():
"""Closed way is stored with all tags."""
t.assert_cached_way(20401)
building = t.query_row(t.db_conf, 'osm_all', 20401)
building = t.query_row(t.db_conf, 'osm_all', -20401)
assert building['tags'] == {'random': 'tag', 'building': 'yes'}
def test_mapped_closed_way_area_yes():
"""Closed way with area=yes is not stored as linestring."""
t.assert_cached_way(20501)
elem = t.query_row(t.db_conf, 'osm_all', 20501)
elem = t.query_row(t.db_conf, 'osm_all', -20501)
assert elem['geometry'].type == 'Polygon', elem['geometry'].type
assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'yes'}
def test_mapped_closed_way_area_no():
"""Closed way with area=no is not stored as polygon."""
t.assert_cached_way(20502)
elem = t.query_row(t.db_conf, 'osm_all', 20502)
elem = t.query_row(t.db_conf, 'osm_all', -20502)
assert elem['geometry'].type == 'LineString', elem['geometry'].type
assert elem['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian', 'area': 'no'}
def test_mapped_closed_way_without_area():
"""Closed way without area is stored as mapped (linestring and polygon)."""
t.assert_cached_way(20601)
elems = t.query_row(t.db_conf, 'osm_all', 20601)
elems = t.query_row(t.db_conf, 'osm_all', -20601)
assert len(elems) == 2
elems.sort(key=lambda x: x['geometry'].type)
@ -94,6 +96,52 @@ def test_mapped_closed_way_without_area():
assert elems[1]['geometry'].type == 'Polygon', elems[1]['geometry'].type
assert elems[1]['tags'] == {'random': 'tag', 'landuse': 'grass', 'highway': 'pedestrian'}
def test_duplicate_ids_1():
"""Points/lines/polygons with same ID are inserted."""
node = t.query_row(t.db_conf, 'osm_all', 31101)
assert node['geometry'].type == 'Point', node['geometry'].type
assert node['tags'] == {'amenity': 'cafe'}
assert node['geometry'].distance(t.merc_point(80, 47)) < 1
ways = t.query_row(t.db_conf, 'osm_all', -31101)
ways.sort(key=lambda x: x['geometry'].type)
assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type
assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type
assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L)
assert rel['geometry'].type == 'Polygon', rel['geometry'].type
assert rel['tags'] == {'building': 'yes'}
#######################################################################
def test_update():
"""Diff import applies"""
t.imposm3_update(t.db_conf, './build/single_table.osc.gz', mapping_file)
#######################################################################
def test_duplicate_ids_2():
"""Node moved and ways/rels with same ID are still present."""
node = t.query_row(t.db_conf, 'osm_all', 31101)
assert node['geometry'].type == 'Point', node['geometry'].type
assert node['tags'] == {'amenity': 'cafe'}
assert node['geometry'].distance(t.merc_point(81, 47)) < 1
ways = t.query_row(t.db_conf, 'osm_all', -31101)
ways.sort(key=lambda x: x['geometry'].type)
assert ways[0]['geometry'].type == 'LineString', ways[0]['geometry'].type
assert ways[0]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
assert ways[1]['geometry'].type == 'Polygon', ways[1]['geometry'].type
assert ways[1]['tags'] == {'landuse': 'park', 'highway': 'secondary'}
rel = t.query_row(t.db_conf, 'osm_all', RELOFFSET-31101L)
assert rel['geometry'].type == 'Polygon', rel['geometry'].type
assert rel['tags'] == {'building': 'yes'}
#######################################################################
def test_deploy_and_revert_deploy():

19
writer/relations.go
View File

@ -1,6 +1,9 @@
package writer
import (
"sync"
"time"
"github.com/omniscale/imposm3/cache"
"github.com/omniscale/imposm3/database"
"github.com/omniscale/imposm3/element"
@ -9,12 +12,11 @@ import (
"github.com/omniscale/imposm3/geom/geos"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/stats"
"sync"
"time"
)
type RelationWriter struct {
OsmElemWriter
singleIdSpace bool
rel chan *element.Relation
polygonMatcher mapping.RelWayMatcher
}
@ -22,6 +24,7 @@ type RelationWriter struct {
func NewRelationWriter(
osmCache *cache.OSMCache,
diffCache *cache.DiffCache,
singleIdSpace bool,
rel chan *element.Relation,
inserter database.Inserter,
progress *stats.Statistics,
@ -37,6 +40,7 @@ func NewRelationWriter(
inserter: inserter,
srid: srid,
},
singleIdSpace: singleIdSpace,
polygonMatcher: matcher,
rel: rel,
}
@ -44,6 +48,13 @@ func NewRelationWriter(
return &rw.OsmElemWriter
}
func (rw *RelationWriter) relId(id int64) int64 {
if !rw.singleIdSpace {
return -id
}
return element.RelIdOffset - id
}
func (rw *RelationWriter) loop() {
geos := geos.NewGeos()
geos.SetHandleSrid(rw.srid)
@ -117,7 +128,7 @@ NextRel:
}
for _, g := range parts {
rel := element.Relation(*r)
rel.Id = -r.Id
rel.Id = rw.relId(r.Id)
rel.Geom = &element.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)}
err := rw.inserter.InsertPolygon(rel.OSMElem, matches)
if err != nil {
@ -129,7 +140,7 @@ NextRel:
}
} else {
rel := element.Relation(*r)
rel.Id = -r.Id
rel.Id = rw.relId(r.Id)
err := rw.inserter.InsertPolygon(rel.OSMElem, matches)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {

15
writer/ways.go
View File

@ -1,6 +1,8 @@
package writer
import (
"sync"
"github.com/omniscale/imposm3/cache"
"github.com/omniscale/imposm3/database"
"github.com/omniscale/imposm3/element"
@ -9,11 +11,11 @@ import (
"github.com/omniscale/imposm3/geom/geos"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/stats"
"sync"
)
type WayWriter struct {
OsmElemWriter
singleIdSpace bool
ways chan *element.Way
lineMatcher mapping.WayMatcher
polygonMatcher mapping.WayMatcher
@ -22,6 +24,7 @@ type WayWriter struct {
func NewWayWriter(
osmCache *cache.OSMCache,
diffCache *cache.DiffCache,
singleIdSpace bool,
ways chan *element.Way,
inserter database.Inserter,
progress *stats.Statistics,
@ -38,6 +41,7 @@ func NewWayWriter(
inserter: inserter,
srid: srid,
},
singleIdSpace: singleIdSpace,
lineMatcher: lineMatcher,
polygonMatcher: polygonMatcher,
ways: ways,
@ -46,6 +50,13 @@ func NewWayWriter(
return &ww.OsmElemWriter
}
func (ww *WayWriter) wayId(id int64) int64 {
if !ww.singleIdSpace {
return id
}
return -id
}
func (ww *WayWriter) loop() {
geos := geos.NewGeos()
geos.SetHandleSrid(ww.srid)
@ -67,6 +78,8 @@ func (ww *WayWriter) loop() {
}
ww.NodesToSrid(w.Nodes)
w.Id = ww.wayId(w.Id)
inserted := false
if matches := ww.lineMatcher.MatchWay(w); len(matches) > 0 {
err := ww.buildAndInsert(geos, w, matches, false)