vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

writer: remove old-style-multipolygon handling

master
Oliver Tonnhofer 2017-11-17 13:56:39 +01:00
parent 7bb3908bce
commit e79386a1af
10 changed files with 37 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cache/config.go vendored
View File

@ -21,13 +21,12 @@ type coordsCacheOptions struct {
BunchCacheCapacity int
}
type osmCacheOptions struct {
Coords coordsCacheOptions
Ways cacheOptions
Nodes cacheOptions
Relations cacheOptions
InsertedWays cacheOptions
CoordsIndex cacheOptions
WaysIndex cacheOptions
Coords coordsCacheOptions
Ways cacheOptions
Nodes cacheOptions
Relations cacheOptions
CoordsIndex cacheOptions
WaysIndex cacheOptions
}
const defaultConfig = `
@ -62,13 +61,6 @@ const defaultConfig = `
"MaxOpenFiles": 64,
"BlockRestartInterval": 128
},
"InsertedWays": {
"CacheSizeM": 0,
"WriteBufferSizeM": 0,
"BlockSizeK": 0,
"MaxOpenFiles": 0,
"BlockRestartInterval": 0
},
"CoordsIndex": {
"CacheSizeM": 32,
"WriteBufferSizeM": 128,

22
cache/osm.go vendored
View File

@ -17,13 +17,12 @@ var (
const SKIP int64 = -1
type OSMCache struct {
dir string
Coords *DeltaCoordsCache
Ways *WaysCache
Nodes *NodesCache
Relations *RelationsCache
InsertedWays *InsertedWaysCache
opened bool
dir string
Coords *DeltaCoordsCache
Ways *WaysCache
Nodes *NodesCache
Relations *RelationsCache
opened bool
}
func (c *OSMCache) Close() {
@ -43,10 +42,6 @@ func (c *OSMCache) Close() {
c.Relations.Close()
c.Relations = nil
}
if c.InsertedWays != nil {
c.InsertedWays.Close()
c.InsertedWays = nil
}
}
func NewOSMCache(dir string) *OSMCache {
@ -78,11 +73,6 @@ func (c *OSMCache) Open() error {
c.Close()
return err
}
c.InsertedWays, err = newInsertedWaysCache(filepath.Join(c.dir, "inserted_ways"))
if err != nil {
c.Close()
return err
}
c.opened = true
return nil
}

60
cache/ways.go vendored
View File

@ -112,63 +112,3 @@ func (self *WaysCache) FillMembers(members []element.Member) error {
}
return nil
}
type InsertedWaysCache struct {
cache
}
func newInsertedWaysCache(path string) (*InsertedWaysCache, error) {
cache := InsertedWaysCache{}
cache.options = &globalCacheOptions.InsertedWays
err := cache.open(path)
if err != nil {
return nil, err
}
return &cache, err
}
func (p *InsertedWaysCache) PutWay(way *element.Way) error {
keyBuf := idToKeyBuf(way.Id)
return p.db.Put(p.wo, keyBuf, []byte{})
}
func (p *InsertedWaysCache) PutMembers(members []element.Member) error {
batch := levigo.NewWriteBatch()
defer batch.Close()
for _, m := range members {
if m.Type != element.WAY {
continue
}
keyBuf := idToKeyBuf(m.Id)
batch.Put(keyBuf, []byte{})
}
return p.db.Write(p.wo, batch)
}
func (p *InsertedWaysCache) DeleteMembers(members []element.Member) error {
batch := levigo.NewWriteBatch()
defer batch.Close()
for _, m := range members {
if m.Type != element.WAY {
continue
}
keyBuf := idToKeyBuf(m.Id)
batch.Delete(keyBuf)
}
return p.db.Write(p.wo, batch)
}
func (p *InsertedWaysCache) IsInserted(id int64) (bool, error) {
keyBuf := idToKeyBuf(id)
data, err := p.db.Get(p.ro, keyBuf)
if err != nil {
return false, err
}
if data == nil {
return false, nil
}
return true, nil
}

47
mapping/matcher.go
View File

@ -279,50 +279,3 @@ func (r *rowBuilder) MakeMemberRow(rel *element.Relation, member *element.Member
}
return row
}
// SelectRelationPolygons returns a slice of all members that are already
// imported as part of the relation.
// Outer members are "imported" if they share the same destination table. Inner members
// are "imported" when they also share the same key/value.
func SelectRelationPolygons(polygonTagMatcher RelWayMatcher, rel *element.Relation) []element.Member {
relMatches := polygonTagMatcher.MatchRelation(rel)
result := []element.Member{}
for _, m := range rel.Members {
if m.Type != element.WAY {
continue
}
memberMatches := polygonTagMatcher.MatchWay(m.Way)
if m.Role == "outer" && dstEquals(relMatches, memberMatches) {
result = append(result, m)
} else if matchEquals(relMatches, memberMatches) {
result = append(result, m)
}
}
return result
}
// matchEquals returns true if both matches share key/value and table
func matchEquals(matchesA, matchesB []Match) bool {
for _, matchA := range matchesA {
for _, matchB := range matchesB {
if matchA.Key == matchB.Key &&
matchA.Value == matchB.Value &&
matchA.Table == matchB.Table {
return true
}
}
}
return false
}
// dstEquals returns true if both matches share a single destination table
func dstEquals(matchesA, matchesB []Match) bool {
for _, matchA := range matchesA {
for _, matchB := range matchesB {
if matchA.Table == matchB.Table {
return true
}
}
}
return false
}

137
mapping/matcher_test.go
View File

@ -20,140 +20,3 @@ func BenchmarkTagMatch(b *testing.B) {
}
}
}
func makeMember(id int64, tags element.Tags) element.Member {
way := &element.Way{
OSMElem: element.OSMElem{Id: id, Tags: tags},
Refs: []int64{0, 1, 2, 0}, // fake closed way, req. for SelectRelationPolygons
}
return element.Member{Id: id, Type: element.WAY, Role: "", Way: way}
}
func makeMemberRole(id int64, tags element.Tags, role string) element.Member {
way := &element.Way{
OSMElem: element.OSMElem{Id: id, Tags: tags},
Refs: []int64{0, 1, 2, 0}, // fake closed way, req. for SelectRelationPolygons
}
return element.Member{Id: id, Type: element.WAY, Role: role, Way: way}
}
func TestSelectRelationPolygonsSimple(t *testing.T) {
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "forest"}),
makeMember(1, element.Tags{"landuse": "park"}),
makeMember(2, element.Tags{"waterway": "riverbank"}),
makeMember(4, element.Tags{"foo": "bar"}),
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher,
&r,
)
if len(filtered) != 1 {
t.Fatal(filtered)
}
if filtered[0].Id != 1 {
t.Fatal(filtered[0])
}
}
func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}),
makeMember(1, element.Tags{"landuse": "forest"}),
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher,
&r,
)
if len(filtered) != 1 {
t.Fatal(filtered)
}
if filtered[0].Id != 0 {
t.Fatal(filtered)
}
}
func TestSelectRelationPolygonsMultiple(t *testing.T) {
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "park"}),
makeMember(1, element.Tags{"natural": "forest"}),
makeMember(2, element.Tags{"landuse": "park"}),
makeMember(3, element.Tags{"highway": "pedestrian"}),
makeMember(4, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}),
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher,
&r,
)
if len(filtered) != 3 {
t.Fatal(filtered)
}
if filtered[0].Id != 0 || filtered[1].Id != 2 || filtered[2].Id != 4 {
t.Fatal(filtered)
}
}
func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "forest", "natural": "scrub", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"natural": "scrub"}),
makeMember(1, element.Tags{"landuse": "forest"}),
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher,
&r,
)
// TODO both should be filterd out, but we only get one,
// because we match only one tag per table
if len(filtered) != 1 {
t.Fatal(filtered)
}
}
func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) {
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"waterway": "riverbank", "type": "multipolygon"}
r.Members = []element.Member{
makeMemberRole(0, element.Tags{"waterway": "riverbank", "natural": "water"}, "outer"),
makeMemberRole(1, element.Tags{"natural": "water"}, "inner"),
makeMemberRole(2, element.Tags{"place": "islet"}, "inner"),
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher,
&r,
)
if len(filtered) != 1 {
t.Fatal(filtered)
}
if filtered[0].Id != 0 {
t.Fatal(filtered)
}
}

8
test/complete_db.osm
View File

@ -471,7 +471,7 @@
</way>
<!-- test multipolygon way was _not_ inserted -->
<!-- Old-style-mp: test multipolygon way is now inserted -->
<node id="9001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
<node id="9002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
<node id="9003" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="82"/>
@ -484,6 +484,7 @@
<nd ref="9001"/>
<tag k="landuse" v="park"/>
<tag k="ref" v="42"/> <!-- unrelated tag -->
<tag k="name" v="way 9009"/>
</way>
<relation id="9001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="9009" role="outer"/>
@ -492,7 +493,7 @@
<tag k="type" v="multipolygon"/>
</relation>
<!-- test multipolygon way was _not_ inserted, but hole is -->
<!-- Old-style-mp: test multipolygon way is now inserted -->
<node id="9101" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
<node id="9102" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
<node id="9103" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="82"/>
@ -509,6 +510,7 @@
<nd ref="9101"/>
<tag k="landuse" v="park"/>
<tag k="ref" v="42"/> <!-- unrelated tag -->
<tag k="name" v="way 9109"/>
</way>
<way id="9110" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="9111"/>
@ -775,7 +777,7 @@
<tag k="railway" v="tram"/>
</way>
<!-- test outer way not inserted (different tag but same table) -->
<!-- Old-style-mp: test outer/inner way are now inserted -->
<node id="19001" version="1" timestamp="2011-11-11T00:11:11Z" lat="21" lon="89"/>
<node id="19002" version="1" timestamp="2011-11-11T00:11:11Z" lat="21" lon="90"/>
<node id="19003" version="1" timestamp="2011-11-11T00:11:11Z" lat="24" lon="90"/>

16
test/completedb_test.go
View File

@ -155,12 +155,14 @@ func TestComplete_NodeWayInsertedTwice(t *testing.T) {
}
}
func TestComplete_OuterWayNotInserted(t *testing.T) {
// Outer way with different tag is not inserted twice into same table
func TestComplete_OuterWayInsertedTwice(t *testing.T) {
// Outer way with different tag value is inserted twice into same table
// behavior changed from pre-old-style-mp-removal:
// test outer way not inserted (different tag but same table)
assertRecords(t, []checkElem{
{"osm_landusages", -19001, "farmland", nil},
{"osm_landusages", 19002, "farmyard", nil},
{"osm_landusages", 19001, Missing, nil},
{"osm_landusages", 19001, "farm", nil},
})
}
@ -206,13 +208,13 @@ func TestComplete_WayRelRefAfterDelete1(t *testing.T) {
})
}
func TestComplete_RelationWayNotInserted(t *testing.T) {
// Part of relation was inserted only once.
func TestComplete_OldStyleMpRelationWayInserted(t *testing.T) {
// Old-style-mp: Part of relation is now inserted.
assertRecords(t, []checkElem{
{"osm_landusages", -9001, "park", map[string]string{"name": "rel 9001"}},
{"osm_landusages", 9009, Missing, nil},
{"osm_landusages", 9009, "park", map[string]string{"name": "way 9009"}},
{"osm_landusages", -9101, "park", map[string]string{"name": "rel 9101"}},
{"osm_landusages", 9109, Missing, nil},
{"osm_landusages", 9109, "park", map[string]string{"name": "way 9109"}},
{"osm_landusages", 9110, "scrub", nil},
})
}

26
update/deleter.go
View File

@ -125,32 +125,6 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
}
}
if deleteMembers && deletedPolygon {
// delete members from db and force reinsert of members
// use case: relation is deleted and member now stands for its own
// TODO: still needed after old-style mp removal, remove when #148 is closed
for _, member := range elem.Members {
if member.Type == element.WAY {
d.deletedMembers[member.Id] = struct{}{}
if _, ok := d.deletedWays[member.Id]; ok {
continue
}
for _, r := range d.diffCache.Ways.Get(member.Id) {
if err := d.deleteRelation(r, false, false); err != nil {
return err
}
}
if err := d.deleteWay(member.Id, false); err != nil {
return err
}
}
}
}
if err := d.osmCache.InsertedWays.DeleteMembers(elem.Members); err != nil {
return err
}
if deleted && d.expireor != nil {
if err := d.osmCache.Ways.FillMembers(elem.Members); err != nil {
return err

6
writer/relations.go
View File

@ -197,12 +197,6 @@ func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geo
}
}
for _, m := range mapping.SelectRelationPolygons(rw.polygonMatcher, r) {
err = rw.osmCache.InsertedWays.PutWay(m.Way)
if err != nil {
log.Warn(err)
}
}
return true
}

29
writer/ways.go
View File

@ -72,11 +72,6 @@ func (ww *WayWriter) loop() {
if len(w.Tags) == 0 {
continue
}
insertedAsRelation, err := ww.osmCache.InsertedWays.IsInserted(w.Id)
if err != nil {
log.Warn(err)
continue
}
filled := false
// fill loads all coords. call only if we have a match
@ -95,6 +90,7 @@ func (ww *WayWriter) loop() {
w.Id = ww.wayId(w.Id)
var err error
inserted := false
insertedPolygon := false
if matches := ww.lineMatcher.MatchWay(w); len(matches) > 0 {
@ -109,20 +105,17 @@ func (ww *WayWriter) loop() {
continue
}
}
if !insertedAsRelation {
// only add polygons that were not inserted as a MultiPolygon relation
if matches := ww.polygonMatcher.MatchWay(w); len(matches) > 0 {
if !fill(w) {
continue
}
if w.IsClosed() {
err, insertedPolygon = ww.buildAndInsert(geos, w, matches, true)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
if matches := ww.polygonMatcher.MatchWay(w); len(matches) > 0 {
if !fill(w) {
continue
}
if w.IsClosed() {
err, insertedPolygon = ww.buildAndInsert(geos, w, matches, true)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
}
}
}