diff --git a/cache/config.go b/cache/config.go index fbce55e..2baf84a 100644 --- a/cache/config.go +++ b/cache/config.go @@ -21,13 +21,12 @@ type coordsCacheOptions struct { BunchCacheCapacity int } type osmCacheOptions struct { - Coords coordsCacheOptions - Ways cacheOptions - Nodes cacheOptions - Relations cacheOptions - InsertedWays cacheOptions - CoordsIndex cacheOptions - WaysIndex cacheOptions + Coords coordsCacheOptions + Ways cacheOptions + Nodes cacheOptions + Relations cacheOptions + CoordsIndex cacheOptions + WaysIndex cacheOptions } const defaultConfig = ` @@ -62,13 +61,6 @@ const defaultConfig = ` "MaxOpenFiles": 64, "BlockRestartInterval": 128 }, - "InsertedWays": { - "CacheSizeM": 0, - "WriteBufferSizeM": 0, - "BlockSizeK": 0, - "MaxOpenFiles": 0, - "BlockRestartInterval": 0 - }, "CoordsIndex": { "CacheSizeM": 32, "WriteBufferSizeM": 128, diff --git a/cache/osm.go b/cache/osm.go index aa28980..e8a8f89 100644 --- a/cache/osm.go +++ b/cache/osm.go @@ -17,13 +17,12 @@ var ( const SKIP int64 = -1 type OSMCache struct { - dir string - Coords *DeltaCoordsCache - Ways *WaysCache - Nodes *NodesCache - Relations *RelationsCache - InsertedWays *InsertedWaysCache - opened bool + dir string + Coords *DeltaCoordsCache + Ways *WaysCache + Nodes *NodesCache + Relations *RelationsCache + opened bool } func (c *OSMCache) Close() { @@ -43,10 +42,6 @@ func (c *OSMCache) Close() { c.Relations.Close() c.Relations = nil } - if c.InsertedWays != nil { - c.InsertedWays.Close() - c.InsertedWays = nil - } } func NewOSMCache(dir string) *OSMCache { @@ -78,11 +73,6 @@ func (c *OSMCache) Open() error { c.Close() return err } - c.InsertedWays, err = newInsertedWaysCache(filepath.Join(c.dir, "inserted_ways")) - if err != nil { - c.Close() - return err - } c.opened = true return nil } diff --git a/cache/ways.go b/cache/ways.go index f1df80e..9f4d352 100644 --- a/cache/ways.go +++ b/cache/ways.go @@ -112,63 +112,3 @@ func (self *WaysCache) FillMembers(members []element.Member) error { } return nil } - -type InsertedWaysCache struct { - cache -} - -func newInsertedWaysCache(path string) (*InsertedWaysCache, error) { - cache := InsertedWaysCache{} - cache.options = &globalCacheOptions.InsertedWays - - err := cache.open(path) - if err != nil { - return nil, err - } - return &cache, err -} - -func (p *InsertedWaysCache) PutWay(way *element.Way) error { - keyBuf := idToKeyBuf(way.Id) - return p.db.Put(p.wo, keyBuf, []byte{}) -} - -func (p *InsertedWaysCache) PutMembers(members []element.Member) error { - batch := levigo.NewWriteBatch() - defer batch.Close() - - for _, m := range members { - if m.Type != element.WAY { - continue - } - keyBuf := idToKeyBuf(m.Id) - batch.Put(keyBuf, []byte{}) - } - return p.db.Write(p.wo, batch) -} - -func (p *InsertedWaysCache) DeleteMembers(members []element.Member) error { - batch := levigo.NewWriteBatch() - defer batch.Close() - - for _, m := range members { - if m.Type != element.WAY { - continue - } - keyBuf := idToKeyBuf(m.Id) - batch.Delete(keyBuf) - } - return p.db.Write(p.wo, batch) -} - -func (p *InsertedWaysCache) IsInserted(id int64) (bool, error) { - keyBuf := idToKeyBuf(id) - data, err := p.db.Get(p.ro, keyBuf) - if err != nil { - return false, err - } - if data == nil { - return false, nil - } - return true, nil -} diff --git a/mapping/matcher.go b/mapping/matcher.go index 50b9ca1..2c58446 100644 --- a/mapping/matcher.go +++ b/mapping/matcher.go @@ -279,50 +279,3 @@ func (r *rowBuilder) MakeMemberRow(rel *element.Relation, member *element.Member } return row } - -// SelectRelationPolygons returns a slice of all members that are already -// imported as part of the relation. -// Outer members are "imported" if they share the same destination table. Inner members -// are "imported" when they also share the same key/value. -func SelectRelationPolygons(polygonTagMatcher RelWayMatcher, rel *element.Relation) []element.Member { - relMatches := polygonTagMatcher.MatchRelation(rel) - result := []element.Member{} - for _, m := range rel.Members { - if m.Type != element.WAY { - continue - } - memberMatches := polygonTagMatcher.MatchWay(m.Way) - if m.Role == "outer" && dstEquals(relMatches, memberMatches) { - result = append(result, m) - } else if matchEquals(relMatches, memberMatches) { - result = append(result, m) - } - } - return result -} - -// matchEquals returns true if both matches share key/value and table -func matchEquals(matchesA, matchesB []Match) bool { - for _, matchA := range matchesA { - for _, matchB := range matchesB { - if matchA.Key == matchB.Key && - matchA.Value == matchB.Value && - matchA.Table == matchB.Table { - return true - } - } - } - return false -} - -// dstEquals returns true if both matches share a single destination table -func dstEquals(matchesA, matchesB []Match) bool { - for _, matchA := range matchesA { - for _, matchB := range matchesB { - if matchA.Table == matchB.Table { - return true - } - } - } - return false -} diff --git a/mapping/matcher_test.go b/mapping/matcher_test.go index b44be0b..ec8703b 100644 --- a/mapping/matcher_test.go +++ b/mapping/matcher_test.go @@ -20,140 +20,3 @@ func BenchmarkTagMatch(b *testing.B) { } } } - -func makeMember(id int64, tags element.Tags) element.Member { - way := &element.Way{ - OSMElem: element.OSMElem{Id: id, Tags: tags}, - Refs: []int64{0, 1, 2, 0}, // fake closed way, req. for SelectRelationPolygons - } - return element.Member{Id: id, Type: element.WAY, Role: "", Way: way} -} - -func makeMemberRole(id int64, tags element.Tags, role string) element.Member { - way := &element.Way{ - OSMElem: element.OSMElem{Id: id, Tags: tags}, - Refs: []int64{0, 1, 2, 0}, // fake closed way, req. for SelectRelationPolygons - } - return element.Member{Id: id, Type: element.WAY, Role: role, Way: way} -} - -func TestSelectRelationPolygonsSimple(t *testing.T) { - mapping, err := NewMapping("test_mapping.yml") - if err != nil { - t.Fatal(err) - } - r := element.Relation{} - r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} - r.Members = []element.Member{ - makeMember(0, element.Tags{"landuse": "forest"}), - makeMember(1, element.Tags{"landuse": "park"}), - makeMember(2, element.Tags{"waterway": "riverbank"}), - makeMember(4, element.Tags{"foo": "bar"}), - } - filtered := SelectRelationPolygons( - mapping.PolygonMatcher, - &r, - ) - if len(filtered) != 1 { - t.Fatal(filtered) - } - if filtered[0].Id != 1 { - t.Fatal(filtered[0]) - } -} - -func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) { - mapping, err := NewMapping("test_mapping.yml") - if err != nil { - t.Fatal(err) - } - r := element.Relation{} - r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} - r.Members = []element.Member{ - makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}), - makeMember(1, element.Tags{"landuse": "forest"}), - } - filtered := SelectRelationPolygons( - mapping.PolygonMatcher, - &r, - ) - if len(filtered) != 1 { - t.Fatal(filtered) - } - if filtered[0].Id != 0 { - t.Fatal(filtered) - } -} - -func TestSelectRelationPolygonsMultiple(t *testing.T) { - mapping, err := NewMapping("test_mapping.yml") - if err != nil { - t.Fatal(err) - } - r := element.Relation{} - r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} - r.Members = []element.Member{ - makeMember(0, element.Tags{"landuse": "park"}), - makeMember(1, element.Tags{"natural": "forest"}), - makeMember(2, element.Tags{"landuse": "park"}), - makeMember(3, element.Tags{"highway": "pedestrian"}), - makeMember(4, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}), - } - filtered := SelectRelationPolygons( - mapping.PolygonMatcher, - &r, - ) - if len(filtered) != 3 { - t.Fatal(filtered) - } - if filtered[0].Id != 0 || filtered[1].Id != 2 || filtered[2].Id != 4 { - t.Fatal(filtered) - } -} - -func TestSelectRelationPolygonsMultipleTags(t *testing.T) { - mapping, err := NewMapping("test_mapping.yml") - if err != nil { - t.Fatal(err) - } - r := element.Relation{} - r.Tags = element.Tags{"landuse": "forest", "natural": "scrub", "type": "multipolygon"} - r.Members = []element.Member{ - makeMember(0, element.Tags{"natural": "scrub"}), - makeMember(1, element.Tags{"landuse": "forest"}), - } - filtered := SelectRelationPolygons( - mapping.PolygonMatcher, - &r, - ) - // TODO both should be filterd out, but we only get one, - // because we match only one tag per table - if len(filtered) != 1 { - t.Fatal(filtered) - } -} - -func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) { - mapping, err := NewMapping("test_mapping.yml") - if err != nil { - t.Fatal(err) - } - r := element.Relation{} - r.Tags = element.Tags{"waterway": "riverbank", "type": "multipolygon"} - r.Members = []element.Member{ - makeMemberRole(0, element.Tags{"waterway": "riverbank", "natural": "water"}, "outer"), - makeMemberRole(1, element.Tags{"natural": "water"}, "inner"), - makeMemberRole(2, element.Tags{"place": "islet"}, "inner"), - } - filtered := SelectRelationPolygons( - mapping.PolygonMatcher, - &r, - ) - - if len(filtered) != 1 { - t.Fatal(filtered) - } - if filtered[0].Id != 0 { - t.Fatal(filtered) - } -} diff --git a/test/complete_db.osm b/test/complete_db.osm index 75f3f1e..2c71909 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -471,7 +471,7 @@ - + @@ -484,6 +484,7 @@ + @@ -492,7 +493,7 @@ - + @@ -509,6 +510,7 @@ + @@ -775,7 +777,7 @@ - + diff --git a/test/completedb_test.go b/test/completedb_test.go index 4bf0773..1c2c060 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -155,12 +155,14 @@ func TestComplete_NodeWayInsertedTwice(t *testing.T) { } } -func TestComplete_OuterWayNotInserted(t *testing.T) { - // Outer way with different tag is not inserted twice into same table +func TestComplete_OuterWayInsertedTwice(t *testing.T) { + // Outer way with different tag value is inserted twice into same table + // behavior changed from pre-old-style-mp-removal: + // test outer way not inserted (different tag but same table) assertRecords(t, []checkElem{ {"osm_landusages", -19001, "farmland", nil}, {"osm_landusages", 19002, "farmyard", nil}, - {"osm_landusages", 19001, Missing, nil}, + {"osm_landusages", 19001, "farm", nil}, }) } @@ -206,13 +208,13 @@ func TestComplete_WayRelRefAfterDelete1(t *testing.T) { }) } -func TestComplete_RelationWayNotInserted(t *testing.T) { - // Part of relation was inserted only once. +func TestComplete_OldStyleMpRelationWayInserted(t *testing.T) { + // Old-style-mp: Part of relation is now inserted. assertRecords(t, []checkElem{ {"osm_landusages", -9001, "park", map[string]string{"name": "rel 9001"}}, - {"osm_landusages", 9009, Missing, nil}, + {"osm_landusages", 9009, "park", map[string]string{"name": "way 9009"}}, {"osm_landusages", -9101, "park", map[string]string{"name": "rel 9101"}}, - {"osm_landusages", 9109, Missing, nil}, + {"osm_landusages", 9109, "park", map[string]string{"name": "way 9109"}}, {"osm_landusages", 9110, "scrub", nil}, }) } diff --git a/update/deleter.go b/update/deleter.go index fb64c6e..0e84e7f 100644 --- a/update/deleter.go +++ b/update/deleter.go @@ -125,32 +125,6 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) } } - if deleteMembers && deletedPolygon { - // delete members from db and force reinsert of members - // use case: relation is deleted and member now stands for its own - - // TODO: still needed after old-style mp removal, remove when #148 is closed - for _, member := range elem.Members { - if member.Type == element.WAY { - d.deletedMembers[member.Id] = struct{}{} - if _, ok := d.deletedWays[member.Id]; ok { - continue - } - for _, r := range d.diffCache.Ways.Get(member.Id) { - if err := d.deleteRelation(r, false, false); err != nil { - return err - } - } - if err := d.deleteWay(member.Id, false); err != nil { - return err - } - } - } - } - - if err := d.osmCache.InsertedWays.DeleteMembers(elem.Members); err != nil { - return err - } if deleted && d.expireor != nil { if err := d.osmCache.Ways.FillMembers(elem.Members); err != nil { return err diff --git a/writer/relations.go b/writer/relations.go index d2910f7..b2a690e 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -197,12 +197,6 @@ func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geo } } - for _, m := range mapping.SelectRelationPolygons(rw.polygonMatcher, r) { - err = rw.osmCache.InsertedWays.PutWay(m.Way) - if err != nil { - log.Warn(err) - } - } return true } diff --git a/writer/ways.go b/writer/ways.go index 25b63e5..3ba3af4 100644 --- a/writer/ways.go +++ b/writer/ways.go @@ -72,11 +72,6 @@ func (ww *WayWriter) loop() { if len(w.Tags) == 0 { continue } - insertedAsRelation, err := ww.osmCache.InsertedWays.IsInserted(w.Id) - if err != nil { - log.Warn(err) - continue - } filled := false // fill loads all coords. call only if we have a match @@ -95,6 +90,7 @@ func (ww *WayWriter) loop() { w.Id = ww.wayId(w.Id) + var err error inserted := false insertedPolygon := false if matches := ww.lineMatcher.MatchWay(w); len(matches) > 0 { @@ -109,20 +105,17 @@ func (ww *WayWriter) loop() { continue } } - if !insertedAsRelation { - // only add polygons that were not inserted as a MultiPolygon relation - if matches := ww.polygonMatcher.MatchWay(w); len(matches) > 0 { - if !fill(w) { - continue - } - if w.IsClosed() { - err, insertedPolygon = ww.buildAndInsert(geos, w, matches, true) - if err != nil { - if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { - log.Warn(err) - } - continue + if matches := ww.polygonMatcher.MatchWay(w); len(matches) > 0 { + if !fill(w) { + continue + } + if w.IsClosed() { + err, insertedPolygon = ww.buildAndInsert(geos, w, matches, true) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) } + continue } } }