From 766118b324ae230d9f19dd1ff353a715e5d3e993 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Fri, 8 Jan 2016 16:08:00 +0100 Subject: [PATCH] update relations when member node changed --- cache/diff.go | 53 +++++++++++++++++++++++++++++++++---- diff/deleter.go | 9 +++++++ diff/process.go | 24 ++++++++++------- test/complete_db.osc | 7 +++++ test/complete_db.osm | 20 ++++++++++++++ test/completedb_test.go | 12 +++++++++ test/route_relation.osc | 32 ++++++++++++++++++++++ test/route_relation.osm | 12 +++++++++ test/route_relation_test.go | 29 +++++++++++++++----- writer/relations.go | 1 + 10 files changed, 178 insertions(+), 21 deletions(-) diff --git a/cache/diff.go b/cache/diff.go index 0060c85..128e260 100644 --- a/cache/diff.go +++ b/cache/diff.go @@ -1,7 +1,6 @@ package cache import ( - "github.com/jmhodges/levigo" "log" "os" "path/filepath" @@ -9,6 +8,8 @@ import ( "sort" "sync" + "github.com/jmhodges/levigo" + "github.com/omniscale/imposm3/cache/binary" "github.com/omniscale/imposm3/element" ) @@ -20,10 +21,11 @@ func (a byInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a byInt64) Less(i, j int) bool { return a[i] < a[j] } type DiffCache struct { - Dir string - Coords *CoordsRefIndex - Ways *WaysRefIndex - opened bool + Dir string + Coords *CoordsRefIndex // Stores which ways a coord references + CoordsRel *CoordsRelRefIndex // Stores which relations a coord references + Ways *WaysRefIndex // Stores which relations a way references + opened bool } func NewDiffCache(dir string) *DiffCache { @@ -36,6 +38,10 @@ func (c *DiffCache) Close() { c.Coords.Close() c.Coords = nil } + if c.CoordsRel != nil { + c.CoordsRel.Close() + c.CoordsRel = nil + } if c.Ways != nil { c.Ways.Close() c.Ways = nil @@ -46,6 +52,9 @@ func (c *DiffCache) Flush() { if c.Coords != nil { c.Coords.Flush() } + if c.CoordsRel != nil { + c.CoordsRel.Flush() + } if c.Ways != nil { c.Ways.Flush() } @@ -58,6 +67,11 @@ func (c *DiffCache) Open() error { c.Close() return err } + c.CoordsRel, err = newCoordsRelRefIndex(filepath.Join(c.Dir, "coords_rel_index")) + if err != nil { + c.Close() + return err + } c.Ways, err = newWaysRefIndex(filepath.Join(c.Dir, "ways_index")) if err != nil { c.Close() @@ -74,6 +88,9 @@ func (c *DiffCache) Exists() bool { if _, err := os.Stat(filepath.Join(c.Dir, "coords_index")); !os.IsNotExist(err) { return true } + if _, err := os.Stat(filepath.Join(c.Dir, "coords_rel_index")); !os.IsNotExist(err) { + return true + } if _, err := os.Stat(filepath.Join(c.Dir, "ways_index")); !os.IsNotExist(err) { return true } @@ -87,6 +104,9 @@ func (c *DiffCache) Remove() error { if err := os.RemoveAll(filepath.Join(c.Dir, "coords_index")); err != nil { return err } + if err := os.RemoveAll(filepath.Join(c.Dir, "coords_rel_index")); err != nil { + return err + } if err := os.RemoveAll(filepath.Join(c.Dir, "ways_index")); err != nil { return err } @@ -198,6 +218,9 @@ func newRefIndex(path string, opts *cacheOptions) (*bunchRefCache, error) { type CoordsRefIndex struct { bunchRefCache } +type CoordsRelRefIndex struct { + bunchRefCache +} type WaysRefIndex struct { bunchRefCache } @@ -210,6 +233,14 @@ func newCoordsRefIndex(dir string) (*CoordsRefIndex, error) { return &CoordsRefIndex{*cache}, nil } +func newCoordsRelRefIndex(dir string) (*CoordsRelRefIndex, error) { + cache, err := newRefIndex(dir, &globalCacheOptions.CoordsIndex) + if err != nil { + return nil, err + } + return &CoordsRelRefIndex{*cache}, nil +} + func newWaysRefIndex(dir string) (*WaysRefIndex, error) { cache, err := newRefIndex(dir, &globalCacheOptions.WaysIndex) if err != nil { @@ -365,6 +396,18 @@ func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) { } } +func (index *CoordsRelRefIndex) AddFromMembers(relId int64, members []element.Member) { + for _, member := range members { + if member.Type == element.NODE { + if index.linearImport { + index.addc <- idRef{id: member.Id, ref: relId} + } else { + index.Add(member.Id, relId) + } + } + } +} + func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) { for _, member := range members { if member.Type == element.WAY { diff --git a/diff/deleter.go b/diff/deleter.go index 99482c3..bf60f1e 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -258,6 +258,15 @@ func (d *Deleter) Delete(delElem parser.DiffElem) error { } } } + dependers = d.diffCache.CoordsRel.Get(delElem.Node.Id) + for _, rel := range dependers { + if _, ok := d.deletedRelations[rel]; ok { + continue + } + if err := d.deleteRelation(rel, false, false); err != nil { + return err + } + } } if !delElem.Add { if err := d.diffCache.Coords.Delete(delElem.Node.Id); err != nil { diff --git a/diff/process.go b/diff/process.go index c15da9a..6302756 100644 --- a/diff/process.go +++ b/diff/process.go @@ -175,9 +175,9 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi nodeWriter.SetExpireor(expireor) nodeWriter.Start() - nodeIds := make(map[int64]bool) - wayIds := make(map[int64]bool) - relIds := make(map[int64]bool) + nodeIds := make(map[int64]struct{}) + wayIds := make(map[int64]struct{}) + relIds := make(map[int64]struct{}) step := log.StartStep("Parsing changes, updating cache and removing elements") @@ -251,7 +251,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put relation %v", elem.Rel) } - relIds[elem.Rel.Id] = true + relIds[elem.Rel.Id] = struct{}{} } } else if elem.Way != nil { // check if first coord is cached to avoid caching @@ -265,7 +265,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put way %v", elem.Way) } - wayIds[elem.Way.Id] = true + wayIds[elem.Way.Id] = struct{}{} } } else if elem.Node != nil { addNode := true @@ -283,7 +283,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi if err != nil { return diffError(err, "put coord %v", elem.Node) } - nodeIds[elem.Node.Id] = true + nodeIds[elem.Node.Id] = struct{}{} } } } @@ -301,7 +301,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi // mark member ways from deleted relations for re-insert for id, _ := range deleter.DeletedMemberWays() { - wayIds[id] = true + wayIds[id] = struct{}{} } progress.Stop() @@ -314,16 +314,22 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi for nodeId, _ := range nodeIds { dependers := diffCache.Coords.Get(nodeId) for _, way := range dependers { - wayIds[way] = true + wayIds[way] = struct{}{} } } // mark depending relations for (re)insert + for nodeId, _ := range nodeIds { + dependers := diffCache.CoordsRel.Get(nodeId) + for _, rel := range dependers { + relIds[rel] = struct{}{} + } + } for wayId, _ := range wayIds { dependers := diffCache.Ways.Get(wayId) // mark depending relations for (re)insert for _, rel := range dependers { - relIds[rel] = true + relIds[rel] = struct{}{} } } diff --git a/test/complete_db.osc b/test/complete_db.osc index b3c8c1c..3999726 100644 --- a/test/complete_db.osc +++ b/test/complete_db.osc @@ -225,4 +225,11 @@ + + + + + + + diff --git a/test/complete_db.osm b/test/complete_db.osm index a6b7b8c..5d4e2cd 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -1019,6 +1019,26 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test/completedb_test.go b/test/completedb_test.go index e28aded..6b6b794 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -297,6 +297,12 @@ func TestDuplicateIds(t *testing.T) { }) } +func TestRelationUpdatedByNode(t *testing.T) { + // Relations was updated after modified node. + + assertArea(t, checkElem{"osm_buildings", -52121, "yes", nil}, 13653930440.868315) +} + func TestGeneralizedBananaPolygonIsValid(t *testing.T) { // Generalized polygons are valid. @@ -605,6 +611,12 @@ func TestDuplicateIds2(t *testing.T) { }) } +func TestRelationUpdatedByNode2(t *testing.T) { + // Relations was updated after modified node. + + assertArea(t, checkElem{"osm_buildings", -52121, "yes", nil}, 16276875196.653734) +} + func TestUpdatedWay2(t *testing.T) { // All nodes of straightened way are updated. diff --git a/test/route_relation.osc b/test/route_relation.osc index ff6ad12..66d3456 100644 --- a/test/route_relation.osc +++ b/test/route_relation.osc @@ -9,4 +9,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/route_relation.osm b/test/route_relation.osm index 25c42cc..608a5af 100644 --- a/test/route_relation.osm +++ b/test/route_relation.osm @@ -128,4 +128,16 @@ + + + + + + + + + + + + diff --git a/test/route_relation_test.go b/test/route_relation_test.go index a03facc..89b796c 100644 --- a/test/route_relation_test.go +++ b/test/route_relation_test.go @@ -2,7 +2,6 @@ package test import ( "database/sql" - "strconv" "testing" @@ -92,6 +91,7 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { t.Fatal(g.Length()) } + // tag from member is updated rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100503") if len(rows) != 1 { t.Fatal(rows) @@ -99,16 +99,31 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { if rows[0]["name"] != "new name" { t.Error(rows[0]) } -} -func TestRouteRelation_MemberNotUpdated(t *testing.T) { - // check that member is not updated if no node/way changed - rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -100903 AND member = 100501") + // member is removed + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100512") + if len(rows) != 0 { + t.Fatal(rows) + } + + // role from member is updated + rows = ts.queryDynamic(t, "osm_route_members", "osm_id = -100902 AND member = 100102") if len(rows) != 1 { t.Fatal(rows) } - if id, err := strconv.ParseInt(rows[0]["id"], 10, 32); err != nil || id > 27 { - t.Error("member was re-inserted", rows) + if rows[0]["role"] != "halt" { + t.Error(rows[0]) } } + +func TestRouteRelation_MemberUpdatedByNode(t *testing.T) { + // check that member is updated after node was modified + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101") + if len(rows) != 1 { + t.Fatal(rows) + } + if rows[0]["name"] != "Stop2" { + t.Error(rows[0]) + } +} diff --git a/writer/relations.go b/writer/relations.go index 9c8b92d..1925c87 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -115,6 +115,7 @@ NextRel: if inserted && rw.diffCache != nil { rw.diffCache.Ways.AddFromMembers(r.Id, allMembers) + rw.diffCache.CoordsRel.AddFromMembers(r.Id, allMembers) for _, member := range allMembers { if member.Way != nil { rw.diffCache.Coords.AddFromWay(member.Way)