vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix duplicate inserts of ways and relations when handling diffs 

closes #65 and #66
master
Oliver Tonnhofer 2015-09-29 13:58:06 +02:00
parent cc686a63c0
commit fa99248c7a
6 changed files with 114 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
diff/deleter.go
View File

@ -116,6 +116,9 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
if _, ok := d.deletedWays[member.Id]; ok {
continue
}
if err := d.deleteRelation(member.Id, false, false); err != nil {
return err
}
if err := d.deleteWay(member.Id, false); err != nil {
return err
}
@ -212,10 +215,6 @@ func (d *Deleter) deleteNode(id int64) error {
}
func (d *Deleter) Delete(delElem parser.DiffElem) error {
if !delElem.Del {
panic("del=false")
}
if delElem.Rel != nil {
if err := d.deleteRelation(delElem.Rel.Id, true, true); err != nil {
return err

9
diff/process.go
View File

@ -156,10 +156,13 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
}
progress.AddCoords(1)
}
// always delete, to prevent duplicate elements from overlap of initial
// import and diff import
if err := deleter.Delete(elem); err != nil {
return diffError(err, "delete element", elem)
}
if elem.Del {
if err := deleter.Delete(elem); err != nil {
return diffError(err, "delete element", elem)
}
if !elem.Add {
// no new or modified elem -> remove from cache
if elem.Rel != nil {

32
test/complete_db.osc
View File

@ -185,4 +185,36 @@
</node>
</modify>
<!-- test if additional create inserts duplicate elements (checks #66) -->
<create>
<node id="201001" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="10"/>
<node id="201002" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="11"/>
<node id="201003" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="10"/>
<node id="201004" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="11"/>
<way id="201001" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="201001"/>
<nd ref="201002"/>
<nd ref="201003"/>
<nd ref="201004"/>
<nd ref="201001"/>
<tag k="highway" v="residential"/>
</way>
<relation id="201001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="201001" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="park"/>
</relation>
</create>
<!-- test modification of one relation (201102) does not duplicate
relation (201101) with shared way (checks #65) -->
<modify>
<relation id="201102" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="201101" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="forest"/>
</relation>
</modify>
</osmChange>

43
test/complete_db.osm
View File

@ -1052,5 +1052,48 @@
<tag k="landuse" v="wood"/>
</way>
<!-- test if additional create inserts duplicate elements (checks #66) -->
<node id="201001" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="10"/>
<node id="201002" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="11"/>
<node id="201003" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="10"/>
<node id="201004" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="11"/>
<way id="201001" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="201001"/>
<nd ref="201002"/>
<nd ref="201003"/>
<nd ref="201004"/>
<nd ref="201001"/>
<tag k="highway" v="residential"/>
</way>
<relation id="201001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="201001" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="park"/>
</relation>
<!-- test modification of one relation (201102) does not duplicate
relation (201101) with shared way (checks #65) -->
<node id="201101" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="10"/>
<node id="201102" version="1" timestamp="2011-11-11T00:11:11Z" lat="32" lon="11"/>
<node id="201103" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="10"/>
<node id="201104" version="1" timestamp="2011-11-11T00:11:11Z" lat="34" lon="11"/>
<way id="201101" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="201101"/>
<nd ref="201102"/>
<nd ref="201103"/>
<nd ref="201104"/>
<nd ref="201101"/>
<tag k="highway" v="residential"/>
</way>
<relation id="201101" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="201101" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="park"/>
</relation>
<relation id="201102" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="201101" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="forest"/>
</relation>
</osm>

20
test/complete_db_test.py
View File

@ -311,6 +311,18 @@ def test_update():
t.imposm3_update(t.db_conf, './build/complete_db.osc.gz', mapping_file)
#######################################################################
def test_no_duplicates():
"""
Relations/ways are only inserted once
Checks #66
"""
highways = t.query_duplicates(t.db_conf, 'osm_roads')
# one duplicate for test_node_way_inserted_twice is expected
assert highways == [[18001, 2]]
landusages = t.query_duplicates(t.db_conf, 'osm_landusages')
assert not landusages
def test_updated_landusage():
"""Multipolygon relation was modified"""
t.assert_cached_node(1001, (13.5, 47.5))
@ -463,6 +475,14 @@ def test_update_node_to_coord_2():
assert not t.query_row(t.db_conf, 'osm_amenities', 70001)
assert t.query_row(t.db_conf, 'osm_amenities', 70002)
def test_no_duplicate_insert():
"""
Relation is not inserted again if a nother relation with the same way was modified
Checks #65
"""
assert t.query_row(t.db_conf, 'osm_landusages', -201101)['type'] == 'park'
assert t.query_row(t.db_conf, 'osm_landusages', -201102)['type'] == 'forest'
assert t.query_row(t.db_conf, 'osm_roads', 201101)['type'] == 'residential'
#######################################################################
def test_deploy_and_revert_deploy():

10
test/helper.py
View File

@ -90,6 +90,16 @@ def query_row(db_conf, table, osmid):
return results[0]
return results
def query_duplicates(db_conf, table):
conn = _test_connection(db_conf)
cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute('select osm_id, count(osm_id) from %s.%s group by osm_id having count(osm_id) > 1' % (TEST_SCHEMA_PRODUCTION, table))
results = []
for row in cur.fetchall():
results.append(row)
cur.close()
return results
def imposm3_import(db_conf, pbf, mapping_file):
_close_test_connection(db_conf)
conn = pg_db_url(db_conf)