From f2d40b047b541d321d70a42a04b033669d7853bc Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 28 Apr 2014 11:43:25 +0200 Subject: [PATCH] log more sql errors during import and diff updates --- database/database.go | 12 ++--- database/postgis/postgis.go | 21 +++++--- database/postgis/router.go | 8 +-- diff/deleter.go | 103 +++++++++++++++++++++++------------- diff/process.go | 4 +- writer/nodes.go | 17 +++--- writer/relations.go | 31 ++++++----- writer/ways.go | 49 ++++++++++------- 8 files changed, 153 insertions(+), 92 deletions(-) diff --git a/database/database.go b/database/database.go index 20e4d22..0078e90 100644 --- a/database/database.go +++ b/database/database.go @@ -37,9 +37,9 @@ type Inserter interface { ProbePolygon(element.OSMElem) (bool, interface{}) // InsertXxx inserts element of that type into the database. // element.Geom is set to that type. - InsertPoint(element.OSMElem, interface{}) - InsertLineString(element.OSMElem, interface{}) - InsertPolygon(element.OSMElem, interface{}) + InsertPoint(element.OSMElem, interface{}) error + InsertLineString(element.OSMElem, interface{}) error + InsertPolygon(element.OSMElem, interface{}) error // SelectRelationPolygons returns a slice of all members that are already // imported with a relation with tags. SelectRelationPolygons(element.Tags, []element.Member) []element.Member @@ -107,9 +107,9 @@ func (n *nullDb) Begin() error func (n *nullDb) End() error { return nil } func (n *nullDb) Close() error { return nil } func (n *nullDb) Abort() error { return nil } -func (n *nullDb) InsertPoint(element.OSMElem, interface{}) {} -func (n *nullDb) InsertLineString(element.OSMElem, interface{}) {} -func (n *nullDb) InsertPolygon(element.OSMElem, interface{}) {} +func (n *nullDb) InsertPoint(element.OSMElem, interface{}) error { return nil } +func (n *nullDb) InsertLineString(element.OSMElem, interface{}) error { return nil } +func (n *nullDb) InsertPolygon(element.OSMElem, interface{}) error { return nil } func (n *nullDb) ProbePoint(element.OSMElem) (bool, interface{}) { return true, nil } func (n *nullDb) ProbeLineString(element.OSMElem) (bool, interface{}) { return true, nil } func (n *nullDb) ProbePolygon(element.OSMElem) (bool, interface{}) { return true, nil } diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 965c71e..500703b 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -416,20 +416,25 @@ func (pg *PostGIS) Open() error { return nil } -func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches interface{}) { +func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches interface{}) error { if matches, ok := matches.([]mapping.Match); ok { for _, match := range matches { row := match.Row(&elem) - pg.txRouter.Insert(match.Table.Name, row) + if err := pg.txRouter.Insert(match.Table.Name, row); err != nil { + return err + } } } + return nil } -func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) { +func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) error { if matches, ok := matches.([]mapping.Match); ok { for _, match := range matches { row := match.Row(&elem) - pg.txRouter.Insert(match.Table.Name, row) + if err := pg.txRouter.Insert(match.Table.Name, row); err != nil { + return err + } } if pg.updateGeneralizedTables { for _, generalizedTable := range pg.generalizedFromMatches(matches) { @@ -438,13 +443,16 @@ func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) { } } + return nil } -func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) { +func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) error { if matches, ok := matches.([]mapping.Match); ok { for _, match := range matches { row := match.Row(&elem) - pg.txRouter.Insert(match.Table.Name, row) + if err := pg.txRouter.Insert(match.Table.Name, row); err != nil { + return err + } } if pg.updateGeneralizedTables { for _, generalizedTable := range pg.generalizedFromMatches(matches) { @@ -453,6 +461,7 @@ func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) { } } + return nil } func (pg *PostGIS) ProbePoint(elem element.OSMElem) (bool, interface{}) { diff --git a/database/postgis/router.go b/database/postgis/router.go index 5ffb517..e9f28c9 100644 --- a/database/postgis/router.go +++ b/database/postgis/router.go @@ -80,18 +80,18 @@ func (txr *TxRouter) Abort() error { return nil } -func (txr *TxRouter) Insert(table string, row []interface{}) { +func (txr *TxRouter) Insert(table string, row []interface{}) error { tt, ok := txr.Tables[table] if !ok { panic("unknown table " + table) } - tt.Insert(row) + return tt.Insert(row) } -func (txr *TxRouter) Delete(table string, id int64) { +func (txr *TxRouter) Delete(table string, id int64) error { tt, ok := txr.Tables[table] if !ok { panic("unknown table " + table) } - tt.Delete(id) + return tt.Delete(id) } diff --git a/diff/deleter.go b/diff/deleter.go index e56438d..5aa5b4a 100644 --- a/diff/deleter.go +++ b/diff/deleter.go @@ -50,35 +50,39 @@ func (d *Deleter) DeletedMemberWays() map[int64]struct{} { return d.deletedMembers } -func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) { +func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) error { d.deletedRelations[id] = struct{}{} elem, err := d.osmCache.Relations.GetRelation(id) if err != nil { if err == cache.NotFound { - return + return nil } - // TODO - log.Print("rel", id, err) - return + return err } if elem.Tags == nil { - return + return nil } if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok { - d.delDb.Delete(-elem.Id, matches) + if err := d.delDb.Delete(-elem.Id, matches); err != nil { + return err + } } else { // handle relations with tags from members by deleting // from all tables e := element.OSMElem(elem.OSMElem) e.Id = -e.Id - d.delDb.DeleteElem(e) + if err := d.delDb.DeleteElem(e); err != nil { + return err + } } if deleteRefs { for _, m := range elem.Members { if m.Type == element.WAY { - d.diffCache.Ways.DeleteRef(m.Id, id) + if err := d.diffCache.Ways.DeleteRef(m.Id, id); err != nil { + return err + } } } } @@ -92,12 +96,16 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) if _, ok := d.deletedWays[member.Id]; ok { continue } - d.deleteWay(member.Id, false) + if err := d.deleteWay(member.Id, false); err != nil { + return err + } } } } - d.osmCache.InsertedWays.DeleteMembers(elem.Members) + if err := d.osmCache.InsertedWays.DeleteMembers(elem.Members); err != nil { + return err + } if d.expireor != nil { for _, m := range elem.Members { if m.Way == nil { @@ -111,81 +119,91 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) expire.ExpireNodes(d.expireor, m.Way.Nodes) } } + return nil } -func (d *Deleter) deleteWay(id int64, deleteRefs bool) { +func (d *Deleter) deleteWay(id int64, deleteRefs bool) error { d.deletedWays[id] = struct{}{} elem, err := d.osmCache.Ways.GetWay(id) if err != nil { if err == cache.NotFound { - return + return nil } - // TODO - log.Print("way", id, err) - return + return err } if elem.Tags == nil { - return + return nil } deleted := false if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok { - d.delDb.Delete(elem.Id, matches) + if err := d.delDb.Delete(elem.Id, matches); err != nil { + return err + } deleted = true } if ok, matches := d.delDb.ProbeLineString(elem.OSMElem); ok { - d.delDb.Delete(elem.Id, matches) + if err := d.delDb.Delete(elem.Id, matches); err != nil { + return err + } deleted = true } if deleted && deleteRefs { for _, n := range elem.Refs { - d.diffCache.Coords.DeleteRef(n, id) + if err := d.diffCache.Coords.DeleteRef(n, id); err != nil { + return err + } } } if deleted && d.expireor != nil { err := d.osmCache.Coords.FillWay(elem) if err != nil { - return + return err } expire.ExpireNodes(d.expireor, elem.Nodes) } + return nil } -func (d *Deleter) deleteNode(id int64) { +func (d *Deleter) deleteNode(id int64) error { elem, err := d.osmCache.Nodes.GetNode(id) if err != nil { if err == cache.NotFound { - return + return nil } - // TODO - log.Print("node", id, err) - return + return err } if elem.Tags == nil { - return + return nil } deleted := false if ok, matches := d.delDb.ProbePoint(elem.OSMElem); ok { - d.delDb.Delete(elem.Id, matches) + if err := d.delDb.Delete(elem.Id, matches); err != nil { + return err + } deleted = true } if deleted && d.expireor != nil { d.expireor.Expire(elem.Long, elem.Lat) } - + return nil } -func (d *Deleter) Delete(delElem parser.DiffElem) { +func (d *Deleter) Delete(delElem parser.DiffElem) error { if !delElem.Del { panic("del=false") } if delElem.Rel != nil { - d.deleteRelation(delElem.Rel.Id, true, true) + if err := d.deleteRelation(delElem.Rel.Id, true, true); err != nil { + return err + } } else if delElem.Way != nil { - d.deleteWay(delElem.Way.Id, true) + if err := d.deleteWay(delElem.Way.Id, true); err != nil { + return err + } if delElem.Mod { dependers := d.diffCache.Ways.Get(delElem.Way.Id) @@ -193,18 +211,24 @@ func (d *Deleter) Delete(delElem parser.DiffElem) { if _, ok := d.deletedRelations[rel]; ok { continue } - d.deleteRelation(rel, false, false) + if err := d.deleteRelation(rel, false, false); err != nil { + return err + } } } } else if delElem.Node != nil { - d.deleteNode(delElem.Node.Id) + if err := d.deleteNode(delElem.Node.Id); err != nil { + return err + } if delElem.Mod { dependers := d.diffCache.Coords.Get(delElem.Node.Id) for _, way := range dependers { if _, ok := d.deletedWays[way]; ok { continue } - d.deleteWay(way, false) + if err := d.deleteWay(way, false); err != nil { + return err + } dependers := d.diffCache.Ways.Get(way) if len(dependers) >= 1 { // mark member ways from deleted relations for re-insert @@ -214,12 +238,17 @@ func (d *Deleter) Delete(delElem parser.DiffElem) { if _, ok := d.deletedRelations[rel]; ok { continue } - d.deleteRelation(rel, false, false) + if err := d.deleteRelation(rel, false, false); err != nil { + return err + } } } } if !delElem.Add { - d.diffCache.Coords.Delete(delElem.Node.Id) + if err := d.diffCache.Coords.Delete(delElem.Node.Id); err != nil { + return err + } } } + return nil } diff --git a/diff/process.go b/diff/process.go index 7f5154b..aa688e8 100644 --- a/diff/process.go +++ b/diff/process.go @@ -141,7 +141,9 @@ For: progress.AddCoords(1) } if elem.Del { - deleter.Delete(elem) + if err := deleter.Delete(elem); err != nil { + return err + } if !elem.Add { if elem.Rel != nil { if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil { diff --git a/writer/nodes.go b/writer/nodes.go index 56e82f6..2958a25 100644 --- a/writer/nodes.go +++ b/writer/nodes.go @@ -47,12 +47,9 @@ func (nw *NodeWriter) loop() { } point, err := geom.Point(geos, *n) if err != nil { - if err, ok := err.(ErrorLevel); ok { - if err.Level() <= 0 { - continue - } + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) } - log.Warn(err) continue } @@ -69,10 +66,16 @@ func (nw *NodeWriter) loop() { continue } if len(parts) >= 1 { - nw.inserter.InsertPoint(n.OSMElem, matches) + if err := nw.inserter.InsertPoint(n.OSMElem, matches); err != nil { + log.Warn(err) + continue + } } } else { - nw.inserter.InsertPoint(n.OSMElem, matches) + if err := nw.inserter.InsertPoint(n.OSMElem, matches); err != nil { + log.Warn(err) + continue + } } } diff --git a/writer/relations.go b/writer/relations.go index e3bcaf6..6058c40 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -73,12 +73,9 @@ NextRel: // relation tags) prepedRel, err := geom.PrepareRelation(r, rw.srid) if err != nil { - if err, ok := err.(ErrorLevel); ok { - if err.Level() <= 0 { - continue NextRel - } + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) } - log.Warn(err) continue NextRel } @@ -94,13 +91,9 @@ NextRel: if r.Geom != nil && r.Geom.Geom != nil { geos.Destroy(r.Geom.Geom) } - if err, ok := err.(ErrorLevel); ok { - if err.Level() <= 0 { - continue NextRel - - } + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) } - log.Warn(err) continue NextRel } @@ -118,12 +111,24 @@ NextRel: rel := element.Relation(*r) rel.Id = -r.Id rel.Geom = &element.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)} - rw.inserter.InsertPolygon(rel.OSMElem, matches) + err := rw.inserter.InsertPolygon(rel.OSMElem, matches) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + continue + } } } else { rel := element.Relation(*r) rel.Id = -r.Id - rw.inserter.InsertPolygon(rel.OSMElem, matches) + err := rw.inserter.InsertPolygon(rel.OSMElem, matches) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + continue + } } for _, m := range rw.inserter.SelectRelationPolygons(r.Tags, r.Members) { diff --git a/writer/ways.go b/writer/ways.go index c75a446..cde1d1f 100644 --- a/writer/ways.go +++ b/writer/ways.go @@ -58,13 +58,25 @@ func (ww *WayWriter) loop() { inserted := false if ok, matches := ww.inserter.ProbeLineString(w.OSMElem); ok { - ww.buildAndInsert(geos, w, matches, false) + err := ww.buildAndInsert(geos, w, matches, false) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + continue + } inserted = true } if w.IsClosed() && !insertedAsRelation { // only add polygons that were not inserted as a MultiPolygon relation if ok, matches := ww.inserter.ProbePolygon(w.OSMElem); ok { - ww.buildAndInsert(geos, w, matches, true) + err := ww.buildAndInsert(geos, w, matches, true) + if err != nil { + if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { + log.Warn(err) + } + continue + } inserted = true } } @@ -79,7 +91,7 @@ func (ww *WayWriter) loop() { ww.wg.Done() } -func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interface{}, isPolygon bool) { +func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interface{}, isPolygon bool) error { var err error var geosgeom *geos.Geom // make copy to avoid interference with polygon/linestring matches @@ -91,41 +103,42 @@ func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interf geosgeom, err = geom.LineString(g, way.Nodes) } if err != nil { - if err, ok := err.(ErrorLevel); ok { - if err.Level() <= 0 { - return - } - } - log.Warn(err) - return + return err } way.Geom, err = geom.AsGeomElement(g, geosgeom) if err != nil { - log.Warn(err) - return + return err } if ww.limiter != nil { parts, err := ww.limiter.Clip(way.Geom.Geom) if err != nil { - log.Warn(err) - return + return err } for _, p := range parts { way := element.Way(*w) way.Geom = &element.Geometry{Geom: p, Wkb: g.AsEwkbHex(p)} if isPolygon { - ww.inserter.InsertPolygon(way.OSMElem, matches) + if err := ww.inserter.InsertPolygon(way.OSMElem, matches); err != nil { + return err + } } else { - ww.inserter.InsertLineString(way.OSMElem, matches) + if err := ww.inserter.InsertLineString(way.OSMElem, matches); err != nil { + return err + } } } } else { if isPolygon { - ww.inserter.InsertPolygon(way.OSMElem, matches) + if err := ww.inserter.InsertPolygon(way.OSMElem, matches); err != nil { + return err + } } else { - ww.inserter.InsertLineString(way.OSMElem, matches) + if err := ww.inserter.InsertLineString(way.OSMElem, matches); err != nil { + return err + } } } + return nil }