vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

log more sql errors during import and diff updates

master
Oliver Tonnhofer 2014-04-28 11:43:25 +02:00
parent 90154d7c51
commit f2d40b047b
8 changed files with 153 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
database/database.go
View File

@ -37,9 +37,9 @@ type Inserter interface {
ProbePolygon(element.OSMElem) (bool, interface{})
// InsertXxx inserts element of that type into the database.
// element.Geom is set to that type.
InsertPoint(element.OSMElem, interface{})
InsertLineString(element.OSMElem, interface{})
InsertPolygon(element.OSMElem, interface{})
InsertPoint(element.OSMElem, interface{}) error
InsertLineString(element.OSMElem, interface{}) error
InsertPolygon(element.OSMElem, interface{}) error
// SelectRelationPolygons returns a slice of all members that are already
// imported with a relation with tags.
SelectRelationPolygons(element.Tags, []element.Member) []element.Member
@ -107,9 +107,9 @@ func (n *nullDb) Begin() error
func (n *nullDb) End() error { return nil }
func (n *nullDb) Close() error { return nil }
func (n *nullDb) Abort() error { return nil }
func (n *nullDb) InsertPoint(element.OSMElem, interface{}) {}
func (n *nullDb) InsertLineString(element.OSMElem, interface{}) {}
func (n *nullDb) InsertPolygon(element.OSMElem, interface{}) {}
func (n *nullDb) InsertPoint(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) InsertLineString(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) InsertPolygon(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) ProbePoint(element.OSMElem) (bool, interface{}) { return true, nil }
func (n *nullDb) ProbeLineString(element.OSMElem) (bool, interface{}) { return true, nil }
func (n *nullDb) ProbePolygon(element.OSMElem) (bool, interface{}) { return true, nil }

21
database/postgis/postgis.go
View File

@ -416,20 +416,25 @@ func (pg *PostGIS) Open() error {
return nil
}
func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches interface{}) {
func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
pg.txRouter.Insert(match.Table.Name, row)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
}
}
return nil
}
func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) {
func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
pg.txRouter.Insert(match.Table.Name, row)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
@ -438,13 +443,16 @@ func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) {
}
}
return nil
}
func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) {
func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
pg.txRouter.Insert(match.Table.Name, row)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
@ -453,6 +461,7 @@ func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) {
}
}
return nil
}
func (pg *PostGIS) ProbePoint(elem element.OSMElem) (bool, interface{}) {

8
database/postgis/router.go
View File

@ -80,18 +80,18 @@ func (txr *TxRouter) Abort() error {
return nil
}
func (txr *TxRouter) Insert(table string, row []interface{}) {
func (txr *TxRouter) Insert(table string, row []interface{}) error {
tt, ok := txr.Tables[table]
if !ok {
panic("unknown table " + table)
}
tt.Insert(row)
return tt.Insert(row)
}
func (txr *TxRouter) Delete(table string, id int64) {
func (txr *TxRouter) Delete(table string, id int64) error {
tt, ok := txr.Tables[table]
if !ok {
panic("unknown table " + table)
}
tt.Delete(id)
return tt.Delete(id)
}

103
diff/deleter.go
View File

@ -50,35 +50,39 @@ func (d *Deleter) DeletedMemberWays() map[int64]struct{} {
return d.deletedMembers
}
func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) {
func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool) error {
d.deletedRelations[id] = struct{}{}
elem, err := d.osmCache.Relations.GetRelation(id)
if err != nil {
if err == cache.NotFound {
return
return nil
}
// TODO
log.Print("rel", id, err)
return
return err
}
if elem.Tags == nil {
return
return nil
}
if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok {
d.delDb.Delete(-elem.Id, matches)
if err := d.delDb.Delete(-elem.Id, matches); err != nil {
return err
}
} else {
// handle relations with tags from members by deleting
// from all tables
e := element.OSMElem(elem.OSMElem)
e.Id = -e.Id
d.delDb.DeleteElem(e)
if err := d.delDb.DeleteElem(e); err != nil {
return err
}
}
if deleteRefs {
for _, m := range elem.Members {
if m.Type == element.WAY {
d.diffCache.Ways.DeleteRef(m.Id, id)
if err := d.diffCache.Ways.DeleteRef(m.Id, id); err != nil {
return err
}
}
}
}
@ -92,12 +96,16 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
if _, ok := d.deletedWays[member.Id]; ok {
continue
}
d.deleteWay(member.Id, false)
if err := d.deleteWay(member.Id, false); err != nil {
return err
}
}
}
}
d.osmCache.InsertedWays.DeleteMembers(elem.Members)
if err := d.osmCache.InsertedWays.DeleteMembers(elem.Members); err != nil {
return err
}
if d.expireor != nil {
for _, m := range elem.Members {
if m.Way == nil {
@ -111,81 +119,91 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
expire.ExpireNodes(d.expireor, m.Way.Nodes)
}
}
return nil
}
func (d *Deleter) deleteWay(id int64, deleteRefs bool) {
func (d *Deleter) deleteWay(id int64, deleteRefs bool) error {
d.deletedWays[id] = struct{}{}
elem, err := d.osmCache.Ways.GetWay(id)
if err != nil {
if err == cache.NotFound {
return
return nil
}
// TODO
log.Print("way", id, err)
return
return err
}
if elem.Tags == nil {
return
return nil
}
deleted := false
if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok {
d.delDb.Delete(elem.Id, matches)
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}
deleted = true
}
if ok, matches := d.delDb.ProbeLineString(elem.OSMElem); ok {
d.delDb.Delete(elem.Id, matches)
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}
deleted = true
}
if deleted && deleteRefs {
for _, n := range elem.Refs {
d.diffCache.Coords.DeleteRef(n, id)
if err := d.diffCache.Coords.DeleteRef(n, id); err != nil {
return err
}
}
}
if deleted && d.expireor != nil {
err := d.osmCache.Coords.FillWay(elem)
if err != nil {
return
return err
}
expire.ExpireNodes(d.expireor, elem.Nodes)
}
return nil
}
func (d *Deleter) deleteNode(id int64) {
func (d *Deleter) deleteNode(id int64) error {
elem, err := d.osmCache.Nodes.GetNode(id)
if err != nil {
if err == cache.NotFound {
return
return nil
}
// TODO
log.Print("node", id, err)
return
return err
}
if elem.Tags == nil {
return
return nil
}
deleted := false
if ok, matches := d.delDb.ProbePoint(elem.OSMElem); ok {
d.delDb.Delete(elem.Id, matches)
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}
deleted = true
}
if deleted && d.expireor != nil {
d.expireor.Expire(elem.Long, elem.Lat)
}
return nil
}
func (d *Deleter) Delete(delElem parser.DiffElem) {
func (d *Deleter) Delete(delElem parser.DiffElem) error {
if !delElem.Del {
panic("del=false")
}
if delElem.Rel != nil {
d.deleteRelation(delElem.Rel.Id, true, true)
if err := d.deleteRelation(delElem.Rel.Id, true, true); err != nil {
return err
}
} else if delElem.Way != nil {
d.deleteWay(delElem.Way.Id, true)
if err := d.deleteWay(delElem.Way.Id, true); err != nil {
return err
}
if delElem.Mod {
dependers := d.diffCache.Ways.Get(delElem.Way.Id)
@ -193,18 +211,24 @@ func (d *Deleter) Delete(delElem parser.DiffElem) {
if _, ok := d.deletedRelations[rel]; ok {
continue
}
d.deleteRelation(rel, false, false)
if err := d.deleteRelation(rel, false, false); err != nil {
return err
}
}
}
} else if delElem.Node != nil {
d.deleteNode(delElem.Node.Id)
if err := d.deleteNode(delElem.Node.Id); err != nil {
return err
}
if delElem.Mod {
dependers := d.diffCache.Coords.Get(delElem.Node.Id)
for _, way := range dependers {
if _, ok := d.deletedWays[way]; ok {
continue
}
d.deleteWay(way, false)
if err := d.deleteWay(way, false); err != nil {
return err
}
dependers := d.diffCache.Ways.Get(way)
if len(dependers) >= 1 {
// mark member ways from deleted relations for re-insert
@ -214,12 +238,17 @@ func (d *Deleter) Delete(delElem parser.DiffElem) {
if _, ok := d.deletedRelations[rel]; ok {
continue
}
d.deleteRelation(rel, false, false)
if err := d.deleteRelation(rel, false, false); err != nil {
return err
}
}
}
}
if !delElem.Add {
d.diffCache.Coords.Delete(delElem.Node.Id)
if err := d.diffCache.Coords.Delete(delElem.Node.Id); err != nil {
return err
}
}
}
return nil
}

4
diff/process.go
View File

@ -141,7 +141,9 @@ For:
progress.AddCoords(1)
}
if elem.Del {
deleter.Delete(elem)
if err := deleter.Delete(elem); err != nil {
return err
}
if !elem.Add {
if elem.Rel != nil {
if err := osmCache.Relations.DeleteRelation(elem.Rel.Id); err != nil {

17
writer/nodes.go
View File

@ -47,12 +47,9 @@ func (nw *NodeWriter) loop() {
}
point, err := geom.Point(geos, *n)
if err != nil {
if err, ok := err.(ErrorLevel); ok {
if err.Level() <= 0 {
continue
}
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
log.Warn(err)
continue
}
@ -69,10 +66,16 @@ func (nw *NodeWriter) loop() {
continue
}
if len(parts) >= 1 {
nw.inserter.InsertPoint(n.OSMElem, matches)
if err := nw.inserter.InsertPoint(n.OSMElem, matches); err != nil {
log.Warn(err)
continue
}
}
} else {
nw.inserter.InsertPoint(n.OSMElem, matches)
if err := nw.inserter.InsertPoint(n.OSMElem, matches); err != nil {
log.Warn(err)
continue
}
}
}

31
writer/relations.go
View File

@ -73,12 +73,9 @@ NextRel:
// relation tags)
prepedRel, err := geom.PrepareRelation(r, rw.srid)
if err != nil {
if err, ok := err.(ErrorLevel); ok {
if err.Level() <= 0 {
continue NextRel
}
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
log.Warn(err)
continue NextRel
}
@ -94,13 +91,9 @@ NextRel:
if r.Geom != nil && r.Geom.Geom != nil {
geos.Destroy(r.Geom.Geom)
}
if err, ok := err.(ErrorLevel); ok {
if err.Level() <= 0 {
continue NextRel
}
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
log.Warn(err)
continue NextRel
}
@ -118,12 +111,24 @@ NextRel:
rel := element.Relation(*r)
rel.Id = -r.Id
rel.Geom = &element.Geometry{Geom: g, Wkb: geos.AsEwkbHex(g)}
rw.inserter.InsertPolygon(rel.OSMElem, matches)
err := rw.inserter.InsertPolygon(rel.OSMElem, matches)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
}
}
} else {
rel := element.Relation(*r)
rel.Id = -r.Id
rw.inserter.InsertPolygon(rel.OSMElem, matches)
err := rw.inserter.InsertPolygon(rel.OSMElem, matches)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
}
}
for _, m := range rw.inserter.SelectRelationPolygons(r.Tags, r.Members) {

49
writer/ways.go
View File

@ -58,13 +58,25 @@ func (ww *WayWriter) loop() {
inserted := false
if ok, matches := ww.inserter.ProbeLineString(w.OSMElem); ok {
ww.buildAndInsert(geos, w, matches, false)
err := ww.buildAndInsert(geos, w, matches, false)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
}
inserted = true
}
if w.IsClosed() && !insertedAsRelation {
// only add polygons that were not inserted as a MultiPolygon relation
if ok, matches := ww.inserter.ProbePolygon(w.OSMElem); ok {
ww.buildAndInsert(geos, w, matches, true)
err := ww.buildAndInsert(geos, w, matches, true)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
log.Warn(err)
}
continue
}
inserted = true
}
}
@ -79,7 +91,7 @@ func (ww *WayWriter) loop() {
ww.wg.Done()
}
func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interface{}, isPolygon bool) {
func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interface{}, isPolygon bool) error {
var err error
var geosgeom *geos.Geom
// make copy to avoid interference with polygon/linestring matches
@ -91,41 +103,42 @@ func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interf
geosgeom, err = geom.LineString(g, way.Nodes)
}
if err != nil {
if err, ok := err.(ErrorLevel); ok {
if err.Level() <= 0 {
return
}
}
log.Warn(err)
return
return err
}
way.Geom, err = geom.AsGeomElement(g, geosgeom)
if err != nil {
log.Warn(err)
return
return err
}
if ww.limiter != nil {
parts, err := ww.limiter.Clip(way.Geom.Geom)
if err != nil {
log.Warn(err)
return
return err
}
for _, p := range parts {
way := element.Way(*w)
way.Geom = &element.Geometry{Geom: p, Wkb: g.AsEwkbHex(p)}
if isPolygon {
ww.inserter.InsertPolygon(way.OSMElem, matches)
if err := ww.inserter.InsertPolygon(way.OSMElem, matches); err != nil {
return err
}
} else {
ww.inserter.InsertLineString(way.OSMElem, matches)
if err := ww.inserter.InsertLineString(way.OSMElem, matches); err != nil {
return err
}
}
}
} else {
if isPolygon {
ww.inserter.InsertPolygon(way.OSMElem, matches)
if err := ww.inserter.InsertPolygon(way.OSMElem, matches); err != nil {
return err
}
} else {
ww.inserter.InsertLineString(way.OSMElem, matches)
if err := ww.inserter.InsertLineString(way.OSMElem, matches); err != nil {
return err
}
}
}
return nil
}