vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactored tag matching from database to writer 

tag matching is independent of the database and can belong to the writer
package. also cleans up the database.Inserter interface.
master
Oliver Tonnhofer 2014-05-05 10:21:46 +02:00
parent 21a389b2cd
commit 088a4d89f3
10 changed files with 181 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
database/database.go
View File

@ -29,20 +29,11 @@ type BulkBeginner interface {
}
type Inserter interface {
// ProbeXxx returns true if the element should be inserted.
// The interface{} value is passed to InsertXxx when that element
// gets inserted (can be used to pass a match object to the insert call).
ProbePoint(element.OSMElem) (bool, interface{})
ProbeLineString(element.OSMElem) (bool, interface{})
ProbePolygon(element.OSMElem) (bool, interface{})
// InsertXxx inserts element of that type into the database.
// element.Geom is set to that type.
InsertPoint(element.OSMElem, interface{}) error
InsertLineString(element.OSMElem, interface{}) error
InsertPolygon(element.OSMElem, interface{}) error
// SelectRelationPolygons returns a slice of all members that are already
// imported with a relation with tags.
SelectRelationPolygons(element.Tags, []element.Member) []element.Member
InsertPoint(element.OSMElem, []mapping.Match) error
InsertLineString(element.OSMElem, []mapping.Match) error
InsertPolygon(element.OSMElem, []mapping.Match) error
}
type Deployer interface {
@ -102,18 +93,14 @@ func Open(conf Config, m *mapping.Mapping) (DB, error) {
// nullDb is a dummy database that imports into /dev/null
type nullDb struct{}
func (n *nullDb) Init() error { return nil }
func (n *nullDb) Begin() error { return nil }
func (n *nullDb) End() error { return nil }
func (n *nullDb) Close() error { return nil }
func (n *nullDb) Abort() error { return nil }
func (n *nullDb) InsertPoint(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) InsertLineString(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) InsertPolygon(element.OSMElem, interface{}) error { return nil }
func (n *nullDb) ProbePoint(element.OSMElem) (bool, interface{}) { return true, nil }
func (n *nullDb) ProbeLineString(element.OSMElem) (bool, interface{}) { return true, nil }
func (n *nullDb) ProbePolygon(element.OSMElem) (bool, interface{}) { return true, nil }
func (n *nullDb) SelectRelationPolygons(element.Tags, []element.Member) []element.Member { return nil }
func (n *nullDb) Init() error { return nil }
func (n *nullDb) Begin() error { return nil }
func (n *nullDb) End() error { return nil }
func (n *nullDb) Close() error { return nil }
func (n *nullDb) Abort() error { return nil }
func (n *nullDb) InsertPoint(element.OSMElem, []mapping.Match) error { return nil }
func (n *nullDb) InsertLineString(element.OSMElem, []mapping.Match) error { return nil }
func (n *nullDb) InsertPolygon(element.OSMElem, []mapping.Match) error { return nil }
func newNullDb(conf Config, m *mapping.Mapping) (DB, error) {
return &nullDb{}, nil

110
database/postgis/postgis.go
View File

@ -402,9 +402,6 @@ type PostGIS struct {
GeneralizedTables map[string]*GeneralizedTableSpec
Prefix string
txRouter *TxRouter
pointTagMatcher *mapping.TagMatcher
lineStringTagMatcher *mapping.TagMatcher
polygonTagMatcher *mapping.TagMatcher
updateGeneralizedTables bool
updatedIds map[string][]int64
}
@ -424,103 +421,46 @@ func (pg *PostGIS) Open() error {
return nil
}
func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
func (pg *PostGIS) InsertPoint(elem element.OSMElem, matches []mapping.Match) error {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
}
return nil
}
func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
func (pg *PostGIS) InsertLineString(elem element.OSMElem, matches []mapping.Match) error {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
pg.updatedIds[generalizedTable.Name] = append(pg.updatedIds[generalizedTable.Name], elem.Id)
}
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
pg.updatedIds[generalizedTable.Name] = append(pg.updatedIds[generalizedTable.Name], elem.Id)
}
}
return nil
}
func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
func (pg *PostGIS) InsertPolygon(elem element.OSMElem, matches []mapping.Match) error {
for _, match := range matches {
row := match.Row(&elem)
if err := pg.txRouter.Insert(match.Table.Name, row); err != nil {
return err
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
pg.updatedIds[generalizedTable.Name] = append(pg.updatedIds[generalizedTable.Name], elem.Id)
}
}
if pg.updateGeneralizedTables {
for _, generalizedTable := range pg.generalizedFromMatches(matches) {
pg.updatedIds[generalizedTable.Name] = append(pg.updatedIds[generalizedTable.Name], elem.Id)
}
}
return nil
}
func (pg *PostGIS) ProbePoint(elem element.OSMElem) (bool, interface{}) {
if matches := pg.pointTagMatcher.Match(&elem.Tags); len(matches) > 0 {
return true, matches
}
return false, nil
}
func (pg *PostGIS) ProbeLineString(elem element.OSMElem) (bool, interface{}) {
if matches := pg.lineStringTagMatcher.Match(&elem.Tags); len(matches) > 0 {
return true, matches
}
return false, nil
}
func (pg *PostGIS) ProbePolygon(elem element.OSMElem) (bool, interface{}) {
if matches := pg.polygonTagMatcher.Match(&elem.Tags); len(matches) > 0 {
return true, matches
}
return false, nil
}
func (pg *PostGIS) SelectRelationPolygons(tags element.Tags, members []element.Member) []element.Member {
relMatches := pg.polygonTagMatcher.Match(&tags)
result := []element.Member{}
for _, m := range members {
if m.Type != element.WAY {
continue
}
memberMatches := pg.polygonTagMatcher.Match(&m.Way.Tags)
if matchEquals(relMatches, memberMatches) {
result = append(result, m)
}
}
return result
}
func matchEquals(matchesA, matchesB []mapping.Match) bool {
for _, matchA := range matchesA {
for _, matchB := range matchesB {
if matchA.Key == matchB.Key &&
matchA.Value == matchB.Value &&
matchA.Table == matchB.Table {
return true
}
}
}
return false
}
func (pg *PostGIS) Delete(id int64, matches interface{}) error {
if matches, ok := matches.([]mapping.Match); ok {
for _, match := range matches {
@ -642,10 +582,6 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
db.prepareGeneralizedTableSources()
db.prepareGeneralizations()
db.pointTagMatcher = m.PointMatcher()
db.lineStringTagMatcher = m.LineStringMatcher()
db.polygonTagMatcher = m.PolygonMatcher()
db.Params = params
err = db.Open()
if err != nil {

8
diff/deleter.go
View File

@ -63,7 +63,7 @@ func (d *Deleter) deleteRelation(id int64, deleteRefs bool, deleteMembers bool)
if elem.Tags == nil {
return nil
}
if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok {
if matches := d.tmPolygons.Match(&elem.Tags); len(matches) > 0 {
if err := d.delDb.Delete(-elem.Id, matches); err != nil {
return err
}
@ -136,13 +136,13 @@ func (d *Deleter) deleteWay(id int64, deleteRefs bool) error {
return nil
}
deleted := false
if ok, matches := d.delDb.ProbePolygon(elem.OSMElem); ok {
if matches := d.tmPolygons.Match(&elem.Tags); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}
deleted = true
}
if ok, matches := d.delDb.ProbeLineString(elem.OSMElem); ok {
if matches := d.tmLineStrings.Match(&elem.Tags); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}
@ -178,7 +178,7 @@ func (d *Deleter) deleteNode(id int64) error {
}
deleted := false
if ok, matches := d.delDb.ProbePoint(elem.OSMElem); ok {
if matches := d.tmPoints.Match(&elem.Tags); len(matches) > 0 {
if err := d.delDb.Delete(elem.Id, matches); err != nil {
return err
}

13
diff/process.go
View File

@ -99,19 +99,26 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
db, progress, config.BaseOptions.Srid)
db, progress,
tagmapping.PolygonMatcher(),
config.BaseOptions.Srid)
relWriter.SetLimiter(geometryLimiter)
relWriter.SetExpireor(expireor)
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
progress, config.BaseOptions.Srid)
progress,
tagmapping.PolygonMatcher(),
tagmapping.LineStringMatcher(),
config.BaseOptions.Srid)
wayWriter.SetLimiter(geometryLimiter)
wayWriter.SetExpireor(expireor)
wayWriter.Start()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
progress, config.BaseOptions.Srid)
progress,
tagmapping.PointMatcher(),
config.BaseOptions.Srid)
nodeWriter.SetLimiter(geometryLimiter)
nodeWriter.SetExpireor(expireor)
nodeWriter.Start()

12
import_/import.go
View File

@ -171,7 +171,9 @@ func Import() {
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, diffCache, relations,
db, progress, config.BaseOptions.Srid)
db, progress,
tagmapping.PolygonMatcher(),
config.BaseOptions.Srid)
relWriter.SetLimiter(geometryLimiter)
relWriter.EnableConcurrent()
relWriter.Start()
@ -180,7 +182,9 @@ func Import() {
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
progress, config.BaseOptions.Srid)
progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),
config.BaseOptions.Srid)
wayWriter.SetLimiter(geometryLimiter)
wayWriter.EnableConcurrent()
wayWriter.Start()
@ -189,7 +193,9 @@ func Import() {
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
progress, config.BaseOptions.Srid)
progress,
tagmapping.PointMatcher(),
config.BaseOptions.Srid)
nodeWriter.SetLimiter(geometryLimiter)
nodeWriter.EnableConcurrent()
nodeWriter.Start()

30
mapping/matcher.go
View File

@ -78,3 +78,33 @@ func (tagMatcher *TagMatcher) Match(tags *element.Tags) []Match {
}
return matches
}
// SelectRelationPolygons returns a slice of all members that are already
// imported with a relation with tags.
func SelectRelationPolygons(polygonTagMatcher *TagMatcher, tags element.Tags, members []element.Member) []element.Member {
relMatches := polygonTagMatcher.Match(&tags)
result := []element.Member{}
for _, m := range members {
if m.Type != element.WAY {
continue
}
memberMatches := polygonTagMatcher.Match(&m.Way.Tags)
if matchEquals(relMatches, memberMatches) {
result = append(result, m)
}
}
return result
}
func matchEquals(matchesA, matchesB []Match) bool {
for _, matchA := range matchesA {
for _, matchB := range matchesB {
if matchA.Key == matchB.Key &&
matchA.Value == matchB.Value &&
matchA.Table == matchB.Table {
return true
}
}
}
return false
}

77
database/postgis/postgis_test.go → mapping/matcher_test.go
View File

@ -1,42 +1,38 @@
package postgis
package mapping
import (
"testing"
"imposm3/database"
"imposm3/element"
"imposm3/mapping"
"testing"
)
func BenchmarkTagMatch(b *testing.B) {
m, err := NewMapping("matcher_test_mapping.json")
if err != nil {
b.Fatal(err)
}
matcher := m.PolygonMatcher()
for i := 0; i < b.N; i++ {
t := element.Tags{"landuse": "forest", "name": "Forest", "source": "bling", "tourism": "zoo"}
if m := matcher.Match(&t); len(m) != 1 {
b.Fatal(m)
}
}
}
func makeMember(id int64, tags element.Tags) element.Member {
way := &element.Way{element.OSMElem{id, tags, nil}, nil, nil}
return element.Member{Id: id, Type: element.WAY, Role: "outer", Way: way}
}
func testDb(t *testing.T) *PostGIS {
mapping, err := mapping.NewMapping("test_mapping.json")
if err != nil {
t.Fatal(err)
}
conf := database.Config{
ConnectionParams: "postgis://localhost",
Srid: 3857,
ImportSchema: "",
ProductionSchema: "",
BackupSchema: "",
}
db, err := New(conf, mapping)
if err != nil {
t.Fatal(err)
}
return db.(*PostGIS)
}
func TestSelectRelationPolygonsSimple(t *testing.T) {
db := testDb(t)
filtered := db.SelectRelationPolygons(element.Tags{"landuse": "park"},
mapping, err := NewMapping("test_mapping.json")
if err != nil {
t.Fatal(err)
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher(),
element.Tags{"landuse": "park"},
[]element.Member{
makeMember(0, element.Tags{"landuse": "forest"}),
makeMember(1, element.Tags{"landuse": "park"}),
@ -52,8 +48,13 @@ func TestSelectRelationPolygonsSimple(t *testing.T) {
}
func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
db := testDb(t)
filtered := db.SelectRelationPolygons(element.Tags{"landuse": "park"},
mapping, err := NewMapping("test_mapping.json")
if err != nil {
t.Fatal(err)
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher(),
element.Tags{"landuse": "park"},
[]element.Member{
makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}),
makeMember(1, element.Tags{"landuse": "forest"}),
@ -67,8 +68,13 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
}
func TestSelectRelationPolygonsMultiple(t *testing.T) {
db := testDb(t)
filtered := db.SelectRelationPolygons(element.Tags{"landuse": "park"},
mapping, err := NewMapping("test_mapping.json")
if err != nil {
t.Fatal(err)
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher(),
element.Tags{"landuse": "park"},
[]element.Member{
makeMember(0, element.Tags{"landuse": "park"}),
makeMember(1, element.Tags{"natural": "forest"}),
@ -85,8 +91,13 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) {
}
func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
db := testDb(t)
filtered := db.SelectRelationPolygons(element.Tags{"landuse": "forest", "natural": "scrub"},
mapping, err := NewMapping("test_mapping.json")
if err != nil {
t.Fatal(err)
}
filtered := SelectRelationPolygons(
mapping.PolygonMatcher(),
element.Tags{"landuse": "forest", "natural": "scrub"},
[]element.Member{
makeMember(0, element.Tags{"natural": "scrub"}),
makeMember(1, element.Tags{"landuse": "forest"}),

20
writer/nodes.go
View File

@ -6,6 +6,7 @@ import (
"imposm3/element"
"imposm3/geom"
"imposm3/geom/geos"
"imposm3/mapping"
"imposm3/proj"
"imposm3/stats"
"sync"
@ -13,12 +14,18 @@ import (
type NodeWriter struct {
OsmElemWriter
nodes chan *element.Node
nodes chan *element.Node
pointMatcher *mapping.TagMatcher
}
func NewNodeWriter(osmCache *cache.OSMCache, nodes chan *element.Node,
inserter database.Inserter, progress *stats.Statistics,
srid int) *OsmElemWriter {
func NewNodeWriter(
osmCache *cache.OSMCache,
nodes chan *element.Node,
inserter database.Inserter,
progress *stats.Statistics,
matcher *mapping.TagMatcher,
srid int,
) *OsmElemWriter {
nw := NodeWriter{
OsmElemWriter: OsmElemWriter{
osmCache: osmCache,
@ -27,7 +34,8 @@ func NewNodeWriter(osmCache *cache.OSMCache, nodes chan *element.Node,
inserter: inserter,
srid: srid,
},
nodes: nodes,
pointMatcher: matcher,
nodes: nodes,
}
nw.OsmElemWriter.writer = &nw
return &nw.OsmElemWriter
@ -40,7 +48,7 @@ func (nw *NodeWriter) loop() {
for n := range nw.nodes {
nw.progress.AddNodes(1)
if ok, matches := nw.inserter.ProbePoint(n.OSMElem); ok {
if matches := nw.pointMatcher.Match(&n.Tags); len(matches) > 0 {
proj.NodeToMerc(n)
if nw.expireor != nil {
nw.expireor.Expire(n.Long, n.Lat)

25
writer/relations.go
View File

@ -7,6 +7,7 @@ import (
"imposm3/expire"
"imposm3/geom"
"imposm3/geom/geos"
"imposm3/mapping"
"imposm3/proj"
"imposm3/stats"
"sync"
@ -15,12 +16,19 @@ import (
type RelationWriter struct {
OsmElemWriter
rel chan *element.Relation
rel chan *element.Relation
polygonMatcher *mapping.TagMatcher
}
func NewRelationWriter(osmCache *cache.OSMCache, diffCache *cache.DiffCache, rel chan *element.Relation,
inserter database.Inserter, progress *stats.Statistics,
srid int) *OsmElemWriter {
func NewRelationWriter(
osmCache *cache.OSMCache,
diffCache *cache.DiffCache,
rel chan *element.Relation,
inserter database.Inserter,
progress *stats.Statistics,
matcher *mapping.TagMatcher,
srid int,
) *OsmElemWriter {
rw := RelationWriter{
OsmElemWriter: OsmElemWriter{
osmCache: osmCache,
@ -30,7 +38,8 @@ func NewRelationWriter(osmCache *cache.OSMCache, diffCache *cache.DiffCache, rel
inserter: inserter,
srid: srid,
},
rel: rel,
polygonMatcher: matcher,
rel: rel,
}
rw.OsmElemWriter.writer = &rw
return &rw.OsmElemWriter
@ -80,8 +89,8 @@ NextRel:
}
// check for matches befor building the geometry
ok, matches := rw.inserter.ProbePolygon(r.OSMElem)
if !ok {
matches := rw.polygonMatcher.Match(&r.Tags)
if len(matches) == 0 {
continue NextRel
}
@ -131,7 +140,7 @@ NextRel:
}
}
for _, m := range rw.inserter.SelectRelationPolygons(r.Tags, r.Members) {
for _, m := range mapping.SelectRelationPolygons(rw.polygonMatcher, r.Tags, r.Members) {
err = rw.osmCache.InsertedWays.PutWay(m.Way)
if err != nil {
log.Warn(err)

26
writer/ways.go
View File

@ -7,6 +7,7 @@ import (
"imposm3/expire"
"imposm3/geom"
"imposm3/geom/geos"
"imposm3/mapping"
"imposm3/proj"
"imposm3/stats"
"sync"
@ -14,12 +15,21 @@ import (
type WayWriter struct {
OsmElemWriter
ways chan *element.Way
ways chan *element.Way
lineMatcher *mapping.TagMatcher
polygonMatcher *mapping.TagMatcher
}
func NewWayWriter(osmCache *cache.OSMCache, diffCache *cache.DiffCache, ways chan *element.Way,
func NewWayWriter(
osmCache *cache.OSMCache,
diffCache *cache.DiffCache,
ways chan *element.Way,
inserter database.Inserter,
progress *stats.Statistics, srid int) *OsmElemWriter {
progress *stats.Statistics,
polygonMatcher *mapping.TagMatcher,
lineMatcher *mapping.TagMatcher,
srid int,
) *OsmElemWriter {
ww := WayWriter{
OsmElemWriter: OsmElemWriter{
osmCache: osmCache,
@ -29,7 +39,9 @@ func NewWayWriter(osmCache *cache.OSMCache, diffCache *cache.DiffCache, ways cha
inserter: inserter,
srid: srid,
},
ways: ways,
lineMatcher: lineMatcher,
polygonMatcher: polygonMatcher,
ways: ways,
}
ww.OsmElemWriter.writer = &ww
return &ww.OsmElemWriter
@ -57,7 +69,7 @@ func (ww *WayWriter) loop() {
proj.NodesToMerc(w.Nodes)
inserted := false
if ok, matches := ww.inserter.ProbeLineString(w.OSMElem); ok {
if matches := ww.lineMatcher.Match(&w.Tags); len(matches) > 0 {
err := ww.buildAndInsert(geos, w, matches, false)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
@ -69,7 +81,7 @@ func (ww *WayWriter) loop() {
}
if w.IsClosed() && !insertedAsRelation {
// only add polygons that were not inserted as a MultiPolygon relation
if ok, matches := ww.inserter.ProbePolygon(w.OSMElem); ok {
if matches := ww.polygonMatcher.Match(&w.Tags); len(matches) > 0 {
err := ww.buildAndInsert(geos, w, matches, true)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
@ -91,7 +103,7 @@ func (ww *WayWriter) loop() {
ww.wg.Done()
}
func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches interface{}, isPolygon bool) error {
func (ww *WayWriter) buildAndInsert(g *geos.Geos, w *element.Way, matches []mapping.Match, isPolygon bool) error {
var err error
var geosgeom *geos.Geom
// make copy to avoid interference with polygon/linestring matches