From a64ed7306c98efef8ec9ef3941c85c90c80ed8d8 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 22 May 2013 11:48:34 +0200 Subject: [PATCH] add generalized tables --- database/database.go | 4 ++ database/postgis/fields.go | 12 ++++ database/postgis/postgis.go | 134 ++++++++++++++++++++++++++++++++++-- goposm.go | 86 +++++++++++++---------- mapping.json | 51 +++++++++++++- mapping/config.go | 15 +++- mapping/fields.go | 8 +++ 7 files changed, 263 insertions(+), 47 deletions(-) diff --git a/database/database.go b/database/database.go index a24bd37..7e03cd9 100644 --- a/database/database.go +++ b/database/database.go @@ -22,6 +22,10 @@ type Deployer interface { RemoveBackup() error } +type Generalizer interface { + Generalize() error +} + type Finisher interface { Finish() error } diff --git a/database/postgis/fields.go b/database/postgis/fields.go index b667ee1..a422b08 100644 --- a/database/postgis/fields.go +++ b/database/postgis/fields.go @@ -8,6 +8,7 @@ type ColumnType interface { Name() string PrepareInsertSql(i int, spec *TableSpec) string + GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string } type simpleColumnType struct { @@ -22,6 +23,10 @@ func (t *simpleColumnType) PrepareInsertSql(i int, spec *TableSpec) string { return fmt.Sprintf("$%d", i) } +func (t *simpleColumnType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string { + return colSpec.Name +} + type geometryType struct { name string } @@ -36,6 +41,12 @@ func (t *geometryType) PrepareInsertSql(i int, spec *TableSpec) string { ) } +func (t *geometryType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string { + return fmt.Sprintf(`ST_SimplifyPreserveTopology("%s", %f) as "%s"`, + colSpec.Name, spec.Tolerance, colSpec.Name, + ) +} + var pgTypes map[string]ColumnType func init() { @@ -43,6 +54,7 @@ func init() { "id": &simpleColumnType{"BIGINT"}, "geometry": &geometryType{"GEOMETRY"}, "bool": &simpleColumnType{"BOOL"}, + "boolint": &simpleColumnType{"SMALLINT"}, "string": &simpleColumnType{"VARCHAR"}, "name": &simpleColumnType{"VARCHAR"}, "direction": &simpleColumnType{"SMALLINT"}, diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 1779c6f..6fe1d9f 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -23,6 +23,16 @@ type TableSpec struct { Srid int } +type GeneralizedTableSpec struct { + Name string + SourceName string + Source *TableSpec + SourceGeneralized *GeneralizedTableSpec + Tolerance float64 + Where string + created bool +} + func (col *ColumnSpec) AsSQL() string { return fmt.Sprintf("\"%s\" %s", col.Name, col.Type.Name()) } @@ -86,6 +96,16 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { return &spec } +func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec { + spec := GeneralizedTableSpec{ + Name: pg.Prefix + t.Name, + Tolerance: t.Tolerance, + Where: t.SqlFilter, + SourceName: t.SourceTableName, + } + return &spec +} + type SQLError struct { query string originalError error @@ -162,12 +182,13 @@ func (pg *PostGIS) createSchema(schema string) error { } type PostGIS struct { - Db *sql.DB - Schema string - BackupSchema string - Config database.Config - Tables map[string]*TableSpec - Prefix string + Db *sql.DB + Schema string + BackupSchema string + Config database.Config + Tables map[string]*TableSpec + GeneralizedTables map[string]*GeneralizedTableSpec + Prefix string } func schemasFromConnectionParams(params string) (string, string) { @@ -435,9 +456,105 @@ func (pg *PostGIS) Finish() error { return nil } +func (pg *PostGIS) checkGeneralizedTableSources() { + for name, table := range pg.GeneralizedTables { + if source, ok := pg.Tables[table.SourceName]; ok { + table.Source = source + } else if source, ok := pg.GeneralizedTables[table.SourceName]; ok { + table.SourceGeneralized = source + } else { + log.Printf("missing source '%s' for generalized table '%s'\n", + table.SourceName, name) + } + } + + filled := true + for filled { + filled = false + for _, table := range pg.GeneralizedTables { + if table.Source == nil { + if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil { + table.Source = source.Source + } + filled = true + } + } + } +} + +func (pg *PostGIS) Generalize() error { + fmt.Println("generalizing") + // generalized tables can depend on other generalized tables + // create tables with non-generalized sources first + for _, table := range pg.GeneralizedTables { + if table.SourceGeneralized == nil { + if err := pg.generalizeTable(table); err != nil { + return err + } + table.created = true + } + } + // next create tables with created generalized sources until + // no new source is created + created := true + for created { + created = false + for _, table := range pg.GeneralizedTables { + if !table.created && table.SourceGeneralized.created { + if err := pg.generalizeTable(table); err != nil { + return err + } + table.created = true + created = true + } + } + } + return nil +} + +func (pg *PostGIS) generalizeTable(table *GeneralizedTableSpec) error { + tx, err := pg.Db.Begin() + if err != nil { + return err + } + defer rollbackIfTx(&tx) + + var where string + if table.Where != "" { + where = " WHERE " + table.Where + } + var cols []string + + for _, col := range table.Source.Columns { + cols = append(cols, col.Type.GeneralizeSql(&col, table)) + } + + if err := dropTableIfExists(tx, pg.Schema, table.Name); err != nil { + return err + } + + columnSQL := strings.Join(cols, ",\n") + sql := fmt.Sprintf(`CREATE TABLE "%s"."%s" AS (SELECT %s FROM "%s"."%s"%s)`, + pg.Schema, table.Name, columnSQL, pg.Schema, + pg.Prefix+table.SourceName, where) + fmt.Println(sql) + _, err = tx.Exec(sql) + if err != nil { + return err + } + err = tx.Commit() + if err != nil { + return err + } + tx = nil // set nil to prevent rollback + return nil +} + func New(conf database.Config, m *mapping.Mapping) (database.DB, error) { db := &PostGIS{} db.Tables = make(map[string]*TableSpec) + db.GeneralizedTables = make(map[string]*GeneralizedTableSpec) + db.Config = conf if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") { @@ -457,6 +574,11 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) { for name, table := range m.Tables { db.Tables[name] = NewTableSpec(db, table) } + for name, table := range m.GeneralizedTables { + db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table) + } + db.checkGeneralizedTableSources() + err = db.Open() if err != nil { return nil, err diff --git a/goposm.go b/goposm.go index 531fe77..0ea8cc2 100644 --- a/goposm.go +++ b/goposm.go @@ -135,47 +135,57 @@ func main() { } if *write { - progress.Reset() - err = db.Init() - if err != nil { - log.Fatal(err) + if true { + progress.Reset() + err = db.Init() + if err != nil { + log.Fatal(err) + } + + diffCache := cache.NewDiffCache(*cachedir) + if err = diffCache.Remove(); err != nil { + log.Fatal(err) + } + if err = diffCache.Open(); err != nil { + log.Fatal(err) + } + + insertBuffer := writer.NewInsertBuffer() + dbWriter := writer.NewDbWriter(db, insertBuffer.Out) + + pointsTagMatcher := tagmapping.PointMatcher() + lineStringsTagMatcher := tagmapping.LineStringMatcher() + polygonsTagMatcher := tagmapping.PolygonMatcher() + + relations := osmCache.Relations.Iter() + relWriter := writer.NewRelationWriter(osmCache, relations, + insertBuffer, polygonsTagMatcher, progress) + // blocks till the Relations.Iter() finishes + relWriter.Close() + + ways := osmCache.Ways.Iter() + wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer, + lineStringsTagMatcher, polygonsTagMatcher, progress) + + nodes := osmCache.Nodes.Iter() + nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer, + pointsTagMatcher, progress) + + diffCache.Coords.Close() + + wayWriter.Close() + nodeWriter.Close() + insertBuffer.Close() + dbWriter.Close() } - diffCache := cache.NewDiffCache(*cachedir) - if err = diffCache.Remove(); err != nil { - log.Fatal(err) + if db, ok := db.(database.Generalizer); ok { + if err := db.Generalize(); err != nil { + log.Fatal(err) + } + } else { + log.Fatal("database not generalizeable") } - if err = diffCache.Open(); err != nil { - log.Fatal(err) - } - - insertBuffer := writer.NewInsertBuffer() - dbWriter := writer.NewDbWriter(db, insertBuffer.Out) - - pointsTagMatcher := tagmapping.PointMatcher() - lineStringsTagMatcher := tagmapping.LineStringMatcher() - polygonsTagMatcher := tagmapping.PolygonMatcher() - - relations := osmCache.Relations.Iter() - relWriter := writer.NewRelationWriter(osmCache, relations, - insertBuffer, polygonsTagMatcher, progress) - // blocks till the Relations.Iter() finishes - relWriter.Close() - - ways := osmCache.Ways.Iter() - wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer, - lineStringsTagMatcher, polygonsTagMatcher, progress) - - nodes := osmCache.Nodes.Iter() - nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer, - pointsTagMatcher, progress) - - diffCache.Coords.Close() - - wayWriter.Close() - nodeWriter.Close() - insertBuffer.Close() - dbWriter.Close() if db, ok := db.(database.Finisher); ok { if err := db.Finish(); err != nil { diff --git a/mapping.json b/mapping.json index 9fa4d8e..9bee30d 100644 --- a/mapping.json +++ b/mapping.json @@ -1,4 +1,46 @@ { + "generalized_tables": { + "waterareas_gen1": { + "source": "waterareas", + "sql_filter": "ST_Area(geometry)>50000.000000", + "tolerance": 50.0 + }, + "waterareas_gen0": { + "source": "waterareas_gen1", + "sql_filter": "ST_Area(geometry)>500000.000000", + "tolerance": 200.0 + }, + "roads_gen0": { + "source": "roads_gen1", + "sql_filter": null, + "tolerance": 200.0 + }, + "roads_gen1": { + "source": "roads", + "sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')", + "tolerance": 50.0 + }, + "waterways_gen0": { + "source": "waterways_gen1", + "sql_filter": null, + "tolerance": 200 + }, + "waterways_gen1": { + "source": "waterways", + "sql_filter": null, + "tolerance": 50.0 + }, + "landusages_gen1": { + "source": "landusages", + "sql_filter": "ST_Area(geometry)>50000.000000", + "tolerance": 50.0 + }, + "landusages_gen0": { + "source": "landusages_gen1", + "sql_filter": "ST_Area(geometry)>500000.000000", + "tolerance": 200.0 + } + }, "tables": { "landusages": { "fields": [ @@ -643,12 +685,12 @@ "key": "name" }, { - "type": "bool", + "type": "boolint", "name": "tunnel", "key": "tunnel" }, { - "type": "bool", + "type": "boolint", "name": "bridge", "key": "bridge" }, @@ -676,6 +718,11 @@ "type": "string", "name": "service", "key": "service" + }, + { + "type": "mapping_key", + "name": "class", + "key": null } ], "type": "linestring", diff --git a/mapping/config.go b/mapping/config.go index 8672204..06cadcc 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -21,14 +21,24 @@ type Table struct { Filters *Filters `json:"filters"` } +type GeneralizedTable struct { + Name string + SourceTableName string `json:"source"` + Tolerance float64 `json:"tolerance"` + SqlFilter string `json:"sql_filter"` +} + type Filters struct { ExcludeTags *map[string]string `json:"exclude_tags"` } type Tables map[string]*Table +type GeneralizedTables map[string]*GeneralizedTable + type Mapping struct { - Tables Tables `json:"tables"` + Tables Tables `json:"tables"` + GeneralizedTables GeneralizedTables `json:"generalized_tables"` } type ElementFilter func(elem *element.OSMElem) bool @@ -66,6 +76,9 @@ func (m *Mapping) prepare() { for name, t := range m.Tables { t.Name = name } + for name, t := range m.GeneralizedTables { + t.Name = name + } } func (m *Mapping) mappings(tableType string, mappings TagTables) { diff --git a/mapping/fields.go b/mapping/fields.go index a80f496..202e45d 100644 --- a/mapping/fields.go +++ b/mapping/fields.go @@ -12,6 +12,7 @@ var AvailableFieldTypes map[string]FieldType func init() { AvailableFieldTypes = map[string]FieldType{ "bool": {"bool", "bool", Bool, nil}, + "boolint": {"boolint", "int8", BoolInt, nil}, "id": {"id", "int64", Id, nil}, "string": {"string", "string", String, nil}, "direction": {"direction", "int8", Direction, nil}, @@ -92,6 +93,13 @@ func Bool(val string, elem *element.OSMElem, match Match) interface{} { return true } +func BoolInt(val string, elem *element.OSMElem, match Match) interface{} { + if val == "" || val == "0" || val == "false" || val == "no" { + return 0 + } + return 1 +} + func String(val string, elem *element.OSMElem, match Match) interface{} { return val }