add generalized tables

master
Oliver Tonnhofer 2013-05-22 11:48:34 +02:00
parent ab9480e1ee
commit a64ed7306c
7 changed files with 263 additions and 47 deletions

View File

@ -22,6 +22,10 @@ type Deployer interface {
RemoveBackup() error RemoveBackup() error
} }
type Generalizer interface {
Generalize() error
}
type Finisher interface { type Finisher interface {
Finish() error Finish() error
} }

View File

@ -8,6 +8,7 @@ type ColumnType interface {
Name() string Name() string
PrepareInsertSql(i int, PrepareInsertSql(i int,
spec *TableSpec) string spec *TableSpec) string
GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string
} }
type simpleColumnType struct { type simpleColumnType struct {
@ -22,6 +23,10 @@ func (t *simpleColumnType) PrepareInsertSql(i int, spec *TableSpec) string {
return fmt.Sprintf("$%d", i) return fmt.Sprintf("$%d", i)
} }
func (t *simpleColumnType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
return colSpec.Name
}
type geometryType struct { type geometryType struct {
name string name string
} }
@ -36,6 +41,12 @@ func (t *geometryType) PrepareInsertSql(i int, spec *TableSpec) string {
) )
} }
func (t *geometryType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
return fmt.Sprintf(`ST_SimplifyPreserveTopology("%s", %f) as "%s"`,
colSpec.Name, spec.Tolerance, colSpec.Name,
)
}
var pgTypes map[string]ColumnType var pgTypes map[string]ColumnType
func init() { func init() {
@ -43,6 +54,7 @@ func init() {
"id": &simpleColumnType{"BIGINT"}, "id": &simpleColumnType{"BIGINT"},
"geometry": &geometryType{"GEOMETRY"}, "geometry": &geometryType{"GEOMETRY"},
"bool": &simpleColumnType{"BOOL"}, "bool": &simpleColumnType{"BOOL"},
"boolint": &simpleColumnType{"SMALLINT"},
"string": &simpleColumnType{"VARCHAR"}, "string": &simpleColumnType{"VARCHAR"},
"name": &simpleColumnType{"VARCHAR"}, "name": &simpleColumnType{"VARCHAR"},
"direction": &simpleColumnType{"SMALLINT"}, "direction": &simpleColumnType{"SMALLINT"},

View File

@ -23,6 +23,16 @@ type TableSpec struct {
Srid int Srid int
} }
type GeneralizedTableSpec struct {
Name string
SourceName string
Source *TableSpec
SourceGeneralized *GeneralizedTableSpec
Tolerance float64
Where string
created bool
}
func (col *ColumnSpec) AsSQL() string { func (col *ColumnSpec) AsSQL() string {
return fmt.Sprintf("\"%s\" %s", col.Name, col.Type.Name()) return fmt.Sprintf("\"%s\" %s", col.Name, col.Type.Name())
} }
@ -86,6 +96,16 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
return &spec return &spec
} }
func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
spec := GeneralizedTableSpec{
Name: pg.Prefix + t.Name,
Tolerance: t.Tolerance,
Where: t.SqlFilter,
SourceName: t.SourceTableName,
}
return &spec
}
type SQLError struct { type SQLError struct {
query string query string
originalError error originalError error
@ -162,12 +182,13 @@ func (pg *PostGIS) createSchema(schema string) error {
} }
type PostGIS struct { type PostGIS struct {
Db *sql.DB Db *sql.DB
Schema string Schema string
BackupSchema string BackupSchema string
Config database.Config Config database.Config
Tables map[string]*TableSpec Tables map[string]*TableSpec
Prefix string GeneralizedTables map[string]*GeneralizedTableSpec
Prefix string
} }
func schemasFromConnectionParams(params string) (string, string) { func schemasFromConnectionParams(params string) (string, string) {
@ -435,9 +456,105 @@ func (pg *PostGIS) Finish() error {
return nil return nil
} }
func (pg *PostGIS) checkGeneralizedTableSources() {
for name, table := range pg.GeneralizedTables {
if source, ok := pg.Tables[table.SourceName]; ok {
table.Source = source
} else if source, ok := pg.GeneralizedTables[table.SourceName]; ok {
table.SourceGeneralized = source
} else {
log.Printf("missing source '%s' for generalized table '%s'\n",
table.SourceName, name)
}
}
filled := true
for filled {
filled = false
for _, table := range pg.GeneralizedTables {
if table.Source == nil {
if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil {
table.Source = source.Source
}
filled = true
}
}
}
}
func (pg *PostGIS) Generalize() error {
fmt.Println("generalizing")
// generalized tables can depend on other generalized tables
// create tables with non-generalized sources first
for _, table := range pg.GeneralizedTables {
if table.SourceGeneralized == nil {
if err := pg.generalizeTable(table); err != nil {
return err
}
table.created = true
}
}
// next create tables with created generalized sources until
// no new source is created
created := true
for created {
created = false
for _, table := range pg.GeneralizedTables {
if !table.created && table.SourceGeneralized.created {
if err := pg.generalizeTable(table); err != nil {
return err
}
table.created = true
created = true
}
}
}
return nil
}
func (pg *PostGIS) generalizeTable(table *GeneralizedTableSpec) error {
tx, err := pg.Db.Begin()
if err != nil {
return err
}
defer rollbackIfTx(&tx)
var where string
if table.Where != "" {
where = " WHERE " + table.Where
}
var cols []string
for _, col := range table.Source.Columns {
cols = append(cols, col.Type.GeneralizeSql(&col, table))
}
if err := dropTableIfExists(tx, pg.Schema, table.Name); err != nil {
return err
}
columnSQL := strings.Join(cols, ",\n")
sql := fmt.Sprintf(`CREATE TABLE "%s"."%s" AS (SELECT %s FROM "%s"."%s"%s)`,
pg.Schema, table.Name, columnSQL, pg.Schema,
pg.Prefix+table.SourceName, where)
fmt.Println(sql)
_, err = tx.Exec(sql)
if err != nil {
return err
}
err = tx.Commit()
if err != nil {
return err
}
tx = nil // set nil to prevent rollback
return nil
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) { func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
db := &PostGIS{} db := &PostGIS{}
db.Tables = make(map[string]*TableSpec) db.Tables = make(map[string]*TableSpec)
db.GeneralizedTables = make(map[string]*GeneralizedTableSpec)
db.Config = conf db.Config = conf
if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") { if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") {
@ -457,6 +574,11 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
for name, table := range m.Tables { for name, table := range m.Tables {
db.Tables[name] = NewTableSpec(db, table) db.Tables[name] = NewTableSpec(db, table)
} }
for name, table := range m.GeneralizedTables {
db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table)
}
db.checkGeneralizedTableSources()
err = db.Open() err = db.Open()
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -135,47 +135,57 @@ func main() {
} }
if *write { if *write {
progress.Reset() if true {
err = db.Init() progress.Reset()
if err != nil { err = db.Init()
log.Fatal(err) if err != nil {
log.Fatal(err)
}
diffCache := cache.NewDiffCache(*cachedir)
if err = diffCache.Remove(); err != nil {
log.Fatal(err)
}
if err = diffCache.Open(); err != nil {
log.Fatal(err)
}
insertBuffer := writer.NewInsertBuffer()
dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
pointsTagMatcher := tagmapping.PointMatcher()
lineStringsTagMatcher := tagmapping.LineStringMatcher()
polygonsTagMatcher := tagmapping.PolygonMatcher()
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, relations,
insertBuffer, polygonsTagMatcher, progress)
// blocks till the Relations.Iter() finishes
relWriter.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
lineStringsTagMatcher, polygonsTagMatcher, progress)
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
pointsTagMatcher, progress)
diffCache.Coords.Close()
wayWriter.Close()
nodeWriter.Close()
insertBuffer.Close()
dbWriter.Close()
} }
diffCache := cache.NewDiffCache(*cachedir) if db, ok := db.(database.Generalizer); ok {
if err = diffCache.Remove(); err != nil { if err := db.Generalize(); err != nil {
log.Fatal(err) log.Fatal(err)
}
} else {
log.Fatal("database not generalizeable")
} }
if err = diffCache.Open(); err != nil {
log.Fatal(err)
}
insertBuffer := writer.NewInsertBuffer()
dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
pointsTagMatcher := tagmapping.PointMatcher()
lineStringsTagMatcher := tagmapping.LineStringMatcher()
polygonsTagMatcher := tagmapping.PolygonMatcher()
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, relations,
insertBuffer, polygonsTagMatcher, progress)
// blocks till the Relations.Iter() finishes
relWriter.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
lineStringsTagMatcher, polygonsTagMatcher, progress)
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
pointsTagMatcher, progress)
diffCache.Coords.Close()
wayWriter.Close()
nodeWriter.Close()
insertBuffer.Close()
dbWriter.Close()
if db, ok := db.(database.Finisher); ok { if db, ok := db.(database.Finisher); ok {
if err := db.Finish(); err != nil { if err := db.Finish(); err != nil {

View File

@ -1,4 +1,46 @@
{ {
"generalized_tables": {
"waterareas_gen1": {
"source": "waterareas",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"waterareas_gen0": {
"source": "waterareas_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
},
"roads_gen0": {
"source": "roads_gen1",
"sql_filter": null,
"tolerance": 200.0
},
"roads_gen1": {
"source": "roads",
"sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')",
"tolerance": 50.0
},
"waterways_gen0": {
"source": "waterways_gen1",
"sql_filter": null,
"tolerance": 200
},
"waterways_gen1": {
"source": "waterways",
"sql_filter": null,
"tolerance": 50.0
},
"landusages_gen1": {
"source": "landusages",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"landusages_gen0": {
"source": "landusages_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
}
},
"tables": { "tables": {
"landusages": { "landusages": {
"fields": [ "fields": [
@ -643,12 +685,12 @@
"key": "name" "key": "name"
}, },
{ {
"type": "bool", "type": "boolint",
"name": "tunnel", "name": "tunnel",
"key": "tunnel" "key": "tunnel"
}, },
{ {
"type": "bool", "type": "boolint",
"name": "bridge", "name": "bridge",
"key": "bridge" "key": "bridge"
}, },
@ -676,6 +718,11 @@
"type": "string", "type": "string",
"name": "service", "name": "service",
"key": "service" "key": "service"
},
{
"type": "mapping_key",
"name": "class",
"key": null
} }
], ],
"type": "linestring", "type": "linestring",

View File

@ -21,14 +21,24 @@ type Table struct {
Filters *Filters `json:"filters"` Filters *Filters `json:"filters"`
} }
type GeneralizedTable struct {
Name string
SourceTableName string `json:"source"`
Tolerance float64 `json:"tolerance"`
SqlFilter string `json:"sql_filter"`
}
type Filters struct { type Filters struct {
ExcludeTags *map[string]string `json:"exclude_tags"` ExcludeTags *map[string]string `json:"exclude_tags"`
} }
type Tables map[string]*Table type Tables map[string]*Table
type GeneralizedTables map[string]*GeneralizedTable
type Mapping struct { type Mapping struct {
Tables Tables `json:"tables"` Tables Tables `json:"tables"`
GeneralizedTables GeneralizedTables `json:"generalized_tables"`
} }
type ElementFilter func(elem *element.OSMElem) bool type ElementFilter func(elem *element.OSMElem) bool
@ -66,6 +76,9 @@ func (m *Mapping) prepare() {
for name, t := range m.Tables { for name, t := range m.Tables {
t.Name = name t.Name = name
} }
for name, t := range m.GeneralizedTables {
t.Name = name
}
} }
func (m *Mapping) mappings(tableType string, mappings TagTables) { func (m *Mapping) mappings(tableType string, mappings TagTables) {

View File

@ -12,6 +12,7 @@ var AvailableFieldTypes map[string]FieldType
func init() { func init() {
AvailableFieldTypes = map[string]FieldType{ AvailableFieldTypes = map[string]FieldType{
"bool": {"bool", "bool", Bool, nil}, "bool": {"bool", "bool", Bool, nil},
"boolint": {"boolint", "int8", BoolInt, nil},
"id": {"id", "int64", Id, nil}, "id": {"id", "int64", Id, nil},
"string": {"string", "string", String, nil}, "string": {"string", "string", String, nil},
"direction": {"direction", "int8", Direction, nil}, "direction": {"direction", "int8", Direction, nil},
@ -92,6 +93,13 @@ func Bool(val string, elem *element.OSMElem, match Match) interface{} {
return true return true
} }
func BoolInt(val string, elem *element.OSMElem, match Match) interface{} {
if val == "" || val == "0" || val == "false" || val == "no" {
return 0
}
return 1
}
func String(val string, elem *element.OSMElem, match Match) interface{} { func String(val string, elem *element.OSMElem, match Match) interface{} {
return val return val
} }