vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add generalized tables

master
Oliver Tonnhofer 2013-05-22 11:48:34 +02:00
parent ab9480e1ee
commit a64ed7306c
7 changed files with 263 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
database/database.go
View File

@ -22,6 +22,10 @@ type Deployer interface {
RemoveBackup() error
}
type Generalizer interface {
Generalize() error
}
type Finisher interface {
Finish() error
}

12
database/postgis/fields.go
View File

@ -8,6 +8,7 @@ type ColumnType interface {
Name() string
PrepareInsertSql(i int,
spec *TableSpec) string
GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string
}
type simpleColumnType struct {
@ -22,6 +23,10 @@ func (t *simpleColumnType) PrepareInsertSql(i int, spec *TableSpec) string {
return fmt.Sprintf("$%d", i)
}
func (t *simpleColumnType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
return colSpec.Name
}
type geometryType struct {
name string
}
@ -36,6 +41,12 @@ func (t *geometryType) PrepareInsertSql(i int, spec *TableSpec) string {
)
}
func (t *geometryType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
return fmt.Sprintf(`ST_SimplifyPreserveTopology("%s", %f) as "%s"`,
colSpec.Name, spec.Tolerance, colSpec.Name,
)
}
var pgTypes map[string]ColumnType
func init() {
@ -43,6 +54,7 @@ func init() {
"id": &simpleColumnType{"BIGINT"},
"geometry": &geometryType{"GEOMETRY"},
"bool": &simpleColumnType{"BOOL"},
"boolint": &simpleColumnType{"SMALLINT"},
"string": &simpleColumnType{"VARCHAR"},
"name": &simpleColumnType{"VARCHAR"},
"direction": &simpleColumnType{"SMALLINT"},

134
database/postgis/postgis.go
View File

@ -23,6 +23,16 @@ type TableSpec struct {
Srid int
}
type GeneralizedTableSpec struct {
Name string
SourceName string
Source *TableSpec
SourceGeneralized *GeneralizedTableSpec
Tolerance float64
Where string
created bool
}
func (col *ColumnSpec) AsSQL() string {
return fmt.Sprintf("\"%s\" %s", col.Name, col.Type.Name())
}
@ -86,6 +96,16 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
return &spec
}
func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
spec := GeneralizedTableSpec{
Name: pg.Prefix + t.Name,
Tolerance: t.Tolerance,
Where: t.SqlFilter,
SourceName: t.SourceTableName,
}
return &spec
}
type SQLError struct {
query string
originalError error
@ -162,12 +182,13 @@ func (pg *PostGIS) createSchema(schema string) error {
}
type PostGIS struct {
Db *sql.DB
Schema string
BackupSchema string
Config database.Config
Tables map[string]*TableSpec
Prefix string
Db *sql.DB
Schema string
BackupSchema string
Config database.Config
Tables map[string]*TableSpec
GeneralizedTables map[string]*GeneralizedTableSpec
Prefix string
}
func schemasFromConnectionParams(params string) (string, string) {
@ -435,9 +456,105 @@ func (pg *PostGIS) Finish() error {
return nil
}
func (pg *PostGIS) checkGeneralizedTableSources() {
for name, table := range pg.GeneralizedTables {
if source, ok := pg.Tables[table.SourceName]; ok {
table.Source = source
} else if source, ok := pg.GeneralizedTables[table.SourceName]; ok {
table.SourceGeneralized = source
} else {
log.Printf("missing source '%s' for generalized table '%s'\n",
table.SourceName, name)
}
}
filled := true
for filled {
filled = false
for _, table := range pg.GeneralizedTables {
if table.Source == nil {
if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil {
table.Source = source.Source
}
filled = true
}
}
}
}
func (pg *PostGIS) Generalize() error {
fmt.Println("generalizing")
// generalized tables can depend on other generalized tables
// create tables with non-generalized sources first
for _, table := range pg.GeneralizedTables {
if table.SourceGeneralized == nil {
if err := pg.generalizeTable(table); err != nil {
return err
}
table.created = true
}
}
// next create tables with created generalized sources until
// no new source is created
created := true
for created {
created = false
for _, table := range pg.GeneralizedTables {
if !table.created && table.SourceGeneralized.created {
if err := pg.generalizeTable(table); err != nil {
return err
}
table.created = true
created = true
}
}
}
return nil
}
func (pg *PostGIS) generalizeTable(table *GeneralizedTableSpec) error {
tx, err := pg.Db.Begin()
if err != nil {
return err
}
defer rollbackIfTx(&tx)
var where string
if table.Where != "" {
where = " WHERE " + table.Where
}
var cols []string
for _, col := range table.Source.Columns {
cols = append(cols, col.Type.GeneralizeSql(&col, table))
}
if err := dropTableIfExists(tx, pg.Schema, table.Name); err != nil {
return err
}
columnSQL := strings.Join(cols, ",\n")
sql := fmt.Sprintf(`CREATE TABLE "%s"."%s" AS (SELECT %s FROM "%s"."%s"%s)`,
pg.Schema, table.Name, columnSQL, pg.Schema,
pg.Prefix+table.SourceName, where)
fmt.Println(sql)
_, err = tx.Exec(sql)
if err != nil {
return err
}
err = tx.Commit()
if err != nil {
return err
}
tx = nil // set nil to prevent rollback
return nil
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
db := &PostGIS{}
db.Tables = make(map[string]*TableSpec)
db.GeneralizedTables = make(map[string]*GeneralizedTableSpec)
db.Config = conf
if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") {
@ -457,6 +574,11 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
for name, table := range m.Tables {
db.Tables[name] = NewTableSpec(db, table)
}
for name, table := range m.GeneralizedTables {
db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table)
}
db.checkGeneralizedTableSources()
err = db.Open()
if err != nil {
return nil, err

86
goposm.go
View File

@ -135,47 +135,57 @@ func main() {
}
if *write {
progress.Reset()
err = db.Init()
if err != nil {
log.Fatal(err)
if true {
progress.Reset()
err = db.Init()
if err != nil {
log.Fatal(err)
}
diffCache := cache.NewDiffCache(*cachedir)
if err = diffCache.Remove(); err != nil {
log.Fatal(err)
}
if err = diffCache.Open(); err != nil {
log.Fatal(err)
}
insertBuffer := writer.NewInsertBuffer()
dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
pointsTagMatcher := tagmapping.PointMatcher()
lineStringsTagMatcher := tagmapping.LineStringMatcher()
polygonsTagMatcher := tagmapping.PolygonMatcher()
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, relations,
insertBuffer, polygonsTagMatcher, progress)
// blocks till the Relations.Iter() finishes
relWriter.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
lineStringsTagMatcher, polygonsTagMatcher, progress)
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
pointsTagMatcher, progress)
diffCache.Coords.Close()
wayWriter.Close()
nodeWriter.Close()
insertBuffer.Close()
dbWriter.Close()
}
diffCache := cache.NewDiffCache(*cachedir)
if err = diffCache.Remove(); err != nil {
log.Fatal(err)
if db, ok := db.(database.Generalizer); ok {
if err := db.Generalize(); err != nil {
log.Fatal(err)
}
} else {
log.Fatal("database not generalizeable")
}
if err = diffCache.Open(); err != nil {
log.Fatal(err)
}
insertBuffer := writer.NewInsertBuffer()
dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
pointsTagMatcher := tagmapping.PointMatcher()
lineStringsTagMatcher := tagmapping.LineStringMatcher()
polygonsTagMatcher := tagmapping.PolygonMatcher()
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, relations,
insertBuffer, polygonsTagMatcher, progress)
// blocks till the Relations.Iter() finishes
relWriter.Close()
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
lineStringsTagMatcher, polygonsTagMatcher, progress)
nodes := osmCache.Nodes.Iter()
nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
pointsTagMatcher, progress)
diffCache.Coords.Close()
wayWriter.Close()
nodeWriter.Close()
insertBuffer.Close()
dbWriter.Close()
if db, ok := db.(database.Finisher); ok {
if err := db.Finish(); err != nil {

51
mapping.json
View File

@ -1,4 +1,46 @@
{
"generalized_tables": {
"waterareas_gen1": {
"source": "waterareas",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"waterareas_gen0": {
"source": "waterareas_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
},
"roads_gen0": {
"source": "roads_gen1",
"sql_filter": null,
"tolerance": 200.0
},
"roads_gen1": {
"source": "roads",
"sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')",
"tolerance": 50.0
},
"waterways_gen0": {
"source": "waterways_gen1",
"sql_filter": null,
"tolerance": 200
},
"waterways_gen1": {
"source": "waterways",
"sql_filter": null,
"tolerance": 50.0
},
"landusages_gen1": {
"source": "landusages",
"sql_filter": "ST_Area(geometry)>50000.000000",
"tolerance": 50.0
},
"landusages_gen0": {
"source": "landusages_gen1",
"sql_filter": "ST_Area(geometry)>500000.000000",
"tolerance": 200.0
}
},
"tables": {
"landusages": {
"fields": [
@ -643,12 +685,12 @@
"key": "name"
},
{
"type": "bool",
"type": "boolint",
"name": "tunnel",
"key": "tunnel"
},
{
"type": "bool",
"type": "boolint",
"name": "bridge",
"key": "bridge"
},
@ -676,6 +718,11 @@
"type": "string",
"name": "service",
"key": "service"
},
{
"type": "mapping_key",
"name": "class",
"key": null
}
],
"type": "linestring",

15
mapping/config.go
View File

@ -21,14 +21,24 @@ type Table struct {
Filters *Filters `json:"filters"`
}
type GeneralizedTable struct {
Name string
SourceTableName string `json:"source"`
Tolerance float64 `json:"tolerance"`
SqlFilter string `json:"sql_filter"`
}
type Filters struct {
ExcludeTags *map[string]string `json:"exclude_tags"`
}
type Tables map[string]*Table
type GeneralizedTables map[string]*GeneralizedTable
type Mapping struct {
Tables Tables `json:"tables"`
Tables Tables `json:"tables"`
GeneralizedTables GeneralizedTables `json:"generalized_tables"`
}
type ElementFilter func(elem *element.OSMElem) bool
@ -66,6 +76,9 @@ func (m *Mapping) prepare() {
for name, t := range m.Tables {
t.Name = name
}
for name, t := range m.GeneralizedTables {
t.Name = name
}
}
func (m *Mapping) mappings(tableType string, mappings TagTables) {

8
mapping/fields.go
View File

@ -12,6 +12,7 @@ var AvailableFieldTypes map[string]FieldType
func init() {
AvailableFieldTypes = map[string]FieldType{
"bool": {"bool", "bool", Bool, nil},
"boolint": {"boolint", "int8", BoolInt, nil},
"id": {"id", "int64", Id, nil},
"string": {"string", "string", String, nil},
"direction": {"direction", "int8", Direction, nil},
@ -92,6 +93,13 @@ func Bool(val string, elem *element.OSMElem, match Match) interface{} {
return true
}
func BoolInt(val string, elem *element.OSMElem, match Match) interface{} {
if val == "" || val == "0" || val == "false" || val == "no" {
return 0
}
return 1
}
func String(val string, elem *element.OSMElem, match Match) interface{} {
return val
}