add generalized tables

database/database.go

@@ -22,6 +22,10 @@ type Deployer interface {
 	RemoveBackup() error
 }
 
+type Generalizer interface {
+	Generalize() error
+}
+
 type Finisher interface {
 	Finish() error
 }

database/postgis/fields.go

@@ -8,6 +8,7 @@ type ColumnType interface {
 	Name() string
 	PrepareInsertSql(i int,
 		spec *TableSpec) string
+	GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string
 }
 
 type simpleColumnType struct {
@@ -22,6 +23,10 @@ func (t *simpleColumnType) PrepareInsertSql(i int, spec *TableSpec) string {
 	return fmt.Sprintf("$%d", i)
 }
 
+func (t *simpleColumnType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
+	return colSpec.Name
+}
+
 type geometryType struct {
 	name string
 }
@@ -36,6 +41,12 @@ func (t *geometryType) PrepareInsertSql(i int, spec *TableSpec) string {
 	)
 }
 
+func (t *geometryType) GeneralizeSql(colSpec *ColumnSpec, spec *GeneralizedTableSpec) string {
+	return fmt.Sprintf(`ST_SimplifyPreserveTopology("%s", %f) as "%s"`,
+		colSpec.Name, spec.Tolerance, colSpec.Name,
+	)
+}
+
 var pgTypes map[string]ColumnType
 
 func init() {
@@ -43,6 +54,7 @@ func init() {
 		"id":            &simpleColumnType{"BIGINT"},
 		"geometry":      &geometryType{"GEOMETRY"},
 		"bool":          &simpleColumnType{"BOOL"},
+		"boolint":       &simpleColumnType{"SMALLINT"},
 		"string":        &simpleColumnType{"VARCHAR"},
 		"name":          &simpleColumnType{"VARCHAR"},
 		"direction":     &simpleColumnType{"SMALLINT"},

database/postgis/postgis.go

@@ -23,6 +23,16 @@ type TableSpec struct {
 	Srid         int
 }
 
+type GeneralizedTableSpec struct {
+	Name              string
+	SourceName        string
+	Source            *TableSpec
+	SourceGeneralized *GeneralizedTableSpec
+	Tolerance         float64
+	Where             string
+	created           bool
+}
+
 func (col *ColumnSpec) AsSQL() string {
 	return fmt.Sprintf("\"%s\" %s", col.Name, col.Type.Name())
 }
@@ -86,6 +96,16 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
 	return &spec
 }
 
+func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
+	spec := GeneralizedTableSpec{
+		Name:       pg.Prefix + t.Name,
+		Tolerance:  t.Tolerance,
+		Where:      t.SqlFilter,
+		SourceName: t.SourceTableName,
+	}
+	return &spec
+}
+
 type SQLError struct {
 	query         string
 	originalError error
@@ -162,12 +182,13 @@ func (pg *PostGIS) createSchema(schema string) error {
 }
 
 type PostGIS struct {
-	Db           *sql.DB
+	Db                *sql.DB
-	Schema       string
+	Schema            string
-	BackupSchema string
+	BackupSchema      string
-	Config       database.Config
+	Config            database.Config
-	Tables       map[string]*TableSpec
+	Tables            map[string]*TableSpec
-	Prefix       string
+	GeneralizedTables map[string]*GeneralizedTableSpec
+	Prefix            string
 }
 
 func schemasFromConnectionParams(params string) (string, string) {
@@ -435,9 +456,105 @@ func (pg *PostGIS) Finish() error {
 	return nil
 }
 
+func (pg *PostGIS) checkGeneralizedTableSources() {
+	for name, table := range pg.GeneralizedTables {
+		if source, ok := pg.Tables[table.SourceName]; ok {
+			table.Source = source
+		} else if source, ok := pg.GeneralizedTables[table.SourceName]; ok {
+			table.SourceGeneralized = source
+		} else {
+			log.Printf("missing source '%s' for generalized table '%s'\n",
+				table.SourceName, name)
+		}
+	}
+
+	filled := true
+	for filled {
+		filled = false
+		for _, table := range pg.GeneralizedTables {
+			if table.Source == nil {
+				if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil {
+					table.Source = source.Source
+				}
+				filled = true
+			}
+		}
+	}
+}
+
+func (pg *PostGIS) Generalize() error {
+	fmt.Println("generalizing")
+	// generalized tables can depend on other generalized tables
+	// create tables with non-generalized sources first
+	for _, table := range pg.GeneralizedTables {
+		if table.SourceGeneralized == nil {
+			if err := pg.generalizeTable(table); err != nil {
+				return err
+			}
+			table.created = true
+		}
+	}
+	// next create tables with created generalized sources until
+	// no new source is created
+	created := true
+	for created {
+		created = false
+		for _, table := range pg.GeneralizedTables {
+			if !table.created && table.SourceGeneralized.created {
+				if err := pg.generalizeTable(table); err != nil {
+					return err
+				}
+				table.created = true
+				created = true
+			}
+		}
+	}
+	return nil
+}
+
+func (pg *PostGIS) generalizeTable(table *GeneralizedTableSpec) error {
+	tx, err := pg.Db.Begin()
+	if err != nil {
+		return err
+	}
+	defer rollbackIfTx(&tx)
+
+	var where string
+	if table.Where != "" {
+		where = " WHERE " + table.Where
+	}
+	var cols []string
+
+	for _, col := range table.Source.Columns {
+		cols = append(cols, col.Type.GeneralizeSql(&col, table))
+	}
+
+	if err := dropTableIfExists(tx, pg.Schema, table.Name); err != nil {
+		return err
+	}
+
+	columnSQL := strings.Join(cols, ",\n")
+	sql := fmt.Sprintf(`CREATE TABLE "%s"."%s" AS (SELECT %s FROM "%s"."%s"%s)`,
+		pg.Schema, table.Name, columnSQL, pg.Schema,
+		pg.Prefix+table.SourceName, where)
+	fmt.Println(sql)
+	_, err = tx.Exec(sql)
+	if err != nil {
+		return err
+	}
+	err = tx.Commit()
+	if err != nil {
+		return err
+	}
+	tx = nil // set nil to prevent rollback
+	return nil
+}
+
 func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
 	db := &PostGIS{}
 	db.Tables = make(map[string]*TableSpec)
+	db.GeneralizedTables = make(map[string]*GeneralizedTableSpec)
+
 	db.Config = conf
 
 	if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") {
@@ -457,6 +574,11 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
 	for name, table := range m.Tables {
 		db.Tables[name] = NewTableSpec(db, table)
 	}
+	for name, table := range m.GeneralizedTables {
+		db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table)
+	}
+	db.checkGeneralizedTableSources()
+
 	err = db.Open()
 	if err != nil {
 		return nil, err

goposm.go

@@ -135,47 +135,57 @@ func main() {
 	}
 
 	if *write {
-		progress.Reset()
+		if true {
-		err = db.Init()
+			progress.Reset()
-		if err != nil {
+			err = db.Init()
-			log.Fatal(err)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			diffCache := cache.NewDiffCache(*cachedir)
+			if err = diffCache.Remove(); err != nil {
+				log.Fatal(err)
+			}
+			if err = diffCache.Open(); err != nil {
+				log.Fatal(err)
+			}
+
+			insertBuffer := writer.NewInsertBuffer()
+			dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
+
+			pointsTagMatcher := tagmapping.PointMatcher()
+			lineStringsTagMatcher := tagmapping.LineStringMatcher()
+			polygonsTagMatcher := tagmapping.PolygonMatcher()
+
+			relations := osmCache.Relations.Iter()
+			relWriter := writer.NewRelationWriter(osmCache, relations,
+				insertBuffer, polygonsTagMatcher, progress)
+			// blocks till the Relations.Iter() finishes
+			relWriter.Close()
+
+			ways := osmCache.Ways.Iter()
+			wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
+				lineStringsTagMatcher, polygonsTagMatcher, progress)
+
+			nodes := osmCache.Nodes.Iter()
+			nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
+				pointsTagMatcher, progress)
+
+			diffCache.Coords.Close()
+
+			wayWriter.Close()
+			nodeWriter.Close()
+			insertBuffer.Close()
+			dbWriter.Close()
 		}
 
-		diffCache := cache.NewDiffCache(*cachedir)
+		if db, ok := db.(database.Generalizer); ok {
-		if err = diffCache.Remove(); err != nil {
+			if err := db.Generalize(); err != nil {
-			log.Fatal(err)
+				log.Fatal(err)
+			}
+		} else {
+			log.Fatal("database not generalizeable")
 		}
-		if err = diffCache.Open(); err != nil {
-			log.Fatal(err)
-		}
-
-		insertBuffer := writer.NewInsertBuffer()
-		dbWriter := writer.NewDbWriter(db, insertBuffer.Out)
-
-		pointsTagMatcher := tagmapping.PointMatcher()
-		lineStringsTagMatcher := tagmapping.LineStringMatcher()
-		polygonsTagMatcher := tagmapping.PolygonMatcher()
-
-		relations := osmCache.Relations.Iter()
-		relWriter := writer.NewRelationWriter(osmCache, relations,
-			insertBuffer, polygonsTagMatcher, progress)
-		// blocks till the Relations.Iter() finishes
-		relWriter.Close()
-
-		ways := osmCache.Ways.Iter()
-		wayWriter := writer.NewWayWriter(osmCache, ways, insertBuffer,
-			lineStringsTagMatcher, polygonsTagMatcher, progress)
-
-		nodes := osmCache.Nodes.Iter()
-		nodeWriter := writer.NewNodeWriter(osmCache, nodes, insertBuffer,
-			pointsTagMatcher, progress)
-
-		diffCache.Coords.Close()
-
-		wayWriter.Close()
-		nodeWriter.Close()
-		insertBuffer.Close()
-		dbWriter.Close()
 
 		if db, ok := db.(database.Finisher); ok {
 			if err := db.Finish(); err != nil {

mapping.json

@@ -1,4 +1,46 @@
 {
+    "generalized_tables": {
+        "waterareas_gen1": {
+            "source": "waterareas", 
+            "sql_filter": "ST_Area(geometry)>50000.000000", 
+            "tolerance": 50.0
+        }, 
+        "waterareas_gen0": {
+            "source": "waterareas_gen1", 
+            "sql_filter": "ST_Area(geometry)>500000.000000", 
+            "tolerance": 200.0
+        }, 
+        "roads_gen0": {
+            "source": "roads_gen1", 
+            "sql_filter": null, 
+            "tolerance": 200.0
+        }, 
+        "roads_gen1": {
+            "source": "roads", 
+            "sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway')", 
+            "tolerance": 50.0
+        }, 
+        "waterways_gen0": {
+            "source": "waterways_gen1", 
+            "sql_filter": null, 
+            "tolerance": 200
+        }, 
+        "waterways_gen1": {
+            "source": "waterways", 
+            "sql_filter": null, 
+            "tolerance": 50.0
+        }, 
+        "landusages_gen1": {
+            "source": "landusages", 
+            "sql_filter": "ST_Area(geometry)>50000.000000", 
+            "tolerance": 50.0
+        }, 
+        "landusages_gen0": {
+            "source": "landusages_gen1", 
+            "sql_filter": "ST_Area(geometry)>500000.000000", 
+            "tolerance": 200.0
+        }
+    }, 
     "tables": {
         "landusages": {
             "fields": [
@@ -643,12 +685,12 @@
                     "key": "name"
                 }, 
                 {
-                    "type": "bool", 
+                    "type": "boolint", 
                     "name": "tunnel", 
                     "key": "tunnel"
                 }, 
                 {
-                    "type": "bool", 
+                    "type": "boolint", 
                     "name": "bridge", 
                     "key": "bridge"
                 }, 
@@ -676,6 +718,11 @@
                     "type": "string", 
                     "name": "service", 
                     "key": "service"
+                }, 
+                {
+                    "type": "mapping_key", 
+                    "name": "class", 
+                    "key": null
                 }
             ], 
             "type": "linestring", 

mapping/config.go

@@ -21,14 +21,24 @@ type Table struct {
 	Filters *Filters            `json:"filters"`
 }
 
+type GeneralizedTable struct {
+	Name            string
+	SourceTableName string  `json:"source"`
+	Tolerance       float64 `json:"tolerance"`
+	SqlFilter       string  `json:"sql_filter"`
+}
+
 type Filters struct {
 	ExcludeTags *map[string]string `json:"exclude_tags"`
 }
 
 type Tables map[string]*Table
 
+type GeneralizedTables map[string]*GeneralizedTable
+
 type Mapping struct {
-	Tables Tables `json:"tables"`
+	Tables            Tables            `json:"tables"`
+	GeneralizedTables GeneralizedTables `json:"generalized_tables"`
 }
 
 type ElementFilter func(elem *element.OSMElem) bool
@@ -66,6 +76,9 @@ func (m *Mapping) prepare() {
 	for name, t := range m.Tables {
 		t.Name = name
 	}
+	for name, t := range m.GeneralizedTables {
+		t.Name = name
+	}
 }
 
 func (m *Mapping) mappings(tableType string, mappings TagTables) {

mapping/fields.go

@@ -12,6 +12,7 @@ var AvailableFieldTypes map[string]FieldType
 func init() {
 	AvailableFieldTypes = map[string]FieldType{
 		"bool":          {"bool", "bool", Bool, nil},
+		"boolint":       {"boolint", "int8", BoolInt, nil},
 		"id":            {"id", "int64", Id, nil},
 		"string":        {"string", "string", String, nil},
 		"direction":     {"direction", "int8", Direction, nil},
@@ -92,6 +93,13 @@ func Bool(val string, elem *element.OSMElem, match Match) interface{} {
 	return true
 }
 
+func BoolInt(val string, elem *element.OSMElem, match Match) interface{} {
+	if val == "" || val == "0" || val == "false" || val == "no" {
+		return 0
+	}
+	return 1
+}
+
 func String(val string, elem *element.OSMElem, match Match) interface{} {
 	return val
 }