From e211940af34c66bc5ed13f6db62dfc37f88def4c Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Thu, 11 May 2017 11:08:06 +0200 Subject: [PATCH] separate mapping config from mapping logic create new imposm3/mapping/config package that includes all "dumb" config structs --- database/database.go | 11 +- database/postgis/postgis.go | 3 +- database/postgis/spec.go | 12 +- import_/import.go | 6 +- mapping/columns.go | 35 ++---- mapping/columns_test.go | 15 +-- mapping/config.go | 225 ++++++++++++------------------------ mapping/config/config.go | 113 ++++++++++++++++++ mapping/filter.go | 17 +-- mapping/filter_test.go | 7 +- update/process.go | 8 +- 11 files changed, 236 insertions(+), 216 deletions(-) create mode 100644 mapping/config/config.go diff --git a/database/database.go b/database/database.go index 50a8fdf..b02a8f9 100644 --- a/database/database.go +++ b/database/database.go @@ -7,6 +7,7 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) type Config struct { @@ -67,17 +68,17 @@ type Optimizer interface { Optimize() error } -var databases map[string]func(Config, *mapping.Mapping) (DB, error) +var databases map[string]func(Config, *config.Mapping) (DB, error) func init() { - databases = make(map[string]func(Config, *mapping.Mapping) (DB, error)) + databases = make(map[string]func(Config, *config.Mapping) (DB, error)) } -func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) { +func Register(name string, f func(Config, *config.Mapping) (DB, error)) { databases[name] = f } -func Open(conf Config, m *mapping.Mapping) (DB, error) { +func Open(conf Config, m *config.Mapping) (DB, error) { parts := strings.SplitN(conf.ConnectionParams, ":", 2) connectionType := parts[0] @@ -108,7 +109,7 @@ func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geo return nil } -func newNullDb(conf Config, m *mapping.Mapping) (DB, error) { +func newNullDb(conf Config, m *config.Mapping) (DB, error) { return &nullDb{}, nil } diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 23d16b8..1bae856 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -14,6 +14,7 @@ import ( "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/logging" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) var log = logging.NewLogger("PostGIS") @@ -599,7 +600,7 @@ func (pg *PostGIS) Close() error { return pg.Db.Close() } -func New(conf database.Config, m *mapping.Mapping) (database.DB, error) { +func New(conf database.Config, m *config.Mapping) (database.DB, error) { db := &PostGIS{} db.Tables = make(map[string]*TableSpec) diff --git a/database/postgis/spec.go b/database/postgis/spec.go index c0d8955..5ff3ba6 100644 --- a/database/postgis/spec.go +++ b/database/postgis/spec.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) type ColumnSpec struct { @@ -124,12 +125,11 @@ func (spec *TableSpec) DeleteSQL() string { ) } -func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { +func NewTableSpec(pg *PostGIS, t *config.Table) *TableSpec { var geomType string - switch t.Type { - case mapping.RelationMemberTable: + if mapping.TableType(t.Type) == mapping.RelationMemberTable { geomType = "geometry" - default: + } else { geomType = string(t.Type) } @@ -141,7 +141,7 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { Srid: pg.Config.Srid, } for _, column := range t.Columns { - columnType := column.ColumnType() + columnType := mapping.MakeColumnType(column) if columnType == nil { continue } @@ -156,7 +156,7 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { return &spec } -func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec { +func NewGeneralizedTableSpec(pg *PostGIS, t *config.GeneralizedTable) *GeneralizedTableSpec { spec := GeneralizedTableSpec{ Name: t.Name, FullName: pg.Prefix + t.Name, diff --git a/import_/import.go b/import_/import.go index 5feee39..951dd42 100644 --- a/import_/import.go +++ b/import_/import.go @@ -67,7 +67,7 @@ func Import() { ProductionSchema: config.BaseOptions.Schemas.Production, BackupSchema: config.BaseOptions.Schemas.Backup, } - db, err = database.Open(conf, tagmapping) + db, err = database.Open(conf, &tagmapping.Conf) if err != nil { log.Fatal(err) } @@ -181,7 +181,7 @@ func Import() { relations := osmCache.Relations.Iter() relWriter := writer.NewRelationWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, relations, db, progress, tagmapping.PolygonMatcher(), @@ -196,7 +196,7 @@ func Import() { ways := osmCache.Ways.Iter() wayWriter := writer.NewWayWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, ways, db, progress, tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(), diff --git a/mapping/columns.go b/mapping/columns.go index 68efc6e..5f8108d 100644 --- a/mapping/columns.go +++ b/mapping/columns.go @@ -11,6 +11,7 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/logging" + "github.com/omniscale/imposm3/mapping/config" ) var log = logging.NewLogger("mapping") @@ -47,7 +48,7 @@ func init() { type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{} type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{} -type MakeMakeValue func(string, ColumnType, Column) (MakeValue, error) +type MakeMakeValue func(string, ColumnType, config.Column) (MakeValue, error) type Key string type Value string @@ -100,24 +101,6 @@ func (t *TableSpec) MakeMemberRow(rel *element.Relation, member *element.Member, return row } -func (t *Table) TableSpec() *TableSpec { - result := TableSpec{} - - for _, mappingColumn := range t.Columns { - column := ColumnSpec{} - column.Key = mappingColumn.Key - - columnType := mappingColumn.ColumnType() - if columnType != nil { - column.Type = *columnType - } else { - log.Warn("unhandled type: ", mappingColumn.Type) - } - result.columns = append(result.columns, column) - } - return &result -} - type ColumnType struct { Name string GoType string @@ -200,7 +183,7 @@ func Geometry(val string, elem *element.OSMElem, geom *geom.Geometry, match Matc return string(geom.Wkb) } -func MakePseudoArea(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakePseudoArea(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { log.Print("warn: pseudoarea type is deprecated and will be removed. See area and webmercarea type.") return Area, nil } @@ -232,7 +215,7 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") -func MakeHStoreString(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakeHStoreString(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { var includeAll bool var err error var include map[string]int @@ -257,7 +240,7 @@ func MakeHStoreString(columnName string, columnType ColumnType, column Column) ( return hstoreString, nil } -func MakeWayZOrder(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakeWayZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { if _, ok := column.Args["ranks"]; !ok { return DefaultWayZOrder, nil } @@ -345,7 +328,7 @@ func DefaultWayZOrder(val string, elem *element.OSMElem, geom *geom.Geometry, ma return z } -func MakeZOrder(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakeZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { log.Print("warn: zorder type is deprecated and will be removed. See enumerate type.") _rankList, ok := column.Args["ranks"] if !ok { @@ -392,7 +375,7 @@ func MakeZOrder(columnName string, columnType ColumnType, column Column) (MakeVa return zOrder, nil } -func MakeEnumerate(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakeEnumerate(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { values, err := decodeEnumArg(column, "values") if err != nil { return nil, err @@ -413,7 +396,7 @@ func MakeEnumerate(columnName string, columnType ColumnType, column Column) (Mak return enumerate, nil } -func decodeEnumArg(column Column, key string) (map[string]int, error) { +func decodeEnumArg(column config.Column, key string) (map[string]int, error) { _valuesList, ok := column.Args[key] if !ok { return nil, fmt.Errorf("missing '%v' in args for %s", key, column.Type) @@ -436,7 +419,7 @@ func decodeEnumArg(column Column, key string) (map[string]int, error) { return values, nil } -func MakeSuffixReplace(columnName string, columnType ColumnType, column Column) (MakeValue, error) { +func MakeSuffixReplace(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { _changes, ok := column.Args["suffixes"] if !ok { return nil, errors.New("missing suffixes in args for string_suffixreplace") diff --git a/mapping/columns_test.go b/mapping/columns_test.go index e074ccb..53ee9bb 100644 --- a/mapping/columns_test.go +++ b/mapping/columns_test.go @@ -6,6 +6,7 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/geom/geos" + "github.com/omniscale/imposm3/mapping/config" ) func TestBool(t *testing.T) { @@ -75,7 +76,7 @@ func TestZOrder(t *testing.T) { zOrder, err := MakeZOrder("z_order", AvailableColumnTypes["z_order"], - Column{ + config.Column{ Name: "z_order", Key: "", Type: "z_order", @@ -114,7 +115,7 @@ func TestEnumerate_Match(t *testing.T) { zOrder, err := MakeEnumerate("enumerate", AvailableColumnTypes["enumerate"], - Column{ + config.Column{ Name: "enumerate", Key: "", Type: "enumerate", @@ -149,7 +150,7 @@ func TestEnumerate_Key(t *testing.T) { zOrder, err := MakeEnumerate("enumerate", AvailableColumnTypes["enumerate"], - Column{ + config.Column{ Name: "enumerate", Key: "fips", Type: "enumerate", @@ -183,7 +184,7 @@ func TestEnumerate_Key(t *testing.T) { func TestWayZOrder(t *testing.T) { zOrder, err := MakeWayZOrder("z_order", AvailableColumnTypes["wayzorder"], - Column{ + config.Column{ Name: "zorder", Type: "wayzorder", Args: map[string]interface{}{ @@ -277,7 +278,7 @@ func TestAreaColumn(t *testing.T) { } func TestMakeSuffixReplace(t *testing.T) { - column := Column{ + column := config.Column{ Name: "name", Key: "name", Type: "string_suffixreplace", Args: map[string]interface{}{"suffixes": map[interface{}]interface{}{"Straße": "Str.", "straße": "str."}}} suffixReplace, err := MakeSuffixReplace("name", ColumnType{}, column) @@ -298,7 +299,7 @@ func TestMakeSuffixReplace(t *testing.T) { } func TestHstoreString(t *testing.T) { - column := Column{ + column := config.Column{ Name: "tags", Type: "hstore_tags", } @@ -307,7 +308,7 @@ func TestHstoreString(t *testing.T) { t.Fatal(err) } - column = Column{ + column = config.Column{ Name: "tags", Type: "hstore_tags", Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}}, diff --git a/mapping/config.go b/mapping/config.go index 5aefd1f..3750c33 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -2,133 +2,14 @@ package mapping import ( "errors" - "fmt" "io/ioutil" "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" "gopkg.in/yaml.v2" ) -type Mapping struct { - Tables Tables `yaml:"tables"` - GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` - Tags Tags `yaml:"tags"` - Areas Areas `yaml:"areas"` - // SingleIdSpace mangles the overlapping node/way/relation IDs - // to be unique (nodes positive, ways negative, relations negative -1e17) - SingleIdSpace bool `yaml:"use_single_id_space"` -} - -type Column struct { - Name string `yaml:"name"` - Key Key `yaml:"key"` - Keys []Key `yaml:"keys"` - Type string `yaml:"type"` - Args map[string]interface{} `yaml:"args"` - FromMember bool `yaml:"from_member"` -} - -func (c *Column) ColumnType() *ColumnType { - if columnType, ok := AvailableColumnTypes[c.Type]; ok { - if columnType.MakeFunc != nil { - makeValue, err := columnType.MakeFunc(c.Name, columnType, *c) - if err != nil { - log.Print(err) - return nil - } - columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember} - } - columnType.FromMember = c.FromMember - return &columnType - } - return nil -} - -type Tables map[string]*Table -type Table struct { - Name string - Type TableType `yaml:"type"` - Mapping KeyValues `yaml:"mapping"` - Mappings map[string]SubMapping `yaml:"mappings"` - TypeMappings TypeMappings `yaml:"type_mappings"` - Columns []*Column `yaml:"columns"` - OldFields []*Column `yaml:"fields"` - Filters *Filters `yaml:"filters"` - RelationTypes []string `yaml:"relation_types"` -} - -type GeneralizedTables map[string]*GeneralizedTable -type GeneralizedTable struct { - Name string - SourceTableName string `yaml:"source"` - Tolerance float64 `yaml:"tolerance"` - SqlFilter string `yaml:"sql_filter"` -} - -type Filters struct { - ExcludeTags *[][]string `yaml:"exclude_tags"` -} - -type Areas struct { - AreaTags []Key `yaml:"area_tags"` - LinearTags []Key `yaml:"linear_tags"` -} - -type Tags struct { - LoadAll bool `yaml:"load_all"` - Exclude []Key `yaml:"exclude"` - Include []Key `yaml:"include"` -} - -type orderedValue struct { - value Value - order int -} - -type KeyValues map[Key][]orderedValue - -func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { - if *kv == nil { - *kv = make(map[Key][]orderedValue) - } - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - order := 0 - for _, item := range slice { - k, ok := item.Key.(string) - if !ok { - return fmt.Errorf("mapping key '%s' not a string", k) - } - values, ok := item.Value.([]interface{}) - if !ok { - return fmt.Errorf("mapping key '%s' not a string", k) - } - for _, v := range values { - if v, ok := v.(string); ok { - (*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order}) - } else { - return fmt.Errorf("mapping value '%s' not a string", v) - } - order += 1 - } - } - return nil -} - -type SubMapping struct { - Mapping KeyValues -} - -type TypeMappings struct { - Points KeyValues `yaml:"points"` - LineStrings KeyValues `yaml:"linestrings"` - Polygons KeyValues `yaml:"polygons"` -} - type ElementFilter func(tags element.Tags, key Key, closed bool) bool type orderedDestTable struct { @@ -138,16 +19,16 @@ type orderedDestTable struct { type TagTableMapping map[Key]map[Value][]orderedDestTable -func (tt TagTableMapping) addFromMapping(mapping KeyValues, table DestTable) { +func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) { for key, vals := range mapping { for _, v := range vals { - vals, ok := tt[key] - tbl := orderedDestTable{DestTable: table, order: v.order} + vals, ok := tt[Key(key)] + tbl := orderedDestTable{DestTable: table, order: v.Order} if ok { - vals[v.value] = append(vals[v.value], tbl) + vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl) } else { - tt[key] = make(map[Value][]orderedDestTable) - tt[key][v.value] = append(tt[key][v.value], tbl) + tt[Key(key)] = make(map[Value][]orderedDestTable) + tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl) } } } @@ -202,6 +83,10 @@ const ( RelationMemberTable TableType = "relation_member" ) +type Mapping struct { + Conf config.Mapping +} + func NewMapping(filename string) (*Mapping, error) { f, err := ioutil.ReadFile(filename) if err != nil { @@ -209,7 +94,7 @@ func NewMapping(filename string) (*Mapping, error) { } mapping := Mapping{} - err = yaml.Unmarshal(f, &mapping) + err = yaml.Unmarshal(f, &mapping.Conf) if err != nil { return nil, err } @@ -222,7 +107,7 @@ func NewMapping(filename string) (*Mapping, error) { } func (m *Mapping) prepare() error { - for name, t := range m.Tables { + for name, t := range m.Conf.Tables { t.Name = name if t.OldFields != nil { // todo deprecate 'fields' @@ -230,15 +115,15 @@ func (m *Mapping) prepare() error { } } - for name, t := range m.GeneralizedTables { + for name, t := range m.Conf.GeneralizedTables { t.Name = name } return nil } func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) { - for name, t := range m.Tables { - if t.Type != GeometryTable && t.Type != tableType { + for name, t := range m.Conf.Tables { + if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType { continue } mappings.addFromMapping(t.Mapping, DestTable{Name: name}) @@ -260,26 +145,60 @@ func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) { func (m *Mapping) tables(tableType TableType) map[string]*TableSpec { result := make(map[string]*TableSpec) - for name, t := range m.Tables { - if t.Type == tableType || t.Type == GeometryTable { - result[name] = t.TableSpec() + for name, t := range m.Conf.Tables { + if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable { + result[name] = makeTableSpec(t) } } return result } +func makeTableSpec(tbl *config.Table) *TableSpec { + result := TableSpec{} + + for _, mappingColumn := range tbl.Columns { + column := ColumnSpec{} + column.Key = Key(mappingColumn.Key) + + columnType := MakeColumnType(mappingColumn) + if columnType != nil { + column.Type = *columnType + } else { + log.Warn("unhandled type: ", mappingColumn.Type) + } + result.columns = append(result.columns, column) + } + return &result +} + +func MakeColumnType(c *config.Column) *ColumnType { + if columnType, ok := AvailableColumnTypes[c.Type]; ok { + if columnType.MakeFunc != nil { + makeValue, err := columnType.MakeFunc(c.Name, columnType, *c) + if err != nil { + log.Print(err) + return nil + } + columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember} + } + columnType.FromMember = c.FromMember + return &columnType + } + return nil +} + func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { - for _, t := range m.Tables { - if t.Type != tableType && t.Type != GeometryTable { + for _, t := range m.Conf.Tables { + if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable { continue } for _, col := range t.Columns { if col.Key != "" { - tags[col.Key] = true + tags[Key(col.Key)] = true } for _, k := range col.Keys { - tags[k] = true + tags[Key(k)] = true } } @@ -295,8 +214,8 @@ func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { } } } - for _, k := range m.Tags.Include { - tags[k] = true + for _, k := range m.Conf.Tags.Include { + tags[Key(k)] = true } // always include area tag for closed-way handling @@ -308,24 +227,24 @@ type tableElementFilters map[string][]ElementFilter func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) { var areaTags map[Key]struct{} var linearTags map[Key]struct{} - if m.Areas.AreaTags != nil { + if m.Conf.Areas.AreaTags != nil { areaTags = make(map[Key]struct{}) - for _, tag := range m.Areas.AreaTags { - areaTags[tag] = struct{}{} + for _, tag := range m.Conf.Areas.AreaTags { + areaTags[Key(tag)] = struct{}{} } } - if m.Areas.LinearTags != nil { + if m.Conf.Areas.LinearTags != nil { linearTags = make(map[Key]struct{}) - for _, tag := range m.Areas.LinearTags { - linearTags[tag] = struct{}{} + for _, tag := range m.Conf.Areas.LinearTags { + linearTags[Key(tag)] = struct{}{} } } - for name, t := range m.Tables { - if t.Type != GeometryTable && t.Type != tableType { + for name, t := range m.Conf.Tables { + if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType { continue } - if t.Type == LineStringTable && areaTags != nil { + if TableType(t.Type) == LineStringTable && areaTags != nil { f := func(tags element.Tags, key Key, closed bool) bool { if closed { if tags["area"] == "yes" { @@ -341,7 +260,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilte } filters[name] = append(filters[name], f) } - if t.Type == PolygonTable && linearTags != nil { + if TableType(t.Type) == PolygonTable && linearTags != nil { f := func(tags element.Tags, key Key, closed bool) bool { if closed && tags["area"] == "no" { return false @@ -359,7 +278,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilte } func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) { - for name, t := range m.Tables { + for name, t := range m.Conf.Tables { if t.RelationTypes != nil { relTypes := t.RelationTypes // copy loop var for closure f := func(tags element.Tags, key Key, closed bool) bool { @@ -374,7 +293,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFi } filters[name] = append(filters[name], f) } else { - if t.Type == PolygonTable { + if TableType(t.Type) == PolygonTable { // standard mulipolygon handling (boundary and land_area are for backwards compatibility) f := func(tags element.Tags, key Key, closed bool) bool { if v, ok := tags["type"]; ok { @@ -391,7 +310,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFi } func (m *Mapping) addFilters(filters tableElementFilters) { - for name, t := range m.Tables { + for name, t := range m.Conf.Tables { if t.Filters == nil { continue } diff --git a/mapping/config/config.go b/mapping/config/config.go new file mode 100644 index 0000000..53a1c7c --- /dev/null +++ b/mapping/config/config.go @@ -0,0 +1,113 @@ +package config + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +type Mapping struct { + Tables Tables `yaml:"tables"` + GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` + Tags Tags `yaml:"tags"` + Areas Areas `yaml:"areas"` + // SingleIdSpace mangles the overlapping node/way/relation IDs + // to be unique (nodes positive, ways negative, relations negative -1e17) + SingleIdSpace bool `yaml:"use_single_id_space"` +} + +type Column struct { + Name string `yaml:"name"` + Key Key `yaml:"key"` + Keys []Key `yaml:"keys"` + Type string `yaml:"type"` + Args map[string]interface{} `yaml:"args"` + FromMember bool `yaml:"from_member"` +} + +type Tables map[string]*Table +type Table struct { + Name string + Type string `yaml:"type"` + Mapping KeyValues `yaml:"mapping"` + Mappings map[string]SubMapping `yaml:"mappings"` + TypeMappings TypeMappings `yaml:"type_mappings"` + Columns []*Column `yaml:"columns"` + OldFields []*Column `yaml:"fields"` + Filters *Filters `yaml:"filters"` + RelationTypes []string `yaml:"relation_types"` +} + +type GeneralizedTables map[string]*GeneralizedTable +type GeneralizedTable struct { + Name string + SourceTableName string `yaml:"source"` + Tolerance float64 `yaml:"tolerance"` + SqlFilter string `yaml:"sql_filter"` +} + +type Filters struct { + ExcludeTags *[][]string `yaml:"exclude_tags"` +} + +type Areas struct { + AreaTags []Key `yaml:"area_tags"` + LinearTags []Key `yaml:"linear_tags"` +} + +type Tags struct { + LoadAll bool `yaml:"load_all"` + Exclude []Key `yaml:"exclude"` + Include []Key `yaml:"include"` +} + +type Key string +type Value string + +type orderedValue struct { + Value Value + Order int +} + +type KeyValues map[Key][]orderedValue + +func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { + if *kv == nil { + *kv = make(map[Key][]orderedValue) + } + slice := yaml.MapSlice{} + err := unmarshal(&slice) + if err != nil { + return err + } + order := 0 + for _, item := range slice { + k, ok := item.Key.(string) + if !ok { + return fmt.Errorf("mapping key '%s' not a string", k) + } + values, ok := item.Value.([]interface{}) + if !ok { + return fmt.Errorf("mapping key '%s' not a string", k) + } + for _, v := range values { + if v, ok := v.(string); ok { + (*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{Value: Value(v), Order: order}) + } else { + return fmt.Errorf("mapping value '%s' not a string", v) + } + order += 1 + } + } + return nil +} + +type SubMapping struct { + Mapping KeyValues +} + +type TypeMappings struct { + Points KeyValues `yaml:"points"` + LineStrings KeyValues `yaml:"linestrings"` + Polygons KeyValues `yaml:"polygons"` +} diff --git a/mapping/filter.go b/mapping/filter.go index fb31580..4ea2e58 100644 --- a/mapping/filter.go +++ b/mapping/filter.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" ) type TagFilterer interface { @@ -12,8 +13,8 @@ type TagFilterer interface { } func (m *Mapping) NodeTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } mappings := make(TagTableMapping) m.mappings(PointTable, mappings) @@ -24,8 +25,8 @@ func (m *Mapping) NodeTagFilter() TagFilterer { } func (m *Mapping) WayTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } mappings := make(TagTableMapping) m.mappings(LineStringTable, mappings) @@ -38,8 +39,8 @@ func (m *Mapping) WayTagFilter() TagFilterer { } func (m *Mapping) RelationTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } mappings := make(TagTableMapping) // do not filter out type tag for common relations @@ -92,7 +93,7 @@ type excludeFilter struct { matches []string } -func newExcludeFilter(tags []Key) *excludeFilter { +func newExcludeFilter(tags []config.Key) *excludeFilter { f := excludeFilter{ keys: make(map[Key]struct{}), matches: make([]string, 0), @@ -101,7 +102,7 @@ func newExcludeFilter(tags []Key) *excludeFilter { if strings.ContainsAny(string(t), "?*[") { f.matches = append(f.matches, string(t)) } else { - f.keys[t] = struct{}{} + f.keys[Key(t)] = struct{}{} } } return &f diff --git a/mapping/filter_test.go b/mapping/filter_test.go index 7e55175..83becf6 100644 --- a/mapping/filter_test.go +++ b/mapping/filter_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" ) var mapping *Mapping @@ -349,7 +350,7 @@ func TestExcludeFilter(t *testing.T) { var tags element.Tags // no matches - f = newExcludeFilter([]Key{}) + f = newExcludeFilter([]config.Key{}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}) { @@ -357,7 +358,7 @@ func TestExcludeFilter(t *testing.T) { } // match all - f = newExcludeFilter([]Key{"*"}) + f = newExcludeFilter([]config.Key{"*"}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{}) { @@ -365,7 +366,7 @@ func TestExcludeFilter(t *testing.T) { } // fixed string and wildcard match - f = newExcludeFilter([]Key{"source", "tiger:*"}) + f = newExcludeFilter([]config.Key{"source", "tiger:*"}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{"source:foo": "1"}) { diff --git a/update/process.go b/update/process.go index 260d080..b9513b2 100644 --- a/update/process.go +++ b/update/process.go @@ -119,7 +119,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi ProductionSchema: config.BaseOptions.Schemas.Production, BackupSchema: config.BaseOptions.Schemas.Backup, } - db, err := database.Open(dbConf, tagmapping) + db, err := database.Open(dbConf, &tagmapping.Conf) if err != nil { return errors.New("database open: " + err.Error()) } @@ -144,7 +144,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi delDb, osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, tagmapping.PointMatcher(), tagmapping.LineStringMatcher(), tagmapping.PolygonMatcher(), @@ -162,7 +162,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi nodes := make(chan *element.Node) relWriter := writer.NewRelationWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, relations, db, progress, tagmapping.PolygonMatcher(), @@ -174,7 +174,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi relWriter.Start() wayWriter := writer.NewWayWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, ways, db, progress, tagmapping.PolygonMatcher(),