diff --git a/database/database.go b/database/database.go index 50a8fdf..b02a8f9 100644 --- a/database/database.go +++ b/database/database.go @@ -7,6 +7,7 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) type Config struct { @@ -67,17 +68,17 @@ type Optimizer interface { Optimize() error } -var databases map[string]func(Config, *mapping.Mapping) (DB, error) +var databases map[string]func(Config, *config.Mapping) (DB, error) func init() { - databases = make(map[string]func(Config, *mapping.Mapping) (DB, error)) + databases = make(map[string]func(Config, *config.Mapping) (DB, error)) } -func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) { +func Register(name string, f func(Config, *config.Mapping) (DB, error)) { databases[name] = f } -func Open(conf Config, m *mapping.Mapping) (DB, error) { +func Open(conf Config, m *config.Mapping) (DB, error) { parts := strings.SplitN(conf.ConnectionParams, ":", 2) connectionType := parts[0] @@ -108,7 +109,7 @@ func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geo return nil } -func newNullDb(conf Config, m *mapping.Mapping) (DB, error) { +func newNullDb(conf Config, m *config.Mapping) (DB, error) { return &nullDb{}, nil } diff --git a/database/postgis/fields.go b/database/postgis/columns.go similarity index 100% rename from database/postgis/fields.go rename to database/postgis/columns.go diff --git a/database/postgis/postgis.go b/database/postgis/postgis.go index 23d16b8..1bae856 100644 --- a/database/postgis/postgis.go +++ b/database/postgis/postgis.go @@ -14,6 +14,7 @@ import ( "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/logging" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) var log = logging.NewLogger("PostGIS") @@ -599,7 +600,7 @@ func (pg *PostGIS) Close() error { return pg.Db.Close() } -func New(conf database.Config, m *mapping.Mapping) (database.DB, error) { +func New(conf database.Config, m *config.Mapping) (database.DB, error) { db := &PostGIS{} db.Tables = make(map[string]*TableSpec) diff --git a/database/postgis/spec.go b/database/postgis/spec.go index 8c50428..5ff3ba6 100644 --- a/database/postgis/spec.go +++ b/database/postgis/spec.go @@ -5,11 +5,12 @@ import ( "strings" "github.com/omniscale/imposm3/mapping" + "github.com/omniscale/imposm3/mapping/config" ) type ColumnSpec struct { Name string - FieldType mapping.FieldType + FieldType mapping.ColumnType Type ColumnType } type TableSpec struct { @@ -124,12 +125,11 @@ func (spec *TableSpec) DeleteSQL() string { ) } -func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { +func NewTableSpec(pg *PostGIS, t *config.Table) *TableSpec { var geomType string - switch t.Type { - case mapping.RelationMemberTable: + if mapping.TableType(t.Type) == mapping.RelationMemberTable { geomType = "geometry" - default: + } else { geomType = string(t.Type) } @@ -140,23 +140,23 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec { GeometryType: geomType, Srid: pg.Config.Srid, } - for _, field := range t.Fields { - fieldType := field.FieldType() - if fieldType == nil { + for _, column := range t.Columns { + columnType := mapping.MakeColumnType(column) + if columnType == nil { continue } - pgType, ok := pgTypes[fieldType.GoType] + pgType, ok := pgTypes[columnType.GoType] if !ok { - log.Errorf("unhandled field type %v, using string type", fieldType) + log.Errorf("unhandled column type %v, using string type", columnType) pgType = pgTypes["string"] } - col := ColumnSpec{field.Name, *fieldType, pgType} + col := ColumnSpec{column.Name, *columnType, pgType} spec.Columns = append(spec.Columns, col) } return &spec } -func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec { +func NewGeneralizedTableSpec(pg *PostGIS, t *config.GeneralizedTable) *GeneralizedTableSpec { spec := GeneralizedTableSpec{ Name: t.Name, FullName: pg.Prefix + t.Name, diff --git a/docs/mapping.rst b/docs/mapping.rst index 32161b1..91f847d 100644 --- a/docs/mapping.rst +++ b/docs/mapping.rst @@ -38,10 +38,28 @@ To import all polygons with `tourism=zoo`, `natural=wood` or `natural=land` into … +``relation_types`` +~~~~~~~~~~~~~~~~~~ + +``relation_types`` restricts which relation types should be imported. It is a list with `type` values, e.g. ``[route, master_route]``. + +For tables of type ``relation`` and ``relation_member``: Only import relations which have this type value. You still need to have a mapping. +For tables of type ``polygon``: Only build multi-polygons for relations which have this type value. You still need to have a mapping. Defaults to ``[multipolygon, boundary, land_area]``. + +.. code-block:: yaml + + tables: + routes: + type: relation + relation_types: [route] + mapping: + route: [bus] + + ``columns`` ~~~~~~~~~~~ -``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionaly ``key``, ``args`` and ``from_member``. +``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionally ``key``, ``args`` and ``from_member``. ``name`` ^^^^^^^^^ @@ -349,6 +367,16 @@ To load all tags except ``created_by``, ``source``, and ``tiger:county``, ``tige +To load specific data about amenities for inclusion into an `hstore_tags` column: + +.. code-block:: yaml + + tags: + include: [operator, opening_hours, wheelchair, website, phone, cuisine] + + + + .. _Areas: Areas diff --git a/docs/relations.rst b/docs/relations.rst index 0dbf512..15096e1 100644 --- a/docs/relations.rst +++ b/docs/relations.rst @@ -42,21 +42,7 @@ It will also insert relations of the type ``multipolygon`` with a ``building`` t The roles are ignored by Imposm as not all holes are correctly tagged as ``inner``. Imposm uses geometry operations to verify if a member of a multipolygon is a hole, or if it is a separate polygon. - -For compatibility, multipolygon relations without tags will use the tags from the (longest) outer way. Imposm will insert the following relation as well:: - - - - ... - - - - - - - - - +Old-style multipolygon relations with tags on the outer way, instead of the relation are no longer supported. Other relations @@ -72,7 +58,6 @@ These relations can not be mapped to `simple` linestrings or polygons as they ca The Imposm table types ``relation`` and ``relation_member`` allow you to import all relevant data for these relations. -.. note:: ``relation`` and ``relation_member`` require :ref:`load_all` to have access to all keys. ``relation_member`` ^^^^^^^^^^^^^^^^^^^ @@ -109,6 +94,7 @@ You can use the following mapping:: - key: ref name: ref type: string + relation_type: [route] mapping: route: [bus] @@ -171,6 +157,7 @@ The following mapping imports the bus route relation from above:: - name: network key: network type: string + relation_type: [route] mapping: route: [bus] diff --git a/geom/multipolygon.go b/geom/multipolygon.go index e5e631a..fd00605 100644 --- a/geom/multipolygon.go +++ b/geom/multipolygon.go @@ -16,15 +16,12 @@ type PreparedRelation struct { // PrepareRelation is the first step in building a (multi-)polygon of a Relation. // It builds rings from all ways and returns an error if there are unclosed rings. -// It also merges the Relation.Tags with the Tags of the outer way. func PrepareRelation(rel *element.Relation, srid int, maxRingGap float64) (PreparedRelation, error) { rings, err := buildRings(rel, maxRingGap) if err != nil { return PreparedRelation{}, err } - rel.Tags = relationTags(rel.Tags, rings[0].ways[0].Tags) - return PreparedRelation{rings, rel, srid}, nil } @@ -224,29 +221,6 @@ func buildRelGeometry(g *geos.Geos, rel *element.Relation, rings []*ring) (*geos return result, nil } -func relationTags(relTags, wayTags element.Tags) element.Tags { - result := make(element.Tags) - for k, v := range relTags { - if k == "name" || k == "type" { - continue - } - result[k] = v - } - - if len(result) == 0 { - // relation does not have tags? use way tags - for k, v := range wayTags { - result[k] = v - } - } else { - // add back name (if present) - if name, ok := relTags["name"]; ok { - result["name"] = name - } - } - return result -} - // ringIsHole returns true if rings[idx] is a hole, False if it is a // shell (also if hole in a hole, etc) func ringIsHole(rings []*ring, idx int) bool { diff --git a/geom/multipolygon_test.go b/geom/multipolygon_test.go index dc31b1c..ffa1926 100644 --- a/geom/multipolygon_test.go +++ b/geom/multipolygon_test.go @@ -94,7 +94,7 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) { }) rel := element.Relation{ - OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "rel"}}} + OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "Relation", "natural": "forest", "type": "multipolygon"}}} rel.Members = []element.Member{ {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, @@ -107,10 +107,11 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) { g := geos.NewGeos() defer g.Finish() - if len(rel.Tags) != 2 { + if len(rel.Tags) != 3 { t.Fatal("wrong rel tags", rel.Tags) } - if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Blackwood" { + // name from way is ignored + if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Relation" { t.Fatal("wrong rel tags", rel.Tags) } @@ -147,7 +148,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) { }) rel := element.Relation{ - OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}} + OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}} rel.Members = []element.Member{ {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, @@ -161,7 +162,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) { g := geos.NewGeos() defer g.Finish() - if len(rel.Tags) != 1 { + if len(rel.Tags) != 2 { t.Fatal("wrong rel tags", rel.Tags) } if rel.Tags["landusage"] != "forest" { @@ -214,7 +215,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) { {1, 4, 4}, }) - rel := element.Relation{OSMElem: element.OSMElem{Id: 1}} + rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}} rel.Members = []element.Member{ {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, @@ -230,7 +231,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) { g := geos.NewGeos() defer g.Finish() - if len(rel.Tags) != 1 { + if len(rel.Tags) != 2 { t.Fatal("wrong rel tags", rel.Tags) } if rel.Tags["landusage"] != "forest" { @@ -261,7 +262,7 @@ func TestPolygonFromThreeWays(t *testing.T) { {1, 0, 0}, }) - rel := element.Relation{OSMElem: element.OSMElem{Id: 1}} + rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}} rel.Members = []element.Member{ {Id: 1, Type: element.WAY, Role: "outer", Way: &w1}, {Id: 2, Type: element.WAY, Role: "inner", Way: &w2}, @@ -275,7 +276,7 @@ func TestPolygonFromThreeWays(t *testing.T) { g := geos.NewGeos() defer g.Finish() - if len(rel.Tags) != 1 { + if len(rel.Tags) != 2 { t.Fatal("wrong rel tags", rel.Tags) } if rel.Tags["landusage"] != "forest" { diff --git a/import_/import.go b/import_/import.go index 5feee39..951dd42 100644 --- a/import_/import.go +++ b/import_/import.go @@ -67,7 +67,7 @@ func Import() { ProductionSchema: config.BaseOptions.Schemas.Production, BackupSchema: config.BaseOptions.Schemas.Backup, } - db, err = database.Open(conf, tagmapping) + db, err = database.Open(conf, &tagmapping.Conf) if err != nil { log.Fatal(err) } @@ -181,7 +181,7 @@ func Import() { relations := osmCache.Relations.Iter() relWriter := writer.NewRelationWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, relations, db, progress, tagmapping.PolygonMatcher(), @@ -196,7 +196,7 @@ func Import() { ways := osmCache.Ways.Iter() wayWriter := writer.NewWayWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, ways, db, progress, tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(), diff --git a/mapping/fields.go b/mapping/columns.go similarity index 74% rename from mapping/fields.go rename to mapping/columns.go index 3a51f35..b6ef72d 100644 --- a/mapping/fields.go +++ b/mapping/columns.go @@ -11,14 +11,15 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/logging" + "github.com/omniscale/imposm3/mapping/config" ) var log = logging.NewLogger("mapping") -var AvailableFieldTypes map[string]FieldType +var AvailableColumnTypes map[string]ColumnType func init() { - AvailableFieldTypes = map[string]FieldType{ + AvailableColumnTypes = map[string]ColumnType{ "bool": {"bool", "bool", Bool, nil, nil, false}, "boolint": {"boolint", "int8", BoolInt, nil, nil, false}, "id": {"id", "int64", Id, nil, nil, false}, @@ -47,94 +48,12 @@ func init() { type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{} type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{} -type MakeMakeValue func(string, FieldType, Field) (MakeValue, error) +type MakeMakeValue func(string, ColumnType, config.Column) (MakeValue, error) type Key string type Value string -type FieldSpec struct { - Key Key - Type FieldType -} - -func (f *FieldSpec) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { - if f.Type.Func != nil { - return f.Type.Func(elem.Tags[string(f.Key)], elem, geom, match) - } - return nil -} - -func (f *FieldSpec) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} { - if f.Type.Func != nil { - if f.Type.FromMember { - if member.Elem == nil { - return nil - } - return f.Type.Func(member.Elem.Tags[string(f.Key)], member.Elem, geom, match) - } - return f.Type.Func(rel.Tags[string(f.Key)], &rel.OSMElem, geom, match) - } - if f.Type.MemberFunc != nil { - return f.Type.MemberFunc(rel, member, match) - } - return nil -} - -type TableFields struct { - fields []FieldSpec -} - -func (t *TableFields) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} { - var row []interface{} - for _, field := range t.fields { - row = append(row, field.Value(elem, geom, match)) - } - return row -} - -func (t *TableFields) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} { - var row []interface{} - for _, field := range t.fields { - row = append(row, field.MemberValue(rel, member, geom, match)) - } - return row -} - -func (field *Field) FieldType() *FieldType { - if fieldType, ok := AvailableFieldTypes[field.Type]; ok { - if fieldType.MakeFunc != nil { - makeValue, err := fieldType.MakeFunc(field.Name, fieldType, *field) - if err != nil { - log.Print(err) - return nil - } - fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMember} - } - fieldType.FromMember = field.FromMember - return &fieldType - } - return nil -} - -func (t *Table) TableFields() *TableFields { - result := TableFields{} - - for _, mappingField := range t.Fields { - field := FieldSpec{} - field.Key = mappingField.Key - - fieldType := mappingField.FieldType() - if fieldType != nil { - field.Type = *fieldType - } else { - log.Warn("unhandled type: ", mappingField.Type) - } - result.fields = append(result.fields, field) - } - return &result -} - -type FieldType struct { +type ColumnType struct { Name string GoType string Func MakeValue @@ -216,7 +135,7 @@ func Geometry(val string, elem *element.OSMElem, geom *geom.Geometry, match Matc return string(geom.Wkb) } -func MakePseudoArea(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { +func MakePseudoArea(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { log.Print("warn: pseudoarea type is deprecated and will be removed. See area and webmercarea type.") return Area, nil } @@ -248,14 +167,14 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") -func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { +func MakeHStoreString(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { var includeAll bool var err error var include map[string]int - if _, ok := field.Args["include"]; !ok { + if _, ok := column.Args["include"]; !ok { includeAll = true } else { - include, err = decodeEnumArg(field, "include") + include, err = decodeEnumArg(column, "include") if err != nil { return nil, err } @@ -273,18 +192,18 @@ func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeV return hstoreString, nil } -func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { - if _, ok := field.Args["ranks"]; !ok { +func MakeWayZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { + if _, ok := column.Args["ranks"]; !ok { return DefaultWayZOrder, nil } - ranks, err := decodeEnumArg(field, "ranks") + ranks, err := decodeEnumArg(column, "ranks") if err != nil { return nil, err } levelOffset := len(ranks) defaultRank := 0 - if val, ok := field.Args["default"].(float64); ok { + if val, ok := column.Args["default"].(float64); ok { defaultRank = int(val) } @@ -361,9 +280,9 @@ func DefaultWayZOrder(val string, elem *element.OSMElem, geom *geom.Geometry, ma return z } -func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { +func MakeZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { log.Print("warn: zorder type is deprecated and will be removed. See enumerate type.") - _rankList, ok := field.Args["ranks"] + _rankList, ok := column.Args["ranks"] if !ok { return nil, errors.New("missing ranks in args for zorder") } @@ -374,7 +293,7 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, } var key string - _key, ok := field.Args["key"] + _key, ok := column.Args["key"] if ok { key, ok = _key.(string) if !ok { @@ -408,13 +327,13 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, return zOrder, nil } -func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { - values, err := decodeEnumArg(field, "values") +func MakeEnumerate(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { + values, err := decodeEnumArg(column, "values") if err != nil { return nil, err } enumerate := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { - if field.Key != "" { + if column.Key != "" { if r, ok := values[val]; ok { return r } @@ -429,15 +348,15 @@ func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValu return enumerate, nil } -func decodeEnumArg(field Field, key string) (map[string]int, error) { - _valuesList, ok := field.Args[key] +func decodeEnumArg(column config.Column, key string) (map[string]int, error) { + _valuesList, ok := column.Args[key] if !ok { - return nil, fmt.Errorf("missing '%v' in args for %s", key, field.Type) + return nil, fmt.Errorf("missing '%v' in args for %s", key, column.Type) } valuesList, ok := _valuesList.([]interface{}) if !ok { - return nil, fmt.Errorf("'%v' in args for %s not a list", key, field.Type) + return nil, fmt.Errorf("'%v' in args for %s not a list", key, column.Type) } values := make(map[string]int) @@ -452,8 +371,8 @@ func decodeEnumArg(field Field, key string) (map[string]int, error) { return values, nil } -func MakeSuffixReplace(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { - _changes, ok := field.Args["suffixes"] +func MakeSuffixReplace(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) { + _changes, ok := column.Args["suffixes"] if !ok { return nil, errors.New("missing suffixes in args for string_suffixreplace") } diff --git a/mapping/fields_test.go b/mapping/columns_test.go similarity index 93% rename from mapping/fields_test.go rename to mapping/columns_test.go index b71fb99..53ee9bb 100644 --- a/mapping/fields_test.go +++ b/mapping/columns_test.go @@ -6,6 +6,7 @@ import ( "github.com/omniscale/imposm3/element" "github.com/omniscale/imposm3/geom" "github.com/omniscale/imposm3/geom/geos" + "github.com/omniscale/imposm3/mapping/config" ) func TestBool(t *testing.T) { @@ -74,8 +75,8 @@ func TestZOrder(t *testing.T) { match := Match{} zOrder, err := MakeZOrder("z_order", - AvailableFieldTypes["z_order"], - Field{ + AvailableColumnTypes["z_order"], + config.Column{ Name: "z_order", Key: "", Type: "z_order", @@ -113,8 +114,8 @@ func TestEnumerate_Match(t *testing.T) { // test enumerate by matched mapping key zOrder, err := MakeEnumerate("enumerate", - AvailableFieldTypes["enumerate"], - Field{ + AvailableColumnTypes["enumerate"], + config.Column{ Name: "enumerate", Key: "", Type: "enumerate", @@ -148,8 +149,8 @@ func TestEnumerate_Key(t *testing.T) { // test enumerate by key zOrder, err := MakeEnumerate("enumerate", - AvailableFieldTypes["enumerate"], - Field{ + AvailableColumnTypes["enumerate"], + config.Column{ Name: "enumerate", Key: "fips", Type: "enumerate", @@ -182,8 +183,8 @@ func TestEnumerate_Key(t *testing.T) { func TestWayZOrder(t *testing.T) { zOrder, err := MakeWayZOrder("z_order", - AvailableFieldTypes["wayzorder"], - Field{ + AvailableColumnTypes["wayzorder"], + config.Column{ Name: "zorder", Type: "wayzorder", Args: map[string]interface{}{ @@ -233,7 +234,7 @@ func TestWayZOrder(t *testing.T) { } } -func TestAreaFields(t *testing.T) { +func TestAreaColumn(t *testing.T) { tests := []struct { wkt string expected float32 @@ -277,10 +278,10 @@ func TestAreaFields(t *testing.T) { } func TestMakeSuffixReplace(t *testing.T) { - field := Field{ + column := config.Column{ Name: "name", Key: "name", Type: "string_suffixreplace", Args: map[string]interface{}{"suffixes": map[interface{}]interface{}{"Straße": "Str.", "straße": "str."}}} - suffixReplace, err := MakeSuffixReplace("name", FieldType{}, field) + suffixReplace, err := MakeSuffixReplace("name", ColumnType{}, column) if err != nil { t.Fatal(err) @@ -298,27 +299,27 @@ func TestMakeSuffixReplace(t *testing.T) { } func TestHstoreString(t *testing.T) { - field := Field{ + column := config.Column{ Name: "tags", Type: "hstore_tags", } - hstoreAll, err := MakeHStoreString("tags", FieldType{}, field) + hstoreAll, err := MakeHStoreString("tags", ColumnType{}, column) if err != nil { t.Fatal(err) } - field = Field{ + column = config.Column{ Name: "tags", Type: "hstore_tags", Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}}, } - hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field) + hstoreInclude, err := MakeHStoreString("tags", ColumnType{}, column) if err != nil { t.Fatal(err) } for _, test := range []struct { - field MakeValue + column MakeValue tags element.Tags expected interface{} }{ @@ -331,7 +332,7 @@ func TestHstoreString(t *testing.T) { {hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`}, {hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`}, } { - actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{}) + actual := test.column("", &element.OSMElem{Tags: test.tags}, nil, Match{}) if actual.(string) != test.expected { t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags) } diff --git a/mapping/config.go b/mapping/config.go deleted file mode 100644 index 0366328..0000000 --- a/mapping/config.go +++ /dev/null @@ -1,347 +0,0 @@ -package mapping - -import ( - "errors" - "fmt" - "io/ioutil" - - "github.com/omniscale/imposm3/element" - - "gopkg.in/yaml.v2" -) - -type Field struct { - Name string `yaml:"name"` - Key Key `yaml:"key"` - Keys []Key `yaml:"keys"` - Type string `yaml:"type"` - Args map[string]interface{} `yaml:"args"` - FromMember bool `yaml:"from_member"` -} - -type Table struct { - Name string - Type TableType `yaml:"type"` - Mapping KeyValues `yaml:"mapping"` - Mappings map[string]SubMapping `yaml:"mappings"` - TypeMappings TypeMappings `yaml:"type_mappings"` - Fields []*Field `yaml:"columns"` // TODO rename Fields internaly to Columns - OldFields []*Field `yaml:"fields"` - Filters *Filters `yaml:"filters"` -} - -type GeneralizedTable struct { - Name string - SourceTableName string `yaml:"source"` - Tolerance float64 `yaml:"tolerance"` - SqlFilter string `yaml:"sql_filter"` -} - -type Filters struct { - ExcludeTags *[][]string `yaml:"exclude_tags"` -} - -type Tables map[string]*Table - -type GeneralizedTables map[string]*GeneralizedTable - -type Mapping struct { - Tables Tables `yaml:"tables"` - GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` - Tags Tags `yaml:"tags"` - Areas Areas `yaml:"areas"` - // SingleIdSpace mangles the overlapping node/way/relation IDs - // to be unique (nodes positive, ways negative, relations negative -1e17) - SingleIdSpace bool `yaml:"use_single_id_space"` -} - -type Areas struct { - AreaTags []Key `yaml:"area_tags"` - LinearTags []Key `yaml:"linear_tags"` -} - -type Tags struct { - LoadAll bool `yaml:"load_all"` - Exclude []Key `yaml:"exclude"` - Include []Key `yaml:"include"` -} - -type orderedValue struct { - value Value - order int -} -type KeyValues map[Key][]orderedValue - -func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { - if *kv == nil { - *kv = make(map[Key][]orderedValue) - } - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - order := 0 - for _, item := range slice { - k, ok := item.Key.(string) - if !ok { - return fmt.Errorf("mapping key '%s' not a string", k) - } - values, ok := item.Value.([]interface{}) - if !ok { - return fmt.Errorf("mapping key '%s' not a string", k) - } - for _, v := range values { - if v, ok := v.(string); ok { - (*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order}) - } else { - return fmt.Errorf("mapping value '%s' not a string", v) - } - order += 1 - } - } - return nil -} - -type SubMapping struct { - Mapping KeyValues -} - -type TypeMappings struct { - Points KeyValues `yaml:"points"` - LineStrings KeyValues `yaml:"linestrings"` - Polygons KeyValues `yaml:"polygons"` -} - -type ElementFilter func(tags element.Tags, key Key, closed bool) bool - -type TagTables map[Key]map[Value][]OrderedDestTable - -type DestTable struct { - Name string - SubMapping string -} - -type OrderedDestTable struct { - DestTable - order int -} - -type TableType string - -func (tt *TableType) UnmarshalJSON(data []byte) error { - switch string(data) { - case "": - return errors.New("missing table type") - case `"point"`: - *tt = PointTable - case `"linestring"`: - *tt = LineStringTable - case `"polygon"`: - *tt = PolygonTable - case `"geometry"`: - *tt = GeometryTable - case `"relation"`: - *tt = RelationTable - case `"relation_member"`: - *tt = RelationMemberTable - default: - return errors.New("unknown type " + string(data)) - } - return nil -} - -const ( - PolygonTable TableType = "polygon" - LineStringTable TableType = "linestring" - PointTable TableType = "point" - GeometryTable TableType = "geometry" - RelationTable TableType = "relation" - RelationMemberTable TableType = "relation_member" -) - -func NewMapping(filename string) (*Mapping, error) { - f, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - - mapping := Mapping{} - err = yaml.Unmarshal(f, &mapping) - if err != nil { - return nil, err - } - - err = mapping.prepare() - if err != nil { - return nil, err - } - return &mapping, nil -} - -func (t *Table) ExtraTags() map[Key]bool { - tags := make(map[Key]bool) - for _, field := range t.Fields { - if field.Key != "" { - tags[field.Key] = true - } - for _, k := range field.Keys { - tags[k] = true - } - } - return tags -} - -func (m *Mapping) prepare() error { - for name, t := range m.Tables { - t.Name = name - if t.OldFields != nil { - // todo deprecate 'fields' - t.Fields = t.OldFields - } - } - - for name, t := range m.GeneralizedTables { - t.Name = name - } - return nil -} - -func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) { - for key, vals := range mapping { - for _, v := range vals { - vals, ok := tt[key] - tbl := OrderedDestTable{DestTable: table, order: v.order} - if ok { - vals[v.value] = append(vals[v.value], tbl) - } else { - tt[key] = make(map[Value][]OrderedDestTable) - tt[key][v.value] = append(tt[key][v.value], tbl) - } - } - } -} - -func (m *Mapping) mappings(tableType TableType, mappings TagTables) { - for name, t := range m.Tables { - if t.Type != GeometryTable && t.Type != tableType { - continue - } - mappings.addFromMapping(t.Mapping, DestTable{Name: name}) - - for subMappingName, subMapping := range t.Mappings { - mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName}) - } - - switch tableType { - case PointTable: - mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name}) - case LineStringTable: - mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name}) - case PolygonTable: - mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name}) - } - } -} - -func (m *Mapping) tables(tableType TableType) map[string]*TableFields { - result := make(map[string]*TableFields) - for name, t := range m.Tables { - if t.Type == tableType || t.Type == "geometry" { - result[name] = t.TableFields() - } - } - return result -} - -func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { - for _, t := range m.Tables { - if t.Type != tableType && t.Type != "geometry" { - continue - } - for key, _ := range t.ExtraTags() { - tags[key] = true - } - if t.Filters != nil && t.Filters.ExcludeTags != nil { - for _, keyVal := range *t.Filters.ExcludeTags { - tags[Key(keyVal[0])] = true - } - } - } - for _, k := range m.Tags.Include { - tags[k] = true - } - - // always include area tag for closed-way handling - tags["area"] = true -} - -func (m *Mapping) ElementFilters() map[string][]ElementFilter { - result := make(map[string][]ElementFilter) - - var areaTags map[Key]struct{} - var linearTags map[Key]struct{} - if m.Areas.AreaTags != nil { - areaTags = make(map[Key]struct{}) - for _, tag := range m.Areas.AreaTags { - areaTags[tag] = struct{}{} - } - } - if m.Areas.LinearTags != nil { - linearTags = make(map[Key]struct{}) - for _, tag := range m.Areas.LinearTags { - linearTags[tag] = struct{}{} - } - } - - for name, t := range m.Tables { - if t.Type == LineStringTable && areaTags != nil { - f := func(tags element.Tags, key Key, closed bool) bool { - if closed { - if tags["area"] == "yes" { - return false - } - if tags["area"] != "no" { - if _, ok := areaTags[key]; ok { - return false - } - } - } - return true - } - result[name] = append(result[name], f) - } - if t.Type == PolygonTable && linearTags != nil { - f := func(tags element.Tags, key Key, closed bool) bool { - if closed && tags["area"] == "no" { - return false - } - if tags["area"] != "yes" { - if _, ok := linearTags[key]; ok { - return false - } - } - return true - } - result[name] = append(result[name], f) - } - - if t.Filters == nil { - continue - } - if t.Filters.ExcludeTags != nil { - for _, filterKeyVal := range *t.Filters.ExcludeTags { - f := func(tags element.Tags, key Key, closed bool) bool { - if v, ok := tags[filterKeyVal[0]]; ok { - if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] { - return false - } - } - return true - } - result[name] = append(result[name], f) - } - } - } - return result -} diff --git a/mapping/config/config.go b/mapping/config/config.go new file mode 100644 index 0000000..53a1c7c --- /dev/null +++ b/mapping/config/config.go @@ -0,0 +1,113 @@ +package config + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +type Mapping struct { + Tables Tables `yaml:"tables"` + GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` + Tags Tags `yaml:"tags"` + Areas Areas `yaml:"areas"` + // SingleIdSpace mangles the overlapping node/way/relation IDs + // to be unique (nodes positive, ways negative, relations negative -1e17) + SingleIdSpace bool `yaml:"use_single_id_space"` +} + +type Column struct { + Name string `yaml:"name"` + Key Key `yaml:"key"` + Keys []Key `yaml:"keys"` + Type string `yaml:"type"` + Args map[string]interface{} `yaml:"args"` + FromMember bool `yaml:"from_member"` +} + +type Tables map[string]*Table +type Table struct { + Name string + Type string `yaml:"type"` + Mapping KeyValues `yaml:"mapping"` + Mappings map[string]SubMapping `yaml:"mappings"` + TypeMappings TypeMappings `yaml:"type_mappings"` + Columns []*Column `yaml:"columns"` + OldFields []*Column `yaml:"fields"` + Filters *Filters `yaml:"filters"` + RelationTypes []string `yaml:"relation_types"` +} + +type GeneralizedTables map[string]*GeneralizedTable +type GeneralizedTable struct { + Name string + SourceTableName string `yaml:"source"` + Tolerance float64 `yaml:"tolerance"` + SqlFilter string `yaml:"sql_filter"` +} + +type Filters struct { + ExcludeTags *[][]string `yaml:"exclude_tags"` +} + +type Areas struct { + AreaTags []Key `yaml:"area_tags"` + LinearTags []Key `yaml:"linear_tags"` +} + +type Tags struct { + LoadAll bool `yaml:"load_all"` + Exclude []Key `yaml:"exclude"` + Include []Key `yaml:"include"` +} + +type Key string +type Value string + +type orderedValue struct { + Value Value + Order int +} + +type KeyValues map[Key][]orderedValue + +func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { + if *kv == nil { + *kv = make(map[Key][]orderedValue) + } + slice := yaml.MapSlice{} + err := unmarshal(&slice) + if err != nil { + return err + } + order := 0 + for _, item := range slice { + k, ok := item.Key.(string) + if !ok { + return fmt.Errorf("mapping key '%s' not a string", k) + } + values, ok := item.Value.([]interface{}) + if !ok { + return fmt.Errorf("mapping key '%s' not a string", k) + } + for _, v := range values { + if v, ok := v.(string); ok { + (*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{Value: Value(v), Order: order}) + } else { + return fmt.Errorf("mapping value '%s' not a string", v) + } + order += 1 + } + } + return nil +} + +type SubMapping struct { + Mapping KeyValues +} + +type TypeMappings struct { + Points KeyValues `yaml:"points"` + LineStrings KeyValues `yaml:"linestrings"` + Polygons KeyValues `yaml:"polygons"` +} diff --git a/mapping/doc.go b/mapping/doc.go index 0657cdc..42c2ae8 100644 --- a/mapping/doc.go +++ b/mapping/doc.go @@ -1,4 +1,14 @@ /* -Package mapping provides functions for defining and executing the database schema. +Package mapping provides implements mapping and convertion between OSM elements and database tables, rows and columns. + +The core logic of Imposm is accesible with the Mapping struct. +A Mapping creates filters and matchers based on mapping configuration (.yaml or .json file). + +Filters are for initial filtering (during -read). They remove all tags that are not needed. + +Matchers map OSM elements to zero or more destination tables. Each Match results can convert an OSM element +to a row with all mapped column values. +The matching is dependend on the element type (node, way, relation), the element tags and the destination +table type (point, linestring, polygon, relation, relation_member). */ package mapping diff --git a/mapping/filter.go b/mapping/filter.go index d5507d6..4ea2e58 100644 --- a/mapping/filter.go +++ b/mapping/filter.go @@ -5,67 +5,96 @@ import ( "strings" "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" ) +type TagFilterer interface { + Filter(tags *element.Tags) +} + func (m *Mapping) NodeTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) - m.mappings("point", mappings) + mappings := make(TagTableMapping) + m.mappings(PointTable, mappings) tags := make(map[Key]bool) - m.extraTags("point", tags) - return &TagFilter{mappings, tags} + m.extraTags(PointTable, tags) + m.extraTags(RelationMemberTable, tags) + return &tagFilter{mappings.asTagMap(), tags} } func (m *Mapping) WayTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) - m.mappings("linestring", mappings) - m.mappings("polygon", mappings) + mappings := make(TagTableMapping) + m.mappings(LineStringTable, mappings) + m.mappings(PolygonTable, mappings) tags := make(map[Key]bool) - m.extraTags("linestring", tags) - m.extraTags("polygon", tags) - return &TagFilter{mappings, tags} + m.extraTags(LineStringTable, tags) + m.extraTags(PolygonTable, tags) + m.extraTags(RelationMemberTable, tags) + return &tagFilter{mappings.asTagMap(), tags} } func (m *Mapping) RelationTagFilter() TagFilterer { - if m.Tags.LoadAll { - return newExcludeFilter(m.Tags.Exclude) + if m.Conf.Tags.LoadAll { + return newExcludeFilter(m.Conf.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) - m.mappings("linestring", mappings) - m.mappings("polygon", mappings) + mappings := make(TagTableMapping) + // do not filter out type tag for common relations + mappings["type"] = map[Value][]orderedDestTable{ + "multipolygon": []orderedDestTable{}, + "boundary": []orderedDestTable{}, + "land_area": []orderedDestTable{}, + } + m.mappings(LineStringTable, mappings) + m.mappings(PolygonTable, mappings) + m.mappings(RelationTable, mappings) + m.mappings(RelationMemberTable, mappings) tags := make(map[Key]bool) - m.extraTags("linestring", tags) - m.extraTags("polygon", tags) - // do not filter out type tag - mappings["type"] = map[Value][]OrderedDestTable{ - "multipolygon": []OrderedDestTable{}, - "boundary": []OrderedDestTable{}, - "land_area": []OrderedDestTable{}, - } - return &RelationTagFilter{TagFilter{mappings, tags}} + m.extraTags(LineStringTable, tags) + m.extraTags(PolygonTable, tags) + m.extraTags(RelationTable, tags) + m.extraTags(RelationMemberTable, tags) + return &tagFilter{mappings.asTagMap(), tags} } -type TagFilter struct { - mappings map[Key]map[Value][]OrderedDestTable +type tagMap map[Key]map[Value]struct{} + +type tagFilter struct { + mappings tagMap extraTags map[Key]bool } -type RelationTagFilter struct { - TagFilter +func (f *tagFilter) Filter(tags *element.Tags) { + if tags == nil { + return + } + for k, v := range *tags { + values, ok := f.mappings[Key(k)] + if ok { + if _, ok := values["__any__"]; ok { + continue + } else if _, ok := values[Value(v)]; ok { + continue + } else if _, ok := f.extraTags[Key(k)]; !ok { + delete(*tags, k) + } + } else if _, ok := f.extraTags[Key(k)]; !ok { + delete(*tags, k) + } + } } -type ExcludeFilter struct { +type excludeFilter struct { keys map[Key]struct{} matches []string } -func newExcludeFilter(tags []Key) *ExcludeFilter { - f := ExcludeFilter{ +func newExcludeFilter(tags []config.Key) *excludeFilter { + f := excludeFilter{ keys: make(map[Key]struct{}), matches: make([]string, 0), } @@ -73,14 +102,14 @@ func newExcludeFilter(tags []Key) *ExcludeFilter { if strings.ContainsAny(string(t), "?*[") { f.matches = append(f.matches, string(t)) } else { - f.keys[t] = struct{}{} + f.keys[Key(t)] = struct{}{} } } return &f } -func (f *ExcludeFilter) Filter(tags *element.Tags) bool { - for k, _ := range *tags { +func (f *excludeFilter) Filter(tags *element.Tags) { + for k := range *tags { if _, ok := f.keys[Key(k)]; ok { delete(*tags, k) } else if f.matches != nil { @@ -92,86 +121,4 @@ func (f *ExcludeFilter) Filter(tags *element.Tags) bool { } } } - return true -} - -type TagFilterer interface { - Filter(tags *element.Tags) bool -} - -func (f *TagFilter) Filter(tags *element.Tags) bool { - if tags == nil { - return false - } - foundMapping := false - for k, v := range *tags { - values, ok := f.mappings[Key(k)] - if ok { - if _, ok := values["__any__"]; ok { - foundMapping = true - continue - } else if _, ok := values[Value(v)]; ok { - foundMapping = true - continue - } else if _, ok := f.extraTags[Key(k)]; !ok { - delete(*tags, k) - } - } else if _, ok := f.extraTags[Key(k)]; !ok { - delete(*tags, k) - } - } - if foundMapping { - return true - } else { - *tags = nil - return false - } -} - -func (f *RelationTagFilter) Filter(tags *element.Tags) bool { - if tags == nil { - return false - } - - // TODO improve filtering for relation/relation_member mappings - // right now this only works with tags.load_all:true - if t, ok := (*tags)["type"]; ok { - if t != "multipolygon" && t != "boundary" && t != "land_area" { - *tags = nil - return false - } - if t == "boundary" { - if _, ok := (*tags)["boundary"]; !ok { - // a lot of the boundary relations are not multipolygon - // only import with boundary tags (e.g. boundary=administrative) - *tags = nil - return false - } - } - } else { - *tags = nil - return false - } - tagCount := len(*tags) - f.TagFilter.Filter(tags) - - // we removed tags... - if len(*tags) < tagCount { - expectedTags := 0 - if _, ok := (*tags)["name"]; ok { - expectedTags += 1 - } - if _, ok := (*tags)["type"]; ok { - expectedTags += 1 - } - if len(*tags) == expectedTags { - // but no tags except name and type are left - // remove all, otherwise tags from longest - // way/ring would be used during MP building - *tags = nil - return false - } - } - // always return true here since we found a matching type - return true } diff --git a/mapping/filter_test.go b/mapping/filter_test.go index 209b053..83becf6 100644 --- a/mapping/filter_test.go +++ b/mapping/filter_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" ) var mapping *Mapping @@ -17,28 +18,29 @@ func init() { } } -func stringMapEquals(t *testing.T, expected, actual map[string]string) { +func stringMapEqual(expected, actual map[string]string) bool { if len(expected) != len(actual) { - t.Fatalf("different length in %v and %v\n", expected, actual) + return false } for k, v := range expected { if actualV, ok := actual[k]; ok { if actualV != v { - t.Fatalf("%s != %s in %v and %v\n", v, actualV, expected, actual) + return false } } else { - t.Fatalf("%s not in %v\n", k, actual) + return false } } + return true } -func matchesEqual(t *testing.T, expected []Match, actual []Match) { +func matchesEqual(expected []Match, actual []Match) bool { expectedMatches := make(map[DestTable]Match) actualMatches := make(map[DestTable]Match) if len(expected) != len(actual) { - t.Fatalf("different length in %v and %v\n", expected, actual) + return false } for _, match := range expected { @@ -53,405 +55,293 @@ func matchesEqual(t *testing.T, expected []Match, actual []Match) { if expectedMatch.Table != actualMatch.Table || expectedMatch.Key != actualMatch.Key || expectedMatch.Value != actualMatch.Value { - t.Fatalf("match differ %v != %v", expectedMatch, actualMatch) + return false } } else { - t.Fatalf("%s not in %v", name, actualMatches) + return false + } + } + return true +} + +func TestTagFilterNodes(t *testing.T) { + tests := []struct { + tags element.Tags + expected element.Tags + }{ + {tags: element.Tags{}, expected: element.Tags{}}, + {tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "place": "unknown"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "place": "unknown", "population": "1000"}, expected: element.Tags{"name": "foo", "population": "1000"}}, + {tags: element.Tags{"name": "foo", "place": "village"}, expected: element.Tags{"name": "foo", "place": "village"}}, + {tags: element.Tags{"name": "foo", "place": "village", "population": "1000"}, expected: element.Tags{"name": "foo", "place": "village", "population": "1000"}}, + {tags: element.Tags{"name": "foo", "place": "village", "unknown": "foo"}, expected: element.Tags{"name": "foo", "place": "village"}}, + {tags: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, expected: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}}, + } + + nodes := mapping.NodeTagFilter() + for i, test := range tests { + nodes.Filter(&test.tags) + if !stringMapEqual(test.tags, test.expected) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected) } } } -func TestTagFilterNodes(t *testing.T) { - var tags element.Tags - nodes := mapping.NodeTagFilter() - - tags = element.Tags{"name": "foo"} - if nodes.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "unknown": "baz"} - if nodes.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "place": "unknown"} - if nodes.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "place": "village"} - if nodes.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags) - - tags = element.Tags{"name": "foo", "place": "village", "population": "1000"} - if nodes.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "population": "1000"}, tags) - - tags = element.Tags{"name": "foo", "place": "village", "highway": "unknown"} - if nodes.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags) - - tags = element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"} - if nodes.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, tags) -} - func TestTagFilterWays(t *testing.T) { - var tags element.Tags + tests := []struct { + tags element.Tags + expected element.Tags + }{ + {tags: element.Tags{}, expected: element.Tags{}}, + {tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "highway": "unknown"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}}, + {tags: element.Tags{"name": "foo", "building": "whatever"}, expected: element.Tags{"name": "foo", "building": "whatever"}}, + {tags: element.Tags{"name": "foo", "highway": "track", "unknown": "foo"}, expected: element.Tags{"name": "foo", "highway": "track"}}, + {tags: element.Tags{"name": "foo", "place": "village", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}}, + {tags: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, expected: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}}, + } + ways := mapping.WayTagFilter() - - tags = element.Tags{"name": "foo"} - if ways.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) + for i, test := range tests { + ways.Filter(&test.tags) + if !stringMapEqual(test.tags, test.expected) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected) + } } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "unknown": "baz"} - if ways.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "highway": "unknown"} - if ways.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "highway": "track"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags) - - tags = element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, tags) - - tags = element.Tags{"name": "foo", "place": "village", "highway": "track"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags) - - tags = element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"}, tags) - - // with __any__ value - tags = element.Tags{"name": "foo", "building": "yes"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "building": "yes"}, tags) - - tags = element.Tags{"name": "foo", "building": "whatever"} - if ways.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "building": "whatever"}, tags) } func TestTagFilterRelations(t *testing.T) { - var tags element.Tags + tests := []struct { + tags element.Tags + expected element.Tags + }{ + {tags: element.Tags{}, expected: element.Tags{}}, + {tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "landuse": "unknown"}, expected: element.Tags{"name": "foo"}}, + {tags: element.Tags{"name": "foo", "landuse": "farm"}, expected: element.Tags{"name": "foo", "landuse": "farm"}}, + {tags: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}}, + {tags: element.Tags{"name": "foo", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "type": "multipolygon"}}, + {tags: element.Tags{"name": "foo", "type": "boundary"}, expected: element.Tags{"name": "foo", "type": "boundary"}}, + {tags: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}}, + } + relations := mapping.RelationTagFilter() - - tags = element.Tags{"name": "foo"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) + for i, test := range tests { + relations.Filter(&test.tags) + if !stringMapEqual(test.tags, test.expected) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected) + } } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "unknown": "baz"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "landuse": "unknown"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "landuse": "farm"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"} - if relations.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, tags) - - // skip multipolygon with filtered tags, otherwise tags from - // longest way would be used - tags = element.Tags{"name": "foo", "landuse": "unknown", "type": "multipolygon"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "landuse": "park", "type": "multipolygon"} - if relations.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "type": "multipolygon", "landuse": "park"}, tags) - - tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"} - if relations.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"}, tags) - - // boundary relation for boundary - tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"} - if relations.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"}, tags) - - // boundary relation for non boundary - tags = element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - /* skip boundary with filtered tags, otherwise tags from longest way would - be used */ - tags = element.Tags{"name": "foo", "boundary": "unknown", "type": "boundary"} - if relations.Filter(&tags) != false { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{}, tags) - - tags = element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"} - if relations.Filter(&tags) != true { - t.Fatal("unexpected filter response for", tags) - } - stringMapEquals(t, element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"}, tags) - } func TestPointMatcher(t *testing.T) { - elem := element.Node{} - points := mapping.PointMatcher() - - elem.Tags = element.Tags{"unknown": "baz"} - matchesEqual(t, []Match{}, points.MatchNode(&elem)) - - elem.Tags = element.Tags{"place": "unknown"} - matchesEqual(t, []Match{}, points.MatchNode(&elem)) - - elem.Tags = element.Tags{"place": "city"} - matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem)) - - elem.Tags = element.Tags{"place": "city", "highway": "unknown"} - matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem)) - - elem.Tags = element.Tags{"place": "city", "highway": "bus_stop"} - matchesEqual(t, - []Match{ + tests := []struct { + tags element.Tags + matches []Match + }{ + {element.Tags{"unknown": "baz"}, []Match{}}, + {element.Tags{"place": "unknown"}, []Match{}}, + {element.Tags{"place": "city"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}}, + {element.Tags{"place": "city", "highway": "unknown"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}}, + {element.Tags{"place": "city", "highway": "bus_stop"}, []Match{ {"place", "city", DestTable{Name: "places"}, nil}, {"highway", "bus_stop", DestTable{Name: "transport_points"}, nil}}, - points.MatchNode(&elem)) + }, + } + + elem := element.Node{} + m := mapping.PointMatcher() + for i, test := range tests { + elem.Tags = test.tags + actual := m.MatchNode(&elem) + if !matchesEqual(actual, test.matches) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches) + } + } } func TestLineStringMatcher(t *testing.T) { + tests := []struct { + tags element.Tags + matches []Match + }{ + {element.Tags{"unknown": "baz"}, []Match{}}, + {element.Tags{"highway": "unknown"}, []Match{}}, + {element.Tags{"highway": "pedestrian"}, + []Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}}, + + // exclude_tags area=yes + {element.Tags{"highway": "pedestrian", "area": "yes"}, []Match{}}, + + {element.Tags{"barrier": "hedge"}, + []Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}}, + {element.Tags{"barrier": "hedge", "area": "yes"}, []Match{}}, + + {element.Tags{"aeroway": "runway"}, []Match{}}, + {element.Tags{"aeroway": "runway", "area": "no"}, + []Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}}, + + {element.Tags{"highway": "secondary", "railway": "tram"}, + []Match{ + {"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil}, + {"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}}, + }, + {element.Tags{"highway": "footway", "landuse": "park", "barrier": "hedge"}, + // landusages not a linestring table + []Match{ + {"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil}, + {"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}, + }, + } + elem := element.Way{} // fake closed way for area matching elem.Refs = []int64{1, 2, 3, 4, 1} if !elem.IsClosed() { t.Fatal("way not closed") } - ls := mapping.LineStringMatcher() - - elem.Tags = element.Tags{"unknown": "baz"} - matchesEqual(t, []Match{}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"highway": "unknown"} - matchesEqual(t, []Match{}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"highway": "pedestrian"} - matchesEqual(t, []Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem)) - - // exclude_tags area=yes - elem.Tags = element.Tags{"highway": "pedestrian", "area": "yes"} - matchesEqual(t, []Match{}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"barrier": "hedge"} - matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"} - matchesEqual(t, []Match{}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"aeroway": "runway", "area": "no"} - matchesEqual(t, []Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"aeroway": "runway"} - matchesEqual(t, []Match{}, ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"highway": "secondary", "railway": "tram"} - matchesEqual(t, - []Match{ - {"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil}, - {"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}}, - ls.MatchWay(&elem)) - - elem.Tags = element.Tags{"highway": "footway", "landuse": "park"} - // landusages not a linestring table - matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem)) + m := mapping.LineStringMatcher() + for i, test := range tests { + elem.Tags = test.tags + actual := m.MatchWay(&elem) + if !matchesEqual(actual, test.matches) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches) + } + } } -func TestPolygonMatcher(t *testing.T) { - elem := element.Relation{} - polys := mapping.PolygonMatcher() +func TestPolygonMatcher_MatchWay(t *testing.T) { + tests := []struct { + tags element.Tags + matches []Match + }{ + {element.Tags{}, []Match{}}, + {element.Tags{"unknown": "baz"}, []Match{}}, + {element.Tags{"landuse": "unknown"}, []Match{}}, + {element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}}, + {element.Tags{"building": "yes"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}}, + {element.Tags{"building": "residential"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}}, + // line type requires area=yes + {element.Tags{"barrier": "hedge"}, []Match{}}, + {element.Tags{"barrier": "hedge", "area": "yes"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}}, - elem.Tags = element.Tags{"unknown": "baz"} - matchesEqual(t, []Match{}, polys.MatchRelation(&elem)) + {element.Tags{"building": "shop"}, []Match{ + {"building", "shop", DestTable{Name: "buildings"}, nil}, + {"building", "shop", DestTable{Name: "amenity_areas"}, nil}, + }}, - elem.Tags = element.Tags{"landuse": "unknowns"} - matchesEqual(t, []Match{}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"building": "yes"} - matchesEqual(t, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem)) - elem.Tags = element.Tags{"building": "residential"} - matchesEqual(t, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"barrier": "hedge"} - matchesEqual(t, []Match{}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"} - matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"building": "shop"} - matchesEqual(t, []Match{ - {"building", "shop", DestTable{Name: "buildings"}, nil}, - {"building", "shop", DestTable{Name: "amenity_areas"}, nil}}, - polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"landuse": "farm"} - matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"landuse": "farm", "highway": "secondary"} - matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"landuse": "farm", "aeroway": "apron"} - matchesEqual(t, - []Match{ + {element.Tags{"aeroway": "apron", "landuse": "farm"}, []Match{ {"aeroway", "apron", DestTable{Name: "transport_areas"}, nil}, - {"landuse", "farm", DestTable{Name: "landusages"}, nil}}, - polys.MatchRelation(&elem)) + {"landuse", "farm", DestTable{Name: "landusages"}, nil}, + }}, - elem.Tags = element.Tags{"highway": "footway"} // linear by default - matchesEqual(t, []Match{}, polys.MatchRelation(&elem)) + {element.Tags{"landuse": "farm", "highway": "secondary"}, []Match{ + {"landuse", "farm", DestTable{Name: "landusages"}, nil}, + }}, - elem.Tags = element.Tags{"highway": "footway", "area": "yes"} - matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) + {element.Tags{"highway": "footway"}, []Match{}}, + {element.Tags{"highway": "footway", "area": "yes"}, []Match{ + {"highway", "footway", DestTable{Name: "landusages"}, nil}, + }}, - elem.Tags = element.Tags{"boundary": "administrative", "admin_level": "8"} - matchesEqual(t, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}, polys.MatchRelation(&elem)) + {element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}}, + + /* + landusages mapping has the following order, + check that XxxMatcher always uses the first + + amenity: + - university + landuse: + - forest + leisure: + - park + landuse: + - park + */ + + {element.Tags{"landuse": "forest", "leisure": "park"}, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}}, + {element.Tags{"landuse": "park", "leisure": "park"}, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}}, + {element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"}, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}}, + } + + elem := element.Way{} + // fake closed way for area matching + elem.Refs = []int64{1, 2, 3, 4, 1} + if !elem.IsClosed() { + t.Fatal("way not closed") + } + m := mapping.PolygonMatcher() + for i, test := range tests { + elem.Tags = test.tags + actual := m.MatchWay(&elem) + if !matchesEqual(actual, test.matches) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches) + } + } + + elem.Refs = nil + elem.Tags = element.Tags{"building": "yes"} + actual := m.MatchWay(&elem) + if !matchesEqual([]Match{}, actual) { + t.Error("open way matched as polygon") + } } -func TestMatcherMappingOrder(t *testing.T) { +func TestPolygonMatcher_MatchRelation(t *testing.T) { + // check that only relations with type=multipolygon/boundary are matched as polygon + + tests := []struct { + tags element.Tags + matches []Match + }{ + {element.Tags{}, []Match{}}, + {element.Tags{"unknown": "baz"}, []Match{}}, + {element.Tags{"landuse": "unknown"}, []Match{}}, + {element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}}, + {element.Tags{"building": "yes"}, []Match{}}, + {element.Tags{"building": "yes", "type": "multipolygon"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}}, + {element.Tags{"building": "residential", "type": "multipolygon"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}}, + // line type requires area=yes + {element.Tags{"barrier": "hedge", "type": "multipolygon"}, []Match{}}, + {element.Tags{"barrier": "hedge", "area": "yes", "type": "multipolygon"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}}, + + {element.Tags{"building": "shop", "type": "multipolygon"}, []Match{ + {"building", "shop", DestTable{Name: "buildings"}, nil}, + {"building", "shop", DestTable{Name: "amenity_areas"}, nil}, + }}, + + {element.Tags{"aeroway": "apron", "landuse": "farm", "type": "multipolygon"}, []Match{ + {"aeroway", "apron", DestTable{Name: "transport_areas"}, nil}, + {"landuse", "farm", DestTable{Name: "landusages"}, nil}, + }}, + + {element.Tags{"landuse": "farm", "highway": "secondary", "type": "multipolygon"}, []Match{ + {"landuse", "farm", DestTable{Name: "landusages"}, nil}, + }}, + + {element.Tags{"highway": "footway", "type": "multipolygon"}, []Match{}}, + {element.Tags{"highway": "footway", "area": "yes", "type": "multipolygon"}, []Match{ + {"highway", "footway", DestTable{Name: "landusages"}, nil}, + }}, + + {element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{}}, + {element.Tags{"boundary": "administrative", "admin_level": "8", "type": "boundary"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}}, + } + elem := element.Relation{} - polys := mapping.PolygonMatcher() - - /* - landusages mapping has the following order, - check that XxxMatcher always uses the first - - amenity: - - university - landuse: - - forest - leisure: - - park - landuse: - - park - */ - - elem.Tags = element.Tags{"landuse": "forest", "leisure": "park"} - matchesEqual(t, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"landuse": "park", "leisure": "park"} - matchesEqual(t, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) - - elem.Tags = element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"} - matchesEqual(t, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem)) -} - -func TestFilterNodes(t *testing.T) { - var tags element.Tags - - // test name only - tags = make(element.Tags) - tags["name"] = "foo" - - points := mapping.NodeTagFilter() - if points.Filter(&tags) != false { - t.Fatal("Filter result not false") - } - if len(tags) != 0 { - t.Fatal("Filter result not empty") - } - - // test name + unmapped tags - tags = make(element.Tags) - tags["name"] = "foo" - tags["boring"] = "true" - - if points.Filter(&tags) != false { - t.Fatal("Filter result not false") - } - if len(tags) != 0 { - t.Fatal("Filter result not empty") - } - - // test fields only, but no mapping - tags = make(element.Tags) - tags["population"] = "0" - tags["name"] = "foo" - tags["boring"] = "true" - - if points.Filter(&tags) != false { - t.Fatal("Filter result true", tags) - } - if len(tags) != 0 { - t.Fatal("Filter result not empty", tags) - } - - // ... not with mapped tag (place) - tags = make(element.Tags) - tags["population"] = "0" - tags["name"] = "foo" - tags["boring"] = "true" - tags["place"] = "village" - - if points.Filter(&tags) != true { - t.Fatal("Filter result true", tags) - } - if len(tags) != 3 && tags["population"] == "0" && tags["name"] == "foo" && tags["place"] == "village" { - t.Fatal("Filter result not expected", tags) + m := mapping.PolygonMatcher() + for i, test := range tests { + elem.Tags = test.tags + actual := m.MatchRelation(&elem) + if !matchesEqual(actual, test.matches) { + t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches) + } } } @@ -460,7 +350,7 @@ func TestExcludeFilter(t *testing.T) { var tags element.Tags // no matches - f = newExcludeFilter([]Key{}) + f = newExcludeFilter([]config.Key{}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}) { @@ -468,7 +358,7 @@ func TestExcludeFilter(t *testing.T) { } // match all - f = newExcludeFilter([]Key{"*"}) + f = newExcludeFilter([]config.Key{"*"}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{}) { @@ -476,7 +366,7 @@ func TestExcludeFilter(t *testing.T) { } // fixed string and wildcard match - f = newExcludeFilter([]Key{"source", "tiger:*"}) + f = newExcludeFilter([]config.Key{"source", "tiger:*"}) tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"} f.Filter(&tags) if !reflect.DeepEqual(tags, element.Tags{"source:foo": "1"}) { @@ -495,9 +385,7 @@ func BenchmarkFilterNodes(b *testing.B) { tags["boring"] = "true" points := mapping.NodeTagFilter() - if points.Filter(&tags) != true { - b.Fatal("Filter result true", tags) - } + points.Filter(&tags) if len(tags) != 2 && tags["population"] == "0" && tags["name"] == "foo" { b.Fatal("Filter result not expected", tags) } diff --git a/mapping/mapping.go b/mapping/mapping.go new file mode 100644 index 0000000..0e8d6e9 --- /dev/null +++ b/mapping/mapping.go @@ -0,0 +1,331 @@ +package mapping + +import ( + "errors" + "io/ioutil" + + "github.com/omniscale/imposm3/element" + "github.com/omniscale/imposm3/mapping/config" + + "gopkg.in/yaml.v2" +) + +type orderedDestTable struct { + DestTable + order int +} + +type TagTableMapping map[Key]map[Value][]orderedDestTable + +func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) { + for key, vals := range mapping { + for _, v := range vals { + vals, ok := tt[Key(key)] + tbl := orderedDestTable{DestTable: table, order: v.Order} + if ok { + vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl) + } else { + tt[Key(key)] = make(map[Value][]orderedDestTable) + tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl) + } + } + } +} + +func (tt TagTableMapping) asTagMap() tagMap { + result := make(tagMap) + for k, vals := range tt { + result[k] = make(map[Value]struct{}) + for v := range vals { + result[k][v] = struct{}{} + } + } + return result +} + +type DestTable struct { + Name string + SubMapping string +} + +type TableType string + +func (tt *TableType) UnmarshalJSON(data []byte) error { + switch string(data) { + case "": + return errors.New("missing table type") + case `"point"`: + *tt = PointTable + case `"linestring"`: + *tt = LineStringTable + case `"polygon"`: + *tt = PolygonTable + case `"geometry"`: + *tt = GeometryTable + case `"relation"`: + *tt = RelationTable + case `"relation_member"`: + *tt = RelationMemberTable + default: + return errors.New("unknown type " + string(data)) + } + return nil +} + +const ( + PolygonTable TableType = "polygon" + LineStringTable TableType = "linestring" + PointTable TableType = "point" + GeometryTable TableType = "geometry" + RelationTable TableType = "relation" + RelationMemberTable TableType = "relation_member" +) + +type Mapping struct { + Conf config.Mapping +} + +func NewMapping(filename string) (*Mapping, error) { + f, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + mapping := Mapping{} + err = yaml.Unmarshal(f, &mapping.Conf) + if err != nil { + return nil, err + } + + err = mapping.prepare() + if err != nil { + return nil, err + } + return &mapping, nil +} + +func (m *Mapping) prepare() error { + for name, t := range m.Conf.Tables { + t.Name = name + if t.OldFields != nil { + // todo deprecate 'fields' + t.Columns = t.OldFields + } + } + + for name, t := range m.Conf.GeneralizedTables { + t.Name = name + } + return nil +} + +func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) { + for name, t := range m.Conf.Tables { + if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType { + continue + } + mappings.addFromMapping(t.Mapping, DestTable{Name: name}) + + for subMappingName, subMapping := range t.Mappings { + mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName}) + } + + switch tableType { + case PointTable: + mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name}) + case LineStringTable: + mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name}) + case PolygonTable: + mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name}) + } + } +} + +func (m *Mapping) tables(tableType TableType) map[string]*rowBuilder { + result := make(map[string]*rowBuilder) + for name, t := range m.Conf.Tables { + if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable { + result[name] = makeRowBuilder(t) + } + } + return result +} + +func makeRowBuilder(tbl *config.Table) *rowBuilder { + result := rowBuilder{} + + for _, mappingColumn := range tbl.Columns { + column := valueBuilder{} + column.key = Key(mappingColumn.Key) + + columnType := MakeColumnType(mappingColumn) + if columnType != nil { + column.colType = *columnType + } else { + log.Warn("unhandled type: ", mappingColumn.Type) + } + result.columns = append(result.columns, column) + } + return &result +} + +func MakeColumnType(c *config.Column) *ColumnType { + if columnType, ok := AvailableColumnTypes[c.Type]; ok { + if columnType.MakeFunc != nil { + makeValue, err := columnType.MakeFunc(c.Name, columnType, *c) + if err != nil { + log.Print(err) + return nil + } + columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember} + } + columnType.FromMember = c.FromMember + return &columnType + } + return nil +} + +func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { + for _, t := range m.Conf.Tables { + if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable { + continue + } + + for _, col := range t.Columns { + if col.Key != "" { + tags[Key(col.Key)] = true + } + for _, k := range col.Keys { + tags[Key(k)] = true + } + } + + if t.Filters != nil && t.Filters.ExcludeTags != nil { + for _, keyVal := range *t.Filters.ExcludeTags { + tags[Key(keyVal[0])] = true + } + } + + if tableType == PolygonTable || tableType == RelationTable || tableType == RelationMemberTable { + if t.RelationTypes != nil { + tags["type"] = true + } + } + } + for _, k := range m.Conf.Tags.Include { + tags[Key(k)] = true + } + + // always include area tag for closed-way handling + tags["area"] = true +} + +type elementFilter func(tags element.Tags, key Key, closed bool) bool + +type tableElementFilters map[string][]elementFilter + +func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) { + var areaTags map[Key]struct{} + var linearTags map[Key]struct{} + if m.Conf.Areas.AreaTags != nil { + areaTags = make(map[Key]struct{}) + for _, tag := range m.Conf.Areas.AreaTags { + areaTags[Key(tag)] = struct{}{} + } + } + if m.Conf.Areas.LinearTags != nil { + linearTags = make(map[Key]struct{}) + for _, tag := range m.Conf.Areas.LinearTags { + linearTags[Key(tag)] = struct{}{} + } + } + + for name, t := range m.Conf.Tables { + if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType { + continue + } + if TableType(t.Type) == LineStringTable && areaTags != nil { + f := func(tags element.Tags, key Key, closed bool) bool { + if closed { + if tags["area"] == "yes" { + return false + } + if tags["area"] != "no" { + if _, ok := areaTags[key]; ok { + return false + } + } + } + return true + } + filters[name] = append(filters[name], f) + } + if TableType(t.Type) == PolygonTable && linearTags != nil { + f := func(tags element.Tags, key Key, closed bool) bool { + if closed && tags["area"] == "no" { + return false + } + if tags["area"] != "yes" { + if _, ok := linearTags[key]; ok { + return false + } + } + return true + } + filters[name] = append(filters[name], f) + } + } +} + +func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) { + for name, t := range m.Conf.Tables { + if t.RelationTypes != nil { + relTypes := t.RelationTypes // copy loop var for closure + f := func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags["type"]; ok { + for _, rtype := range relTypes { + if v == rtype { + return true + } + } + } + return false + } + filters[name] = append(filters[name], f) + } else { + if TableType(t.Type) == PolygonTable { + // standard mulipolygon handling (boundary and land_area are for backwards compatibility) + f := func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags["type"]; ok { + if v == "multipolygon" || v == "boundary" || v == "land_area" { + return true + } + } + return false + } + filters[name] = append(filters[name], f) + } + } + } +} + +func (m *Mapping) addFilters(filters tableElementFilters) { + for name, t := range m.Conf.Tables { + if t.Filters == nil { + continue + } + if t.Filters.ExcludeTags != nil { + for _, filterKeyVal := range *t.Filters.ExcludeTags { + f := func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags[filterKeyVal[0]]; ok { + if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] { + return false + } + } + return true + } + filters[name] = append(filters[name], f) + } + } + } +} diff --git a/mapping/matcher.go b/mapping/matcher.go index 3796eae..15c38f0 100644 --- a/mapping/matcher.go +++ b/mapping/matcher.go @@ -6,9 +6,11 @@ import ( ) func (m *Mapping) PointMatcher() NodeMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(PointTable, mappings) - filters := m.ElementFilters() + filters := make(tableElementFilters) + m.addFilters(filters) + m.addTypedFilters(PointTable, filters) return &tagMatcher{ mappings: mappings, tables: m.tables(PointTable), @@ -18,9 +20,11 @@ func (m *Mapping) PointMatcher() NodeMatcher { } func (m *Mapping) LineStringMatcher() WayMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(LineStringTable, mappings) - filters := m.ElementFilters() + filters := make(tableElementFilters) + m.addFilters(filters) + m.addTypedFilters(LineStringTable, filters) return &tagMatcher{ mappings: mappings, tables: m.tables(LineStringTable), @@ -30,48 +34,57 @@ func (m *Mapping) LineStringMatcher() WayMatcher { } func (m *Mapping) PolygonMatcher() RelWayMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(PolygonTable, mappings) - filters := m.ElementFilters() + filters := make(tableElementFilters) + m.addFilters(filters) + m.addTypedFilters(PolygonTable, filters) + relFilters := make(tableElementFilters) + m.addRelationFilters(PolygonTable, relFilters) return &tagMatcher{ mappings: mappings, tables: m.tables(PolygonTable), filters: filters, + relFilters: relFilters, matchAreas: true, } } func (m *Mapping) RelationMatcher() RelationMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(RelationTable, mappings) - filters := m.ElementFilters() + filters := make(tableElementFilters) + m.addFilters(filters) + m.addTypedFilters(PolygonTable, filters) + m.addTypedFilters(RelationTable, filters) + relFilters := make(tableElementFilters) + m.addRelationFilters(RelationTable, relFilters) return &tagMatcher{ mappings: mappings, tables: m.tables(RelationTable), filters: filters, + relFilters: relFilters, matchAreas: true, } } func (m *Mapping) RelationMemberMatcher() RelationMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(RelationMemberTable, mappings) - filters := m.ElementFilters() + filters := make(tableElementFilters) + m.addFilters(filters) + m.addTypedFilters(RelationMemberTable, filters) + relFilters := make(tableElementFilters) + m.addRelationFilters(RelationMemberTable, relFilters) return &tagMatcher{ mappings: mappings, tables: m.tables(RelationMemberTable), filters: filters, + relFilters: relFilters, matchAreas: true, } } -type Match struct { - Key string - Value string - Table DestTable - tableFields *TableFields -} - type NodeMatcher interface { MatchNode(node *element.Node) []Match } @@ -89,23 +102,31 @@ type RelWayMatcher interface { RelationMatcher } -type tagMatcher struct { - mappings TagTables - tables map[string]*TableFields - filters map[string][]ElementFilter - matchAreas bool +type Match struct { + Key string + Value string + Table DestTable + builder *rowBuilder } func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} { - return m.tableFields.MakeRow(elem, geom, *m) + return m.builder.MakeRow(elem, geom, *m) } func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry) []interface{} { - return m.tableFields.MakeMemberRow(rel, member, geom, *m) + return m.builder.MakeMemberRow(rel, member, geom, *m) +} + +type tagMatcher struct { + mappings TagTableMapping + tables map[string]*rowBuilder + filters tableElementFilters + relFilters tableElementFilters + matchAreas bool } func (tm *tagMatcher) MatchNode(node *element.Node) []Match { - return tm.match(node.Tags, false) + return tm.match(node.Tags, false, false) } func (tm *tagMatcher) MatchWay(way *element.Way) []Match { @@ -114,22 +135,22 @@ func (tm *tagMatcher) MatchWay(way *element.Way) []Match { if way.Tags["area"] == "no" { return nil } - return tm.match(way.Tags, true) + return tm.match(way.Tags, true, false) } } else { // match way as linestring if way.IsClosed() { if way.Tags["area"] == "yes" { return nil } - return tm.match(way.Tags, true) + return tm.match(way.Tags, true, false) } - return tm.match(way.Tags, false) + return tm.match(way.Tags, false, false) } return nil } func (tm *tagMatcher) MatchRelation(rel *element.Relation) []Match { - return tm.match(rel.Tags, true) + return tm.match(rel.Tags, true, true) } type orderedMatch struct { @@ -137,17 +158,17 @@ type orderedMatch struct { order int } -func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match { +func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Match { tables := make(map[DestTable]orderedMatch) - addTables := func(k, v string, tbls []OrderedDestTable) { + addTables := func(k, v string, tbls []orderedDestTable) { for _, t := range tbls { this := orderedMatch{ Match: Match{ - Key: k, - Value: v, - Table: t.DestTable, - tableFields: tm.tables[t.Name], + Key: k, + Value: v, + Table: t.DestTable, + builder: tm.tables[t.Name], }, order: t.order, } @@ -187,6 +208,18 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match { } } } + if relation && !filteredOut { + filters, ok := tm.relFilters[t.Name] + if ok { + for _, filter := range filters { + if !filter(tags, Key(match.Key), closed) { + filteredOut = true + break + } + } + } + } + if !filteredOut { matches = append(matches, match.Match) } @@ -194,6 +227,54 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match { return matches } +type valueBuilder struct { + key Key + colType ColumnType +} + +func (v *valueBuilder) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { + if v.colType.Func != nil { + return v.colType.Func(elem.Tags[string(v.key)], elem, geom, match) + } + return nil +} + +func (v *valueBuilder) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} { + if v.colType.Func != nil { + if v.colType.FromMember { + if member.Elem == nil { + return nil + } + return v.colType.Func(member.Elem.Tags[string(v.key)], member.Elem, geom, match) + } + return v.colType.Func(rel.Tags[string(v.key)], &rel.OSMElem, geom, match) + } + if v.colType.MemberFunc != nil { + return v.colType.MemberFunc(rel, member, match) + } + return nil +} + +type rowBuilder struct { + columns []valueBuilder +} + +func (r *rowBuilder) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} { + var row []interface{} + for _, column := range r.columns { + row = append(row, column.Value(elem, geom, match)) + } + return row +} + +func (r *rowBuilder) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} { + var row []interface{} + for _, column := range r.columns { + row = append(row, column.MemberValue(rel, member, geom, match)) + } + return row +} + // SelectRelationPolygons returns a slice of all members that are already // imported as part of the relation. // Outer members are "imported" if they share the same destination table. Inner members diff --git a/mapping/matcher_test.go b/mapping/matcher_test.go index 8e5efa2..6435719 100644 --- a/mapping/matcher_test.go +++ b/mapping/matcher_test.go @@ -43,7 +43,7 @@ func TestSelectRelationPolygonsSimple(t *testing.T) { t.Fatal(err) } r := element.Relation{} - r.Tags = element.Tags{"landuse": "park"} + r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} r.Members = []element.Member{ makeMember(0, element.Tags{"landuse": "forest"}), makeMember(1, element.Tags{"landuse": "park"}), @@ -68,7 +68,7 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) { t.Fatal(err) } r := element.Relation{} - r.Tags = element.Tags{"landuse": "park"} + r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} r.Members = []element.Member{ makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}), makeMember(1, element.Tags{"landuse": "forest"}), @@ -91,7 +91,7 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) { t.Fatal(err) } r := element.Relation{} - r.Tags = element.Tags{"landuse": "park"} + r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"} r.Members = []element.Member{ makeMember(0, element.Tags{"landuse": "park"}), makeMember(1, element.Tags{"natural": "forest"}), @@ -117,7 +117,7 @@ func TestSelectRelationPolygonsMultipleTags(t *testing.T) { t.Fatal(err) } r := element.Relation{} - r.Tags = element.Tags{"landuse": "forest", "natural": "scrub"} + r.Tags = element.Tags{"landuse": "forest", "natural": "scrub", "type": "multipolygon"} r.Members = []element.Member{ makeMember(0, element.Tags{"natural": "scrub"}), makeMember(1, element.Tags{"landuse": "forest"}), @@ -139,7 +139,7 @@ func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) { t.Fatal(err) } r := element.Relation{} - r.Tags = element.Tags{"waterway": "riverbank"} + r.Tags = element.Tags{"waterway": "riverbank", "type": "multipolygon"} r.Members = []element.Member{ makeMemberRole(0, element.Tags{"waterway": "riverbank", "natural": "water"}, "outer"), makeMemberRole(1, element.Tags{"natural": "water"}, "inner"), diff --git a/mapping/test_mapping.json b/mapping/test_mapping.json index 5b14963..2962dfb 100644 --- a/mapping/test_mapping.json +++ b/mapping/test_mapping.json @@ -43,7 +43,7 @@ }, "tables": { "landusages": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -209,7 +209,7 @@ } }, "buildings": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -240,7 +240,7 @@ }, "amenity_areas": { "_comment": "for testing duplicate inserts with __any__ and exact match", - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -270,7 +270,7 @@ } }, "places": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -333,7 +333,7 @@ } }, "transport_areas": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -370,7 +370,7 @@ } }, "admin": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -405,7 +405,7 @@ } }, "aeroways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -436,7 +436,7 @@ } }, "waterways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -473,7 +473,7 @@ } }, "barrierways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -515,7 +515,7 @@ } }, "transport_points": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -566,7 +566,7 @@ } }, "amenities": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -603,7 +603,7 @@ } }, "barrierpoints": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -648,7 +648,7 @@ } }, "housenumbers_interpolated": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -698,7 +698,7 @@ } }, "roads": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -818,7 +818,7 @@ } }, "housenumbers": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", diff --git a/mapping/test_mapping.yml b/mapping/test_mapping.yml index 71be265..c1b2b74 100644 --- a/mapping/test_mapping.yml +++ b/mapping/test_mapping.yml @@ -35,7 +35,7 @@ generalized_tables: tolerance: 50.0 tables: admin: - fields: + columns: - name: osm_id type: id - name: geometry @@ -53,7 +53,7 @@ tables: - administrative type: polygon aeroways: - fields: + columns: - name: osm_id type: id - name: geometry @@ -69,7 +69,7 @@ tables: - taxiway type: linestring amenities: - fields: + columns: - name: osm_id type: id - name: geometry @@ -92,7 +92,7 @@ tables: type: point amenity_areas: _comment: for testing duplicate inserts with __any__ and exact match - fields: + columns: - name: osm_id type: id - name: geometry @@ -107,7 +107,7 @@ tables: - shop type: polygon barrierpoints: - fields: + columns: - name: osm_id type: id - name: geometry @@ -137,7 +137,7 @@ tables: - stile type: point barrierways: - fields: + columns: - name: osm_id type: id - name: geometry @@ -164,7 +164,7 @@ tables: - wire_fence type: linestring buildings: - fields: + columns: - name: osm_id type: id - name: geometry @@ -179,7 +179,7 @@ tables: - __any__ type: polygon housenumbers: - fields: + columns: - name: osm_id type: id - name: geometry @@ -203,7 +203,7 @@ tables: - __any__ type: point housenumbers_interpolated: - fields: + columns: - name: osm_id type: id - name: geometry @@ -230,7 +230,7 @@ tables: - __any__ type: linestring landusages: - fields: + columns: - name: osm_id type: id - name: geometry @@ -363,7 +363,7 @@ tables: - riverbank type: polygon places: - fields: + columns: - name: osm_id type: id - name: geometry @@ -404,7 +404,7 @@ tables: - locality type: point roads: - fields: + columns: - name: osm_id type: id - name: geometry @@ -524,7 +524,7 @@ tables: - groyne type: linestring transport_areas: - fields: + columns: - name: osm_id type: id - name: geometry @@ -545,7 +545,7 @@ tables: - platform type: polygon transport_points: - fields: + columns: - name: osm_id type: id - name: geometry @@ -577,7 +577,7 @@ tables: - subway_entrance type: point waterways: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/test/any_any_mapping.json b/test/any_any_mapping.json index 5e060ed..4cd9a20 100644 --- a/test/any_any_mapping.json +++ b/test/any_any_mapping.json @@ -8,7 +8,7 @@ }, "tables": { "all": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id" @@ -28,7 +28,7 @@ } }, "amenities": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id" diff --git a/test/complete_db.osc b/test/complete_db.osc index 3999726..9be7b66 100644 --- a/test/complete_db.osc +++ b/test/complete_db.osc @@ -35,15 +35,6 @@ - - - - - - - - - @@ -71,6 +62,7 @@ + diff --git a/test/complete_db.osm b/test/complete_db.osm index ed5ceb4..75f3f1e 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -267,8 +267,19 @@ + + + + + + + + + + + @@ -524,7 +535,6 @@ - @@ -540,37 +550,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -606,25 +589,6 @@ - - - - - - - - - - - - - - - - - - - @@ -672,6 +636,7 @@ + @@ -694,7 +659,6 @@ - @@ -708,6 +672,7 @@ + @@ -947,7 +912,7 @@ - + @@ -1031,10 +996,10 @@ - + diff --git a/test/complete_db_mapping.json b/test/complete_db_mapping.json index 874926a..ef6d21f 100644 --- a/test/complete_db_mapping.json +++ b/test/complete_db_mapping.json @@ -54,7 +54,7 @@ }, "tables": { "landusages": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -191,7 +191,7 @@ } }, "buildings": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -228,7 +228,7 @@ } }, "places": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -290,7 +290,7 @@ } }, "transport_areas": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -327,7 +327,7 @@ } }, "admin": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -362,7 +362,7 @@ } }, "aeroways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -393,7 +393,7 @@ } }, "waterways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -430,7 +430,7 @@ } }, "barrierways": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -472,7 +472,7 @@ } }, "transport_points": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -523,7 +523,7 @@ } }, "amenities": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -560,7 +560,7 @@ } }, "barrierpoints": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -605,7 +605,7 @@ } }, "housenumbers_interpolated": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -655,7 +655,7 @@ } }, "roads": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -784,7 +784,7 @@ } }, "housenumbers": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", @@ -829,7 +829,7 @@ } }, "waterareas": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", diff --git a/test/completedb_test.go b/test/completedb_test.go index ba9b4b6..4bf0773 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -61,37 +61,38 @@ func TestComplete_Deploy(t *testing.T) { } } +func TestComplete_OnlyNewStyleMultipolgon(t *testing.T) { + assertRecords(t, []checkElem{ + {"osm_landusages", -1001, "wood", nil}, + {"osm_landusages", -1011, Missing, nil}, + {"osm_landusages", -1021, Missing, nil}, + }) +} + func TestComplete_LandusageToWaterarea1(t *testing.T) { // Parks inserted into landusages cache := ts.cache(t) defer cache.Close() assertCachedWay(t, cache, 11001) - assertCachedWay(t, cache, 12001) assertCachedWay(t, cache, 13001) assertRecords(t, []checkElem{ {"osm_waterareas", 11001, Missing, nil}, - {"osm_waterareas", -12001, Missing, nil}, {"osm_waterareas", -13001, Missing, nil}, {"osm_waterareas_gen0", 11001, Missing, nil}, - {"osm_waterareas_gen0", -12001, Missing, nil}, {"osm_waterareas_gen0", -13001, Missing, nil}, {"osm_waterareas_gen1", 11001, Missing, nil}, - {"osm_waterareas_gen1", -12001, Missing, nil}, {"osm_waterareas_gen1", -13001, Missing, nil}, {"osm_landusages", 11001, "park", nil}, - {"osm_landusages", -12001, "park", nil}, {"osm_landusages", -13001, "park", nil}, {"osm_landusages_gen0", 11001, "park", nil}, - {"osm_landusages_gen0", -12001, "park", nil}, {"osm_landusages_gen0", -13001, "park", nil}, {"osm_landusages_gen1", 11001, "park", nil}, - {"osm_landusages_gen1", -12001, "park", nil}, {"osm_landusages_gen1", -13001, "park", nil}, }) } @@ -106,7 +107,8 @@ func TestComplete_ChangedHoleTags1(t *testing.T) { assertRecords(t, []checkElem{ {"osm_waterareas", 14011, Missing, nil}, {"osm_waterareas", -14011, Missing, nil}, - {"osm_landusages", -14001, "park", nil}, + {"osm_landusages", 14001, "park", nil}, + {"osm_landusages", -14001, Missing, nil}, }) } @@ -218,19 +220,15 @@ func TestComplete_RelationWayNotInserted(t *testing.T) { func TestComplete_RelationWaysInserted(t *testing.T) { // Outer ways of multipolygon are inserted. assertRecords(t, []checkElem{ - {"osm_landusages", -9201, "park", map[string]string{"name": "9209"}}, + // no name on relation + {"osm_landusages", -9201, "park", map[string]string{"name": ""}}, {"osm_landusages", 9201, Missing, nil}, + {"osm_landusages", 9209, Missing, nil}, + {"osm_landusages", 9210, Missing, nil}, // outer ways of multipolygon stand for their own {"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}}, {"osm_roads", 9210, "residential", map[string]string{"name": "9210"}}, - - // no name on relation - {"osm_landusages", -9301, "park", map[string]string{"name": ""}}, - // outer ways of multipolygon stand for their own - {"osm_roads", 9309, "secondary", map[string]string{"name": "9309"}}, - {"osm_roads", 9310, "residential", map[string]string{"name": "9310"}}, }) - } func TestComplete_RelationWayInserted(t *testing.T) { @@ -283,12 +281,12 @@ func TestComplete_RelationBeforeRemove(t *testing.T) { }) } -func TestComplete_RelationWithoutTags(t *testing.T) { - // Relation without tags is inserted. +func TestComplete_OldStyleRelationIsIgnored(t *testing.T) { + // Relation without tags is not inserted. assertRecords(t, []checkElem{ - {"osm_buildings", 50111, Missing, nil}, - {"osm_buildings", -50121, "yes", nil}, + {"osm_buildings", 50111, "yes", nil}, + {"osm_buildings", -50121, Missing, nil}, }) } @@ -483,27 +481,21 @@ func TestComplete_LandusageToWaterarea2(t *testing.T) { assertRecords(t, []checkElem{ {"osm_waterareas", 11001, "water", nil}, - {"osm_waterareas", -12001, "water", nil}, {"osm_waterareas", -13001, "water", nil}, {"osm_waterareas_gen0", 11001, "water", nil}, - {"osm_waterareas_gen0", -12001, "water", nil}, {"osm_waterareas_gen0", -13001, "water", nil}, {"osm_waterareas_gen1", 11001, "water", nil}, - {"osm_waterareas_gen1", -12001, "water", nil}, {"osm_waterareas_gen1", -13001, "water", nil}, {"osm_landusages", 11001, Missing, nil}, - {"osm_landusages", -12001, Missing, nil}, {"osm_landusages", -13001, Missing, nil}, {"osm_landusages_gen0", 11001, Missing, nil}, - {"osm_landusages_gen0", -12001, Missing, nil}, {"osm_landusages_gen0", -13001, Missing, nil}, {"osm_landusages_gen1", 11001, Missing, nil}, - {"osm_landusages_gen1", -12001, Missing, nil}, {"osm_landusages_gen1", -13001, Missing, nil}, }) } @@ -518,6 +510,11 @@ func TestComplete_ChangedHoleTags2(t *testing.T) { assertGeomArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779) assertGeomArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182) + + assertRecords(t, []checkElem{ + {"osm_waterareas", -14011, Missing, nil}, + {"osm_landusages", -14001, "park", nil}, + }) } func TestComplete_SplitOuterMultipolygonWay2(t *testing.T) { diff --git a/test/expire_tiles_mapping.yml b/test/expire_tiles_mapping.yml index 94f0cc1..eae1f2f 100644 --- a/test/expire_tiles_mapping.yml +++ b/test/expire_tiles_mapping.yml @@ -1,7 +1,7 @@ tables: roads: type: linestring - fields: + columns: - name: osm_id type: id - name: type @@ -16,7 +16,7 @@ tables: pois: type: point - fields: + columns: - name: osm_id type: id - name: type @@ -31,7 +31,7 @@ tables: buildings: type: polygon - fields: + columns: - name: osm_id type: id - name: type diff --git a/test/route_relation.osm b/test/route_relation.osm index 9050239..0a0366a 100644 --- a/test/route_relation.osm +++ b/test/route_relation.osm @@ -153,5 +153,15 @@ + + + + + + + + + + diff --git a/test/route_relation_mapping.yml b/test/route_relation_mapping.yml index aa17e6d..c5db48e 100644 --- a/test/route_relation_mapping.yml +++ b/test/route_relation_mapping.yml @@ -1,13 +1,7 @@ -tags: - load_all: true - exclude: - - created_by - - source - tables: master_routes: type: relation_member - fields: + columns: - name: osm_id type: id - name: member @@ -27,11 +21,12 @@ tables: - key: name name: name type: string + relation_types: [route_master] mapping: route_master: [bus] route_members: type: relation_member - fields: + columns: - name: osm_id type: id - key: ref @@ -54,11 +49,12 @@ tables: key: name type: string from_member: true + relation_types: [route] mapping: route: [bus, tram, rail] routes: type: relation - fields: + columns: - name: osm_id type: id - key: ref @@ -66,5 +62,7 @@ tables: type: string - name: tags type: hstore_tags + relation_types: [route, route_master] mapping: + route_master: [bus, tram, rail] route: [bus, tram, rail] diff --git a/test/route_relation_test.go b/test/route_relation_test.go index a3aec6d..88522c8 100644 --- a/test/route_relation_test.go +++ b/test/route_relation_test.go @@ -58,6 +58,23 @@ func TestRouteRelation_RelationData(t *testing.T) { if r.tags["name"] != "Bus 301: A => B" { t.Error(r) } + + // check tags of master relation + r = ts.queryTags(t, "osm_routes", -100911) + if r.tags["name"] != "Bus 301" { + t.Error(r) + } +} + +func TestRouteRelation_MemberUpdatedByNode1(t *testing.T) { + // check that member is updated after node was modified + rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101") + if len(rows) != 1 { + t.Fatal(rows) + } + if rows[0]["name"] != "Stop" { + t.Error(rows[0]) + } } func TestRouteRelation_MemberGeomUpdated1(t *testing.T) { @@ -131,7 +148,7 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) { } -func TestRouteRelation_MemberUpdatedByNode(t *testing.T) { +func TestRouteRelation_MemberUpdatedByNode2(t *testing.T) { // check that member is updated after node was modified rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101") if len(rows) != 1 { diff --git a/test/single_table_mapping.json b/test/single_table_mapping.json index 266d013..093ebce 100644 --- a/test/single_table_mapping.json +++ b/test/single_table_mapping.json @@ -9,7 +9,7 @@ "use_single_id_space": true, "tables": { "all": { - "fields": [ + "columns": [ { "type": "id", "name": "osm_id", diff --git a/test/single_table_test.go b/test/single_table_test.go index ed4b2a4..3bf2b05 100644 --- a/test/single_table_test.go +++ b/test/single_table_test.go @@ -159,7 +159,7 @@ func TestSingleTable_DuplicateIds1(t *testing.T) { } assertHstore(t, []checkElem{ - {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}}, }) assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") } @@ -185,7 +185,7 @@ func TestSingleTable_DuplicateIds2(t *testing.T) { } assertHstore(t, []checkElem{ - {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}}, + {"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}}, }) assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon") } diff --git a/update/process.go b/update/process.go index 7d25fa0..b9513b2 100644 --- a/update/process.go +++ b/update/process.go @@ -119,7 +119,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi ProductionSchema: config.BaseOptions.Schemas.Production, BackupSchema: config.BaseOptions.Schemas.Backup, } - db, err := database.Open(dbConf, tagmapping) + db, err := database.Open(dbConf, &tagmapping.Conf) if err != nil { return errors.New("database open: " + err.Error()) } @@ -144,7 +144,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi delDb, osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, tagmapping.PointMatcher(), tagmapping.LineStringMatcher(), tagmapping.PolygonMatcher(), @@ -162,7 +162,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi nodes := make(chan *element.Node) relWriter := writer.NewRelationWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, relations, db, progress, tagmapping.PolygonMatcher(), @@ -174,7 +174,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi relWriter.Start() wayWriter := writer.NewWayWriter(osmCache, diffCache, - tagmapping.SingleIdSpace, + tagmapping.Conf.SingleIdSpace, ways, db, progress, tagmapping.PolygonMatcher(), @@ -350,11 +350,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi } // insert new relation progress.AddRelations(1) - // filter out unsupported relation types, otherwise they might - // get inserted with the tags from an outer way - if relTagFilter.Filter(&rel.Tags) { - relations <- rel - } + relations <- rel } for wayId, _ := range wayIds { diff --git a/writer/relations.go b/writer/relations.go index 53a8043..d2910f7 100644 --- a/writer/relations.go +++ b/writer/relations.go @@ -134,8 +134,12 @@ NextRel: } func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool { - // prepare relation first (build rings and compute actual - // relation tags) + matches := rw.polygonMatcher.MatchRelation(r) + if matches == nil { + return false + } + + // prepare relation (build rings) prepedRel, err := geomp.PrepareRelation(r, rw.srid, rw.maxGap) if err != nil { if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 { @@ -144,12 +148,6 @@ func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geo return false } - // check for matches befor building the geometry - matches := rw.polygonMatcher.MatchRelation(r) - if matches == nil { - return false - } - // build the multipolygon geom, err := prepedRel.Build() if geom.Geom != nil {