Merge pull request #146 from omniscale/remove-old-style-multipolygon-support

Remove old style multipolygon support

database/database.go

@@ -7,6 +7,7 @@ import (
 	"github.com/omniscale/imposm3/element"
 	"github.com/omniscale/imposm3/geom"
 	"github.com/omniscale/imposm3/mapping"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 type Config struct {
@@ -67,17 +68,17 @@ type Optimizer interface {
 	Optimize() error
 }
 
-var databases map[string]func(Config, *mapping.Mapping) (DB, error)
+var databases map[string]func(Config, *config.Mapping) (DB, error)
 
 func init() {
-	databases = make(map[string]func(Config, *mapping.Mapping) (DB, error))
+	databases = make(map[string]func(Config, *config.Mapping) (DB, error))
 }
 
-func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) {
+func Register(name string, f func(Config, *config.Mapping) (DB, error)) {
 	databases[name] = f
 }
 
-func Open(conf Config, m *mapping.Mapping) (DB, error) {
+func Open(conf Config, m *config.Mapping) (DB, error) {
 	parts := strings.SplitN(conf.ConnectionParams, ":", 2)
 	connectionType := parts[0]
 
@@ -108,7 +109,7 @@ func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geo
 	return nil
 }
 
-func newNullDb(conf Config, m *mapping.Mapping) (DB, error) {
+func newNullDb(conf Config, m *config.Mapping) (DB, error) {
 	return &nullDb{}, nil
 }
 

database/postgis/postgis.go

@@ -14,6 +14,7 @@ import (
 	"github.com/omniscale/imposm3/geom"
 	"github.com/omniscale/imposm3/logging"
 	"github.com/omniscale/imposm3/mapping"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 var log = logging.NewLogger("PostGIS")
@@ -599,7 +600,7 @@ func (pg *PostGIS) Close() error {
 	return pg.Db.Close()
 }
 
-func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
+func New(conf database.Config, m *config.Mapping) (database.DB, error) {
 	db := &PostGIS{}
 
 	db.Tables = make(map[string]*TableSpec)

database/postgis/spec.go

@@ -5,11 +5,12 @@ import (
 	"strings"
 
 	"github.com/omniscale/imposm3/mapping"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 type ColumnSpec struct {
 	Name      string
-	FieldType mapping.FieldType
+	FieldType mapping.ColumnType
 	Type      ColumnType
 }
 type TableSpec struct {
@@ -124,12 +125,11 @@ func (spec *TableSpec) DeleteSQL() string {
 	)
 }
 
-func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
+func NewTableSpec(pg *PostGIS, t *config.Table) *TableSpec {
 	var geomType string
-	switch t.Type {
+	if mapping.TableType(t.Type) == mapping.RelationMemberTable {
-	case mapping.RelationMemberTable:
 		geomType = "geometry"
-	default:
+	} else {
 		geomType = string(t.Type)
 	}
 
@@ -140,23 +140,23 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
 		GeometryType: geomType,
 		Srid:         pg.Config.Srid,
 	}
-	for _, field := range t.Fields {
+	for _, column := range t.Columns {
-		fieldType := field.FieldType()
+		columnType := mapping.MakeColumnType(column)
-		if fieldType == nil {
+		if columnType == nil {
 			continue
 		}
-		pgType, ok := pgTypes[fieldType.GoType]
+		pgType, ok := pgTypes[columnType.GoType]
 		if !ok {
-			log.Errorf("unhandled field type %v, using string type", fieldType)
+			log.Errorf("unhandled column type %v, using string type", columnType)
 			pgType = pgTypes["string"]
 		}
-		col := ColumnSpec{field.Name, *fieldType, pgType}
+		col := ColumnSpec{column.Name, *columnType, pgType}
 		spec.Columns = append(spec.Columns, col)
 	}
 	return &spec
 }
 
-func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
+func NewGeneralizedTableSpec(pg *PostGIS, t *config.GeneralizedTable) *GeneralizedTableSpec {
 	spec := GeneralizedTableSpec{
 		Name:       t.Name,
 		FullName:   pg.Prefix + t.Name,

docs/mapping.rst

@@ -38,10 +38,28 @@ To import all polygons with `tourism=zoo`, `natural=wood` or `natural=land` into
         …
 
 
+``relation_types``
+~~~~~~~~~~~~~~~~~~
+
+``relation_types`` restricts which relation types should be imported. It is a list with `type` values, e.g. ``[route, master_route]``.
+
+For tables of type ``relation`` and ``relation_member``: Only import relations which have this type value. You still need to have a mapping.
+For tables of type ``polygon``: Only build multi-polygons for relations which have this type value. You still need to have a mapping. Defaults to ``[multipolygon, boundary, land_area]``.
+
+.. code-block:: yaml
+
+    tables:
+      routes:
+        type: relation
+        relation_types: [route]
+        mapping:
+          route: [bus]
+
+
 ``columns``
 ~~~~~~~~~~~
 
-``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionaly ``key``, ``args`` and ``from_member``.
+``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionally ``key``, ``args`` and ``from_member``.
 
 ``name``
 ^^^^^^^^^
@@ -349,6 +367,16 @@ To load all tags except ``created_by``, ``source``, and ``tiger:county``, ``tige
 
 
 
+To load specific data about amenities for inclusion into an `hstore_tags` column:
+
+.. code-block:: yaml
+
+    tags:
+      include: [operator, opening_hours, wheelchair, website, phone, cuisine]
+
+
+
+
 .. _Areas:
 
 Areas

docs/relations.rst

@@ -42,21 +42,7 @@ It will also insert relations of the type ``multipolygon`` with a ``building`` t
 
 The roles are ignored by Imposm as not all holes are correctly tagged as ``inner``. Imposm uses geometry operations to verify if a member of a multipolygon is a hole, or if it is a separate polygon.
 
-
+Old-style multipolygon relations with tags on the outer way, instead of the relation are no longer supported.
-For compatibility, multipolygon relations without tags will use the tags from the (longest) outer way. Imposm will insert the following relation as well::
-
-  <way id="18101" version="1" timestamp="2011-11-11T00:11:11Z">
-    <nd ref="1001"/>
-    ...
-    <nd ref="1001"/>
-    <tag k="building" v="yes"/>
-  </way>
-  <relation id="18901" version="1" timestamp="2011-11-11T00:11:11Z">
-    <member type="way" ref="18101" role="outer"/>
-    <member type="way" ref="18102" role="outer"/>
-    <tag k="type" v="multipolygon"/>
-  </relation>
-
 
 
 Other relations
@@ -72,7 +58,6 @@ These relations can not be mapped to `simple` linestrings or polygons as they ca
 
 The Imposm table types ``relation`` and ``relation_member`` allow you to import all relevant data for these relations.
 
-.. note:: ``relation`` and ``relation_member`` require :ref:`load_all<tags>` to have access to all keys.
 
 ``relation_member``
 ^^^^^^^^^^^^^^^^^^^
@@ -109,6 +94,7 @@ You can use the following mapping::
     - key: ref
       name: ref
       type: string
+    relation_type: [route]
     mapping:
       route: [bus]
 
@@ -171,6 +157,7 @@ The following mapping imports the bus route relation from above::
     - name: network
       key: network
       type: string
+    relation_type: [route]
     mapping:
       route: [bus]
 

geom/multipolygon.go

@@ -16,15 +16,12 @@ type PreparedRelation struct {
 
 // PrepareRelation is the first step in building a (multi-)polygon of a Relation.
 // It builds rings from all ways and returns an error if there are unclosed rings.
-// It also merges the Relation.Tags with the Tags of the outer way.
 func PrepareRelation(rel *element.Relation, srid int, maxRingGap float64) (PreparedRelation, error) {
 	rings, err := buildRings(rel, maxRingGap)
 	if err != nil {
 		return PreparedRelation{}, err
 	}
 
-	rel.Tags = relationTags(rel.Tags, rings[0].ways[0].Tags)
-
 	return PreparedRelation{rings, rel, srid}, nil
 }
 
@@ -224,29 +221,6 @@ func buildRelGeometry(g *geos.Geos, rel *element.Relation, rings []*ring) (*geos
 	return result, nil
 }
 
-func relationTags(relTags, wayTags element.Tags) element.Tags {
-	result := make(element.Tags)
-	for k, v := range relTags {
-		if k == "name" || k == "type" {
-			continue
-		}
-		result[k] = v
-	}
-
-	if len(result) == 0 {
-		// relation does not have tags? use way tags
-		for k, v := range wayTags {
-			result[k] = v
-		}
-	} else {
-		// add back name (if present)
-		if name, ok := relTags["name"]; ok {
-			result["name"] = name
-		}
-	}
-	return result
-}
-
 // ringIsHole returns true if rings[idx] is a hole, False if it is a
 // shell (also if hole in a hole, etc)
 func ringIsHole(rings []*ring, idx int) bool {

geom/multipolygon_test.go

@@ -94,7 +94,7 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) {
 	})
 
 	rel := element.Relation{
-		OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "rel"}}}
+		OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "Relation", "natural": "forest", "type": "multipolygon"}}}
 	rel.Members = []element.Member{
 		{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
 		{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@@ -107,10 +107,11 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) {
 	g := geos.NewGeos()
 	defer g.Finish()
 
-	if len(rel.Tags) != 2 {
+	if len(rel.Tags) != 3 {
 		t.Fatal("wrong rel tags", rel.Tags)
 	}
-	if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Blackwood" {
+	// name from way is ignored
+	if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Relation" {
 		t.Fatal("wrong rel tags", rel.Tags)
 	}
 
@@ -147,7 +148,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) {
 	})
 
 	rel := element.Relation{
-		OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}}
+		OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
 	rel.Members = []element.Member{
 		{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
 		{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@@ -161,7 +162,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) {
 	g := geos.NewGeos()
 	defer g.Finish()
 
-	if len(rel.Tags) != 1 {
+	if len(rel.Tags) != 2 {
 		t.Fatal("wrong rel tags", rel.Tags)
 	}
 	if rel.Tags["landusage"] != "forest" {
@@ -214,7 +215,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) {
 		{1, 4, 4},
 	})
 
-	rel := element.Relation{OSMElem: element.OSMElem{Id: 1}}
+	rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
 	rel.Members = []element.Member{
 		{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
 		{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@@ -230,7 +231,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) {
 	g := geos.NewGeos()
 	defer g.Finish()
 
-	if len(rel.Tags) != 1 {
+	if len(rel.Tags) != 2 {
 		t.Fatal("wrong rel tags", rel.Tags)
 	}
 	if rel.Tags["landusage"] != "forest" {
@@ -261,7 +262,7 @@ func TestPolygonFromThreeWays(t *testing.T) {
 		{1, 0, 0},
 	})
 
-	rel := element.Relation{OSMElem: element.OSMElem{Id: 1}}
+	rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
 	rel.Members = []element.Member{
 		{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
 		{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@@ -275,7 +276,7 @@ func TestPolygonFromThreeWays(t *testing.T) {
 	g := geos.NewGeos()
 	defer g.Finish()
 
-	if len(rel.Tags) != 1 {
+	if len(rel.Tags) != 2 {
 		t.Fatal("wrong rel tags", rel.Tags)
 	}
 	if rel.Tags["landusage"] != "forest" {

import_/import.go

@@ -67,7 +67,7 @@ func Import() {
 			ProductionSchema: config.BaseOptions.Schemas.Production,
 			BackupSchema:     config.BaseOptions.Schemas.Backup,
 		}
-		db, err = database.Open(conf, tagmapping)
+		db, err = database.Open(conf, &tagmapping.Conf)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -181,7 +181,7 @@ func Import() {
 
 		relations := osmCache.Relations.Iter()
 		relWriter := writer.NewRelationWriter(osmCache, diffCache,
-			tagmapping.SingleIdSpace,
+			tagmapping.Conf.SingleIdSpace,
 			relations,
 			db, progress,
 			tagmapping.PolygonMatcher(),
@@ -196,7 +196,7 @@ func Import() {
 
 		ways := osmCache.Ways.Iter()
 		wayWriter := writer.NewWayWriter(osmCache, diffCache,
-			tagmapping.SingleIdSpace,
+			tagmapping.Conf.SingleIdSpace,
 			ways, db,
 			progress,
 			tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),

mapping/columns.go

@@ -11,14 +11,15 @@ import (
 	"github.com/omniscale/imposm3/element"
 	"github.com/omniscale/imposm3/geom"
 	"github.com/omniscale/imposm3/logging"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 var log = logging.NewLogger("mapping")
 
-var AvailableFieldTypes map[string]FieldType
+var AvailableColumnTypes map[string]ColumnType
 
 func init() {
-	AvailableFieldTypes = map[string]FieldType{
+	AvailableColumnTypes = map[string]ColumnType{
 		"bool":                 {"bool", "bool", Bool, nil, nil, false},
 		"boolint":              {"boolint", "int8", BoolInt, nil, nil, false},
 		"id":                   {"id", "int64", Id, nil, nil, false},
@@ -47,94 +48,12 @@ func init() {
 type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{}
 type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{}
 
-type MakeMakeValue func(string, FieldType, Field) (MakeValue, error)
+type MakeMakeValue func(string, ColumnType, config.Column) (MakeValue, error)
 
 type Key string
 type Value string
 
-type FieldSpec struct {
+type ColumnType struct {
-	Key  Key
-	Type FieldType
-}
-
-func (f *FieldSpec) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
-	if f.Type.Func != nil {
-		return f.Type.Func(elem.Tags[string(f.Key)], elem, geom, match)
-	}
-	return nil
-}
-
-func (f *FieldSpec) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} {
-	if f.Type.Func != nil {
-		if f.Type.FromMember {
-			if member.Elem == nil {
-				return nil
-			}
-			return f.Type.Func(member.Elem.Tags[string(f.Key)], member.Elem, geom, match)
-		}
-		return f.Type.Func(rel.Tags[string(f.Key)], &rel.OSMElem, geom, match)
-	}
-	if f.Type.MemberFunc != nil {
-		return f.Type.MemberFunc(rel, member, match)
-	}
-	return nil
-}
-
-type TableFields struct {
-	fields []FieldSpec
-}
-
-func (t *TableFields) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} {
-	var row []interface{}
-	for _, field := range t.fields {
-		row = append(row, field.Value(elem, geom, match))
-	}
-	return row
-}
-
-func (t *TableFields) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} {
-	var row []interface{}
-	for _, field := range t.fields {
-		row = append(row, field.MemberValue(rel, member, geom, match))
-	}
-	return row
-}
-
-func (field *Field) FieldType() *FieldType {
-	if fieldType, ok := AvailableFieldTypes[field.Type]; ok {
-		if fieldType.MakeFunc != nil {
-			makeValue, err := fieldType.MakeFunc(field.Name, fieldType, *field)
-			if err != nil {
-				log.Print(err)
-				return nil
-			}
-			fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMember}
-		}
-		fieldType.FromMember = field.FromMember
-		return &fieldType
-	}
-	return nil
-}
-
-func (t *Table) TableFields() *TableFields {
-	result := TableFields{}
-
-	for _, mappingField := range t.Fields {
-		field := FieldSpec{}
-		field.Key = mappingField.Key
-
-		fieldType := mappingField.FieldType()
-		if fieldType != nil {
-			field.Type = *fieldType
-		} else {
-			log.Warn("unhandled type: ", mappingField.Type)
-		}
-		result.fields = append(result.fields, field)
-	}
-	return &result
-}
-
-type FieldType struct {
 	Name       string
 	GoType     string
 	Func       MakeValue
@@ -216,7 +135,7 @@ func Geometry(val string, elem *element.OSMElem, geom *geom.Geometry, match Matc
 	return string(geom.Wkb)
 }
 
-func MakePseudoArea(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakePseudoArea(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
 	log.Print("warn: pseudoarea type is deprecated and will be removed. See area and webmercarea type.")
 	return Area, nil
 }
@@ -248,14 +167,14 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M
 
 var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakeHStoreString(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
 	var includeAll bool
 	var err error
 	var include map[string]int
-	if _, ok := field.Args["include"]; !ok {
+	if _, ok := column.Args["include"]; !ok {
 		includeAll = true
 	} else {
-		include, err = decodeEnumArg(field, "include")
+		include, err = decodeEnumArg(column, "include")
 		if err != nil {
 			return nil, err
 		}
@@ -273,18 +192,18 @@ func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeV
 	return hstoreString, nil
 }
 
-func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakeWayZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
-	if _, ok := field.Args["ranks"]; !ok {
+	if _, ok := column.Args["ranks"]; !ok {
 		return DefaultWayZOrder, nil
 	}
-	ranks, err := decodeEnumArg(field, "ranks")
+	ranks, err := decodeEnumArg(column, "ranks")
 	if err != nil {
 		return nil, err
 	}
 	levelOffset := len(ranks)
 
 	defaultRank := 0
-	if val, ok := field.Args["default"].(float64); ok {
+	if val, ok := column.Args["default"].(float64); ok {
 		defaultRank = int(val)
 	}
 
@@ -361,9 +280,9 @@ func DefaultWayZOrder(val string, elem *element.OSMElem, geom *geom.Geometry, ma
 	return z
 }
 
-func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakeZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
 	log.Print("warn: zorder type is deprecated and will be removed. See enumerate type.")
-	_rankList, ok := field.Args["ranks"]
+	_rankList, ok := column.Args["ranks"]
 	if !ok {
 		return nil, errors.New("missing ranks in args for zorder")
 	}
@@ -374,7 +293,7 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue,
 	}
 
 	var key string
-	_key, ok := field.Args["key"]
+	_key, ok := column.Args["key"]
 	if ok {
 		key, ok = _key.(string)
 		if !ok {
@@ -408,13 +327,13 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue,
 	return zOrder, nil
 }
 
-func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakeEnumerate(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
-	values, err := decodeEnumArg(field, "values")
+	values, err := decodeEnumArg(column, "values")
 	if err != nil {
 		return nil, err
 	}
 	enumerate := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
-		if field.Key != "" {
+		if column.Key != "" {
 			if r, ok := values[val]; ok {
 				return r
 			}
@@ -429,15 +348,15 @@ func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValu
 	return enumerate, nil
 }
 
-func decodeEnumArg(field Field, key string) (map[string]int, error) {
+func decodeEnumArg(column config.Column, key string) (map[string]int, error) {
-	_valuesList, ok := field.Args[key]
+	_valuesList, ok := column.Args[key]
 	if !ok {
-		return nil, fmt.Errorf("missing '%v' in args for %s", key, field.Type)
+		return nil, fmt.Errorf("missing '%v' in args for %s", key, column.Type)
 	}
 
 	valuesList, ok := _valuesList.([]interface{})
 	if !ok {
-		return nil, fmt.Errorf("'%v' in args for %s not a list", key, field.Type)
+		return nil, fmt.Errorf("'%v' in args for %s not a list", key, column.Type)
 	}
 
 	values := make(map[string]int)
@@ -452,8 +371,8 @@ func decodeEnumArg(field Field, key string) (map[string]int, error) {
 	return values, nil
 }
 
-func MakeSuffixReplace(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
+func MakeSuffixReplace(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
-	_changes, ok := field.Args["suffixes"]
+	_changes, ok := column.Args["suffixes"]
 	if !ok {
 		return nil, errors.New("missing suffixes in args for string_suffixreplace")
 	}

mapping/columns_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/omniscale/imposm3/element"
 	"github.com/omniscale/imposm3/geom"
 	"github.com/omniscale/imposm3/geom/geos"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 func TestBool(t *testing.T) {
@@ -74,8 +75,8 @@ func TestZOrder(t *testing.T) {
 	match := Match{}
 
 	zOrder, err := MakeZOrder("z_order",
-		AvailableFieldTypes["z_order"],
+		AvailableColumnTypes["z_order"],
-		Field{
+		config.Column{
 			Name: "z_order",
 			Key:  "",
 			Type: "z_order",
@@ -113,8 +114,8 @@ func TestEnumerate_Match(t *testing.T) {
 	// test enumerate by matched mapping key
 
 	zOrder, err := MakeEnumerate("enumerate",
-		AvailableFieldTypes["enumerate"],
+		AvailableColumnTypes["enumerate"],
-		Field{
+		config.Column{
 			Name: "enumerate",
 			Key:  "",
 			Type: "enumerate",
@@ -148,8 +149,8 @@ func TestEnumerate_Key(t *testing.T) {
 	// test enumerate by key
 
 	zOrder, err := MakeEnumerate("enumerate",
-		AvailableFieldTypes["enumerate"],
+		AvailableColumnTypes["enumerate"],
-		Field{
+		config.Column{
 			Name: "enumerate",
 			Key:  "fips",
 			Type: "enumerate",
@@ -182,8 +183,8 @@ func TestEnumerate_Key(t *testing.T) {
 
 func TestWayZOrder(t *testing.T) {
 	zOrder, err := MakeWayZOrder("z_order",
-		AvailableFieldTypes["wayzorder"],
+		AvailableColumnTypes["wayzorder"],
-		Field{
+		config.Column{
 			Name: "zorder",
 			Type: "wayzorder",
 			Args: map[string]interface{}{
@@ -233,7 +234,7 @@ func TestWayZOrder(t *testing.T) {
 	}
 }
 
-func TestAreaFields(t *testing.T) {
+func TestAreaColumn(t *testing.T) {
 	tests := []struct {
 		wkt      string
 		expected float32
@@ -277,10 +278,10 @@ func TestAreaFields(t *testing.T) {
 }
 
 func TestMakeSuffixReplace(t *testing.T) {
-	field := Field{
+	column := config.Column{
 		Name: "name", Key: "name", Type: "string_suffixreplace",
 		Args: map[string]interface{}{"suffixes": map[interface{}]interface{}{"Straße": "Str.", "straße": "str."}}}
-	suffixReplace, err := MakeSuffixReplace("name", FieldType{}, field)
+	suffixReplace, err := MakeSuffixReplace("name", ColumnType{}, column)
 
 	if err != nil {
 		t.Fatal(err)
@@ -298,27 +299,27 @@ func TestMakeSuffixReplace(t *testing.T) {
 }
 
 func TestHstoreString(t *testing.T) {
-	field := Field{
+	column := config.Column{
 		Name: "tags",
 		Type: "hstore_tags",
 	}
-	hstoreAll, err := MakeHStoreString("tags", FieldType{}, field)
+	hstoreAll, err := MakeHStoreString("tags", ColumnType{}, column)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	field = Field{
+	column = config.Column{
 		Name: "tags",
 		Type: "hstore_tags",
 		Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}},
 	}
-	hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field)
+	hstoreInclude, err := MakeHStoreString("tags", ColumnType{}, column)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	for _, test := range []struct {
-		field    MakeValue
+		column   MakeValue
 		tags     element.Tags
 		expected interface{}
 	}{
@@ -331,7 +332,7 @@ func TestHstoreString(t *testing.T) {
 		{hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`},
 		{hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`},
 	} {
-		actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{})
+		actual := test.column("", &element.OSMElem{Tags: test.tags}, nil, Match{})
 		if actual.(string) != test.expected {
 			t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags)
 		}

mapping/config.go

@@ -1,347 +0,0 @@
-package mapping
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-
-	"github.com/omniscale/imposm3/element"
-
-	"gopkg.in/yaml.v2"
-)
-
-type Field struct {
-	Name       string                 `yaml:"name"`
-	Key        Key                    `yaml:"key"`
-	Keys       []Key                  `yaml:"keys"`
-	Type       string                 `yaml:"type"`
-	Args       map[string]interface{} `yaml:"args"`
-	FromMember bool                   `yaml:"from_member"`
-}
-
-type Table struct {
-	Name         string
-	Type         TableType             `yaml:"type"`
-	Mapping      KeyValues             `yaml:"mapping"`
-	Mappings     map[string]SubMapping `yaml:"mappings"`
-	TypeMappings TypeMappings          `yaml:"type_mappings"`
-	Fields       []*Field              `yaml:"columns"` // TODO rename Fields internaly to Columns
-	OldFields    []*Field              `yaml:"fields"`
-	Filters      *Filters              `yaml:"filters"`
-}
-
-type GeneralizedTable struct {
-	Name            string
-	SourceTableName string  `yaml:"source"`
-	Tolerance       float64 `yaml:"tolerance"`
-	SqlFilter       string  `yaml:"sql_filter"`
-}
-
-type Filters struct {
-	ExcludeTags *[][]string `yaml:"exclude_tags"`
-}
-
-type Tables map[string]*Table
-
-type GeneralizedTables map[string]*GeneralizedTable
-
-type Mapping struct {
-	Tables            Tables            `yaml:"tables"`
-	GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
-	Tags              Tags              `yaml:"tags"`
-	Areas             Areas             `yaml:"areas"`
-	// SingleIdSpace mangles the overlapping node/way/relation IDs
-	// to be unique (nodes positive, ways negative, relations negative -1e17)
-	SingleIdSpace bool `yaml:"use_single_id_space"`
-}
-
-type Areas struct {
-	AreaTags   []Key `yaml:"area_tags"`
-	LinearTags []Key `yaml:"linear_tags"`
-}
-
-type Tags struct {
-	LoadAll bool  `yaml:"load_all"`
-	Exclude []Key `yaml:"exclude"`
-	Include []Key `yaml:"include"`
-}
-
-type orderedValue struct {
-	value Value
-	order int
-}
-type KeyValues map[Key][]orderedValue
-
-func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	if *kv == nil {
-		*kv = make(map[Key][]orderedValue)
-	}
-	slice := yaml.MapSlice{}
-	err := unmarshal(&slice)
-	if err != nil {
-		return err
-	}
-	order := 0
-	for _, item := range slice {
-		k, ok := item.Key.(string)
-		if !ok {
-			return fmt.Errorf("mapping key '%s' not a string", k)
-		}
-		values, ok := item.Value.([]interface{})
-		if !ok {
-			return fmt.Errorf("mapping key '%s' not a string", k)
-		}
-		for _, v := range values {
-			if v, ok := v.(string); ok {
-				(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order})
-			} else {
-				return fmt.Errorf("mapping value '%s' not a string", v)
-			}
-			order += 1
-		}
-	}
-	return nil
-}
-
-type SubMapping struct {
-	Mapping KeyValues
-}
-
-type TypeMappings struct {
-	Points      KeyValues `yaml:"points"`
-	LineStrings KeyValues `yaml:"linestrings"`
-	Polygons    KeyValues `yaml:"polygons"`
-}
-
-type ElementFilter func(tags element.Tags, key Key, closed bool) bool
-
-type TagTables map[Key]map[Value][]OrderedDestTable
-
-type DestTable struct {
-	Name       string
-	SubMapping string
-}
-
-type OrderedDestTable struct {
-	DestTable
-	order int
-}
-
-type TableType string
-
-func (tt *TableType) UnmarshalJSON(data []byte) error {
-	switch string(data) {
-	case "":
-		return errors.New("missing table type")
-	case `"point"`:
-		*tt = PointTable
-	case `"linestring"`:
-		*tt = LineStringTable
-	case `"polygon"`:
-		*tt = PolygonTable
-	case `"geometry"`:
-		*tt = GeometryTable
-	case `"relation"`:
-		*tt = RelationTable
-	case `"relation_member"`:
-		*tt = RelationMemberTable
-	default:
-		return errors.New("unknown type " + string(data))
-	}
-	return nil
-}
-
-const (
-	PolygonTable        TableType = "polygon"
-	LineStringTable     TableType = "linestring"
-	PointTable          TableType = "point"
-	GeometryTable       TableType = "geometry"
-	RelationTable       TableType = "relation"
-	RelationMemberTable TableType = "relation_member"
-)
-
-func NewMapping(filename string) (*Mapping, error) {
-	f, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-
-	mapping := Mapping{}
-	err = yaml.Unmarshal(f, &mapping)
-	if err != nil {
-		return nil, err
-	}
-
-	err = mapping.prepare()
-	if err != nil {
-		return nil, err
-	}
-	return &mapping, nil
-}
-
-func (t *Table) ExtraTags() map[Key]bool {
-	tags := make(map[Key]bool)
-	for _, field := range t.Fields {
-		if field.Key != "" {
-			tags[field.Key] = true
-		}
-		for _, k := range field.Keys {
-			tags[k] = true
-		}
-	}
-	return tags
-}
-
-func (m *Mapping) prepare() error {
-	for name, t := range m.Tables {
-		t.Name = name
-		if t.OldFields != nil {
-			// todo deprecate 'fields'
-			t.Fields = t.OldFields
-		}
-	}
-
-	for name, t := range m.GeneralizedTables {
-		t.Name = name
-	}
-	return nil
-}
-
-func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) {
-	for key, vals := range mapping {
-		for _, v := range vals {
-			vals, ok := tt[key]
-			tbl := OrderedDestTable{DestTable: table, order: v.order}
-			if ok {
-				vals[v.value] = append(vals[v.value], tbl)
-			} else {
-				tt[key] = make(map[Value][]OrderedDestTable)
-				tt[key][v.value] = append(tt[key][v.value], tbl)
-			}
-		}
-	}
-}
-
-func (m *Mapping) mappings(tableType TableType, mappings TagTables) {
-	for name, t := range m.Tables {
-		if t.Type != GeometryTable && t.Type != tableType {
-			continue
-		}
-		mappings.addFromMapping(t.Mapping, DestTable{Name: name})
-
-		for subMappingName, subMapping := range t.Mappings {
-			mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
-		}
-
-		switch tableType {
-		case PointTable:
-			mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
-		case LineStringTable:
-			mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
-		case PolygonTable:
-			mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
-		}
-	}
-}
-
-func (m *Mapping) tables(tableType TableType) map[string]*TableFields {
-	result := make(map[string]*TableFields)
-	for name, t := range m.Tables {
-		if t.Type == tableType || t.Type == "geometry" {
-			result[name] = t.TableFields()
-		}
-	}
-	return result
-}
-
-func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
-	for _, t := range m.Tables {
-		if t.Type != tableType && t.Type != "geometry" {
-			continue
-		}
-		for key, _ := range t.ExtraTags() {
-			tags[key] = true
-		}
-		if t.Filters != nil && t.Filters.ExcludeTags != nil {
-			for _, keyVal := range *t.Filters.ExcludeTags {
-				tags[Key(keyVal[0])] = true
-			}
-		}
-	}
-	for _, k := range m.Tags.Include {
-		tags[k] = true
-	}
-
-	// always include area tag for closed-way handling
-	tags["area"] = true
-}
-
-func (m *Mapping) ElementFilters() map[string][]ElementFilter {
-	result := make(map[string][]ElementFilter)
-
-	var areaTags map[Key]struct{}
-	var linearTags map[Key]struct{}
-	if m.Areas.AreaTags != nil {
-		areaTags = make(map[Key]struct{})
-		for _, tag := range m.Areas.AreaTags {
-			areaTags[tag] = struct{}{}
-		}
-	}
-	if m.Areas.LinearTags != nil {
-		linearTags = make(map[Key]struct{})
-		for _, tag := range m.Areas.LinearTags {
-			linearTags[tag] = struct{}{}
-		}
-	}
-
-	for name, t := range m.Tables {
-		if t.Type == LineStringTable && areaTags != nil {
-			f := func(tags element.Tags, key Key, closed bool) bool {
-				if closed {
-					if tags["area"] == "yes" {
-						return false
-					}
-					if tags["area"] != "no" {
-						if _, ok := areaTags[key]; ok {
-							return false
-						}
-					}
-				}
-				return true
-			}
-			result[name] = append(result[name], f)
-		}
-		if t.Type == PolygonTable && linearTags != nil {
-			f := func(tags element.Tags, key Key, closed bool) bool {
-				if closed && tags["area"] == "no" {
-					return false
-				}
-				if tags["area"] != "yes" {
-					if _, ok := linearTags[key]; ok {
-						return false
-					}
-				}
-				return true
-			}
-			result[name] = append(result[name], f)
-		}
-
-		if t.Filters == nil {
-			continue
-		}
-		if t.Filters.ExcludeTags != nil {
-			for _, filterKeyVal := range *t.Filters.ExcludeTags {
-				f := func(tags element.Tags, key Key, closed bool) bool {
-					if v, ok := tags[filterKeyVal[0]]; ok {
-						if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
-							return false
-						}
-					}
-					return true
-				}
-				result[name] = append(result[name], f)
-			}
-		}
-	}
-	return result
-}

mapping/config/config.go

@@ -0,0 +1,113 @@
+package config
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+type Mapping struct {
+	Tables            Tables            `yaml:"tables"`
+	GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
+	Tags              Tags              `yaml:"tags"`
+	Areas             Areas             `yaml:"areas"`
+	// SingleIdSpace mangles the overlapping node/way/relation IDs
+	// to be unique (nodes positive, ways negative, relations negative -1e17)
+	SingleIdSpace bool `yaml:"use_single_id_space"`
+}
+
+type Column struct {
+	Name       string                 `yaml:"name"`
+	Key        Key                    `yaml:"key"`
+	Keys       []Key                  `yaml:"keys"`
+	Type       string                 `yaml:"type"`
+	Args       map[string]interface{} `yaml:"args"`
+	FromMember bool                   `yaml:"from_member"`
+}
+
+type Tables map[string]*Table
+type Table struct {
+	Name          string
+	Type          string                `yaml:"type"`
+	Mapping       KeyValues             `yaml:"mapping"`
+	Mappings      map[string]SubMapping `yaml:"mappings"`
+	TypeMappings  TypeMappings          `yaml:"type_mappings"`
+	Columns       []*Column             `yaml:"columns"`
+	OldFields     []*Column             `yaml:"fields"`
+	Filters       *Filters              `yaml:"filters"`
+	RelationTypes []string              `yaml:"relation_types"`
+}
+
+type GeneralizedTables map[string]*GeneralizedTable
+type GeneralizedTable struct {
+	Name            string
+	SourceTableName string  `yaml:"source"`
+	Tolerance       float64 `yaml:"tolerance"`
+	SqlFilter       string  `yaml:"sql_filter"`
+}
+
+type Filters struct {
+	ExcludeTags *[][]string `yaml:"exclude_tags"`
+}
+
+type Areas struct {
+	AreaTags   []Key `yaml:"area_tags"`
+	LinearTags []Key `yaml:"linear_tags"`
+}
+
+type Tags struct {
+	LoadAll bool  `yaml:"load_all"`
+	Exclude []Key `yaml:"exclude"`
+	Include []Key `yaml:"include"`
+}
+
+type Key string
+type Value string
+
+type orderedValue struct {
+	Value Value
+	Order int
+}
+
+type KeyValues map[Key][]orderedValue
+
+func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	if *kv == nil {
+		*kv = make(map[Key][]orderedValue)
+	}
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+	order := 0
+	for _, item := range slice {
+		k, ok := item.Key.(string)
+		if !ok {
+			return fmt.Errorf("mapping key '%s' not a string", k)
+		}
+		values, ok := item.Value.([]interface{})
+		if !ok {
+			return fmt.Errorf("mapping key '%s' not a string", k)
+		}
+		for _, v := range values {
+			if v, ok := v.(string); ok {
+				(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{Value: Value(v), Order: order})
+			} else {
+				return fmt.Errorf("mapping value '%s' not a string", v)
+			}
+			order += 1
+		}
+	}
+	return nil
+}
+
+type SubMapping struct {
+	Mapping KeyValues
+}
+
+type TypeMappings struct {
+	Points      KeyValues `yaml:"points"`
+	LineStrings KeyValues `yaml:"linestrings"`
+	Polygons    KeyValues `yaml:"polygons"`
+}

mapping/doc.go

@@ -1,4 +1,14 @@
 /*
-Package mapping provides functions for defining and executing the database schema.
+Package mapping provides implements mapping and convertion between OSM elements and database tables, rows and columns.
+
+The core logic of Imposm is accesible with the Mapping struct.
+A Mapping creates filters and matchers based on mapping configuration (.yaml or .json file).
+
+Filters are for initial filtering (during -read). They remove all tags that are not needed.
+
+Matchers map OSM elements to zero or more destination tables. Each Match results can convert an OSM element
+to a row with all mapped column values.
+The matching is dependend on the element type (node, way, relation), the element tags and the destination
+table type (point, linestring, polygon, relation, relation_member).
 */
 package mapping

mapping/filter.go

@@ -5,67 +5,96 @@ import (
 	"strings"
 
 	"github.com/omniscale/imposm3/element"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
+type TagFilterer interface {
+	Filter(tags *element.Tags)
+}
+
 func (m *Mapping) NodeTagFilter() TagFilterer {
-	if m.Tags.LoadAll {
+	if m.Conf.Tags.LoadAll {
-		return newExcludeFilter(m.Tags.Exclude)
+		return newExcludeFilter(m.Conf.Tags.Exclude)
 	}
-	mappings := make(map[Key]map[Value][]OrderedDestTable)
+	mappings := make(TagTableMapping)
-	m.mappings("point", mappings)
+	m.mappings(PointTable, mappings)
 	tags := make(map[Key]bool)
-	m.extraTags("point", tags)
+	m.extraTags(PointTable, tags)
-	return &TagFilter{mappings, tags}
+	m.extraTags(RelationMemberTable, tags)
+	return &tagFilter{mappings.asTagMap(), tags}
 }
 
 func (m *Mapping) WayTagFilter() TagFilterer {
-	if m.Tags.LoadAll {
+	if m.Conf.Tags.LoadAll {
-		return newExcludeFilter(m.Tags.Exclude)
+		return newExcludeFilter(m.Conf.Tags.Exclude)
 	}
-	mappings := make(map[Key]map[Value][]OrderedDestTable)
+	mappings := make(TagTableMapping)
-	m.mappings("linestring", mappings)
+	m.mappings(LineStringTable, mappings)
-	m.mappings("polygon", mappings)
+	m.mappings(PolygonTable, mappings)
 	tags := make(map[Key]bool)
-	m.extraTags("linestring", tags)
+	m.extraTags(LineStringTable, tags)
-	m.extraTags("polygon", tags)
+	m.extraTags(PolygonTable, tags)
-	return &TagFilter{mappings, tags}
+	m.extraTags(RelationMemberTable, tags)
+	return &tagFilter{mappings.asTagMap(), tags}
 }
 
 func (m *Mapping) RelationTagFilter() TagFilterer {
-	if m.Tags.LoadAll {
+	if m.Conf.Tags.LoadAll {
-		return newExcludeFilter(m.Tags.Exclude)
+		return newExcludeFilter(m.Conf.Tags.Exclude)
 	}
-	mappings := make(map[Key]map[Value][]OrderedDestTable)
+	mappings := make(TagTableMapping)
-	m.mappings("linestring", mappings)
+	// do not filter out type tag for common relations
-	m.mappings("polygon", mappings)
+	mappings["type"] = map[Value][]orderedDestTable{
+		"multipolygon": []orderedDestTable{},
+		"boundary":     []orderedDestTable{},
+		"land_area":    []orderedDestTable{},
+	}
+	m.mappings(LineStringTable, mappings)
+	m.mappings(PolygonTable, mappings)
+	m.mappings(RelationTable, mappings)
+	m.mappings(RelationMemberTable, mappings)
 	tags := make(map[Key]bool)
-	m.extraTags("linestring", tags)
+	m.extraTags(LineStringTable, tags)
-	m.extraTags("polygon", tags)
+	m.extraTags(PolygonTable, tags)
-	// do not filter out type tag
+	m.extraTags(RelationTable, tags)
-	mappings["type"] = map[Value][]OrderedDestTable{
+	m.extraTags(RelationMemberTable, tags)
-		"multipolygon": []OrderedDestTable{},
+	return &tagFilter{mappings.asTagMap(), tags}
-		"boundary":     []OrderedDestTable{},
-		"land_area":    []OrderedDestTable{},
-	}
-	return &RelationTagFilter{TagFilter{mappings, tags}}
 }
 
-type TagFilter struct {
+type tagMap map[Key]map[Value]struct{}
-	mappings  map[Key]map[Value][]OrderedDestTable
+
+type tagFilter struct {
+	mappings  tagMap
 	extraTags map[Key]bool
 }
 
-type RelationTagFilter struct {
+func (f *tagFilter) Filter(tags *element.Tags) {
-	TagFilter
+	if tags == nil {
+		return
+	}
+	for k, v := range *tags {
+		values, ok := f.mappings[Key(k)]
+		if ok {
+			if _, ok := values["__any__"]; ok {
+				continue
+			} else if _, ok := values[Value(v)]; ok {
+				continue
+			} else if _, ok := f.extraTags[Key(k)]; !ok {
+				delete(*tags, k)
+			}
+		} else if _, ok := f.extraTags[Key(k)]; !ok {
+			delete(*tags, k)
+		}
+	}
 }
 
-type ExcludeFilter struct {
+type excludeFilter struct {
 	keys    map[Key]struct{}
 	matches []string
 }
 
-func newExcludeFilter(tags []Key) *ExcludeFilter {
+func newExcludeFilter(tags []config.Key) *excludeFilter {
-	f := ExcludeFilter{
+	f := excludeFilter{
 		keys:    make(map[Key]struct{}),
 		matches: make([]string, 0),
 	}
@@ -73,14 +102,14 @@ func newExcludeFilter(tags []Key) *ExcludeFilter {
 		if strings.ContainsAny(string(t), "?*[") {
 			f.matches = append(f.matches, string(t))
 		} else {
-			f.keys[t] = struct{}{}
+			f.keys[Key(t)] = struct{}{}
 		}
 	}
 	return &f
 }
 
-func (f *ExcludeFilter) Filter(tags *element.Tags) bool {
+func (f *excludeFilter) Filter(tags *element.Tags) {
-	for k, _ := range *tags {
+	for k := range *tags {
 		if _, ok := f.keys[Key(k)]; ok {
 			delete(*tags, k)
 		} else if f.matches != nil {
@@ -92,86 +121,4 @@ func (f *ExcludeFilter) Filter(tags *element.Tags) bool {
 			}
 		}
 	}
-	return true
-}
-
-type TagFilterer interface {
-	Filter(tags *element.Tags) bool
-}
-
-func (f *TagFilter) Filter(tags *element.Tags) bool {
-	if tags == nil {
-		return false
-	}
-	foundMapping := false
-	for k, v := range *tags {
-		values, ok := f.mappings[Key(k)]
-		if ok {
-			if _, ok := values["__any__"]; ok {
-				foundMapping = true
-				continue
-			} else if _, ok := values[Value(v)]; ok {
-				foundMapping = true
-				continue
-			} else if _, ok := f.extraTags[Key(k)]; !ok {
-				delete(*tags, k)
-			}
-		} else if _, ok := f.extraTags[Key(k)]; !ok {
-			delete(*tags, k)
-		}
-	}
-	if foundMapping {
-		return true
-	} else {
-		*tags = nil
-		return false
-	}
-}
-
-func (f *RelationTagFilter) Filter(tags *element.Tags) bool {
-	if tags == nil {
-		return false
-	}
-
-	// TODO improve filtering for relation/relation_member mappings
-	// right now this only works with tags.load_all:true
-	if t, ok := (*tags)["type"]; ok {
-		if t != "multipolygon" && t != "boundary" && t != "land_area" {
-			*tags = nil
-			return false
-		}
-		if t == "boundary" {
-			if _, ok := (*tags)["boundary"]; !ok {
-				// a lot of the boundary relations are not multipolygon
-				// only import with boundary tags (e.g. boundary=administrative)
-				*tags = nil
-				return false
-			}
-		}
-	} else {
-		*tags = nil
-		return false
-	}
-	tagCount := len(*tags)
-	f.TagFilter.Filter(tags)
-
-	// we removed tags...
-	if len(*tags) < tagCount {
-		expectedTags := 0
-		if _, ok := (*tags)["name"]; ok {
-			expectedTags += 1
-		}
-		if _, ok := (*tags)["type"]; ok {
-			expectedTags += 1
-		}
-		if len(*tags) == expectedTags {
-			// but no tags except name and type are left
-			// remove all, otherwise tags from longest
-			// way/ring would be used during MP building
-			*tags = nil
-			return false
-		}
-	}
-	// always return true here since we found a matching type
-	return true
 }

mapping/filter_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/omniscale/imposm3/element"
+	"github.com/omniscale/imposm3/mapping/config"
 )
 
 var mapping *Mapping
@@ -17,28 +18,29 @@ func init() {
 	}
 }
 
-func stringMapEquals(t *testing.T, expected, actual map[string]string) {
+func stringMapEqual(expected, actual map[string]string) bool {
 	if len(expected) != len(actual) {
-		t.Fatalf("different length in %v and %v\n", expected, actual)
+		return false
 	}
 
 	for k, v := range expected {
 		if actualV, ok := actual[k]; ok {
 			if actualV != v {
-				t.Fatalf("%s != %s in %v and %v\n", v, actualV, expected, actual)
+				return false
 			}
 		} else {
-			t.Fatalf("%s not in %v\n", k, actual)
+			return false
 		}
 	}
+	return true
 }
 
-func matchesEqual(t *testing.T, expected []Match, actual []Match) {
+func matchesEqual(expected []Match, actual []Match) bool {
 	expectedMatches := make(map[DestTable]Match)
 	actualMatches := make(map[DestTable]Match)
 
 	if len(expected) != len(actual) {
-		t.Fatalf("different length in %v and %v\n", expected, actual)
+		return false
 	}
 
 	for _, match := range expected {
@@ -53,405 +55,293 @@ func matchesEqual(t *testing.T, expected []Match, actual []Match) {
 			if expectedMatch.Table != actualMatch.Table ||
 				expectedMatch.Key != actualMatch.Key ||
 				expectedMatch.Value != actualMatch.Value {
-				t.Fatalf("match differ %v != %v", expectedMatch, actualMatch)
+				return false
 			}
 		} else {
-			t.Fatalf("%s not in %v", name, actualMatches)
+			return false
+		}
+	}
+	return true
+}
+
+func TestTagFilterNodes(t *testing.T) {
+	tests := []struct {
+		tags     element.Tags
+		expected element.Tags
+	}{
+		{tags: element.Tags{}, expected: element.Tags{}},
+		{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "place": "unknown"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "place": "unknown", "population": "1000"}, expected: element.Tags{"name": "foo", "population": "1000"}},
+		{tags: element.Tags{"name": "foo", "place": "village"}, expected: element.Tags{"name": "foo", "place": "village"}},
+		{tags: element.Tags{"name": "foo", "place": "village", "population": "1000"}, expected: element.Tags{"name": "foo", "place": "village", "population": "1000"}},
+		{tags: element.Tags{"name": "foo", "place": "village", "unknown": "foo"}, expected: element.Tags{"name": "foo", "place": "village"}},
+		{tags: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, expected: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}},
+	}
+
+	nodes := mapping.NodeTagFilter()
+	for i, test := range tests {
+		nodes.Filter(&test.tags)
+		if !stringMapEqual(test.tags, test.expected) {
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
 		}
 	}
 }
 
-func TestTagFilterNodes(t *testing.T) {
-	var tags element.Tags
-	nodes := mapping.NodeTagFilter()
-
-	tags = element.Tags{"name": "foo"}
-	if nodes.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "unknown": "baz"}
-	if nodes.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "unknown"}
-	if nodes.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "village"}
-	if nodes.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "village", "population": "1000"}
-	if nodes.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "population": "1000"}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "village", "highway": "unknown"}
-	if nodes.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}
-	if nodes.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, tags)
-}
-
 func TestTagFilterWays(t *testing.T) {
-	var tags element.Tags
+	tests := []struct {
+		tags     element.Tags
+		expected element.Tags
+	}{
+		{tags: element.Tags{}, expected: element.Tags{}},
+		{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "highway": "unknown"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}},
+		{tags: element.Tags{"name": "foo", "building": "whatever"}, expected: element.Tags{"name": "foo", "building": "whatever"}},
+		{tags: element.Tags{"name": "foo", "highway": "track", "unknown": "foo"}, expected: element.Tags{"name": "foo", "highway": "track"}},
+		{tags: element.Tags{"name": "foo", "place": "village", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}},
+		{tags: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, expected: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}},
+	}
+
 	ways := mapping.WayTagFilter()
-
+	for i, test := range tests {
-	tags = element.Tags{"name": "foo"}
+		ways.Filter(&test.tags)
-	if ways.Filter(&tags) != false {
+		if !stringMapEqual(test.tags, test.expected) {
-		t.Fatal("unexpected filter response for", tags)
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
+		}
 	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "unknown": "baz"}
-	if ways.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "highway": "unknown"}
-	if ways.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "highway": "track"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags)
-
-	tags = element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, tags)
-
-	tags = element.Tags{"name": "foo", "place": "village", "highway": "track"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags)
-
-	tags = element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"}, tags)
-
-	// with __any__ value
-	tags = element.Tags{"name": "foo", "building": "yes"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "building": "yes"}, tags)
-
-	tags = element.Tags{"name": "foo", "building": "whatever"}
-	if ways.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "building": "whatever"}, tags)
 }
 
 func TestTagFilterRelations(t *testing.T) {
-	var tags element.Tags
+	tests := []struct {
+		tags     element.Tags
+		expected element.Tags
+	}{
+		{tags: element.Tags{}, expected: element.Tags{}},
+		{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "landuse": "unknown"}, expected: element.Tags{"name": "foo"}},
+		{tags: element.Tags{"name": "foo", "landuse": "farm"}, expected: element.Tags{"name": "foo", "landuse": "farm"}},
+		{tags: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}},
+		{tags: element.Tags{"name": "foo", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "type": "multipolygon"}},
+		{tags: element.Tags{"name": "foo", "type": "boundary"}, expected: element.Tags{"name": "foo", "type": "boundary"}},
+		{tags: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}},
+	}
+
 	relations := mapping.RelationTagFilter()
-
+	for i, test := range tests {
-	tags = element.Tags{"name": "foo"}
+		relations.Filter(&test.tags)
-	if relations.Filter(&tags) != false {
+		if !stringMapEqual(test.tags, test.expected) {
-		t.Fatal("unexpected filter response for", tags)
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
+		}
 	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "unknown": "baz"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "landuse": "unknown"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "landuse": "farm"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}
-	if relations.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, tags)
-
-	// skip multipolygon with filtered tags, otherwise tags from
-	// longest way would be used
-	tags = element.Tags{"name": "foo", "landuse": "unknown", "type": "multipolygon"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "landuse": "park", "type": "multipolygon"}
-	if relations.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "type": "multipolygon", "landuse": "park"}, tags)
-
-	tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"}
-	if relations.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"}, tags)
-
-	// boundary relation for boundary
-	tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"}
-	if relations.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"}, tags)
-
-	// boundary relation for non boundary
-	tags = element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	/* skip boundary with filtered tags, otherwise tags from longest way would
-	be used */
-	tags = element.Tags{"name": "foo", "boundary": "unknown", "type": "boundary"}
-	if relations.Filter(&tags) != false {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{}, tags)
-
-	tags = element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"}
-	if relations.Filter(&tags) != true {
-		t.Fatal("unexpected filter response for", tags)
-	}
-	stringMapEquals(t, element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"}, tags)
-
 }
 
 func TestPointMatcher(t *testing.T) {
-	elem := element.Node{}
+	tests := []struct {
-	points := mapping.PointMatcher()
+		tags    element.Tags
-
+		matches []Match
-	elem.Tags = element.Tags{"unknown": "baz"}
+	}{
-	matchesEqual(t, []Match{}, points.MatchNode(&elem))
+		{element.Tags{"unknown": "baz"}, []Match{}},
-
+		{element.Tags{"place": "unknown"}, []Match{}},
-	elem.Tags = element.Tags{"place": "unknown"}
+		{element.Tags{"place": "city"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}},
-	matchesEqual(t, []Match{}, points.MatchNode(&elem))
+		{element.Tags{"place": "city", "highway": "unknown"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}},
-
+		{element.Tags{"place": "city", "highway": "bus_stop"}, []Match{
-	elem.Tags = element.Tags{"place": "city"}
-	matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem))
-
-	elem.Tags = element.Tags{"place": "city", "highway": "unknown"}
-	matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem))
-
-	elem.Tags = element.Tags{"place": "city", "highway": "bus_stop"}
-	matchesEqual(t,
-		[]Match{
 			{"place", "city", DestTable{Name: "places"}, nil},
 			{"highway", "bus_stop", DestTable{Name: "transport_points"}, nil}},
-		points.MatchNode(&elem))
+		},
+	}
+
+	elem := element.Node{}
+	m := mapping.PointMatcher()
+	for i, test := range tests {
+		elem.Tags = test.tags
+		actual := m.MatchNode(&elem)
+		if !matchesEqual(actual, test.matches) {
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
+		}
+	}
 }
 
 func TestLineStringMatcher(t *testing.T) {
+	tests := []struct {
+		tags    element.Tags
+		matches []Match
+	}{
+		{element.Tags{"unknown": "baz"}, []Match{}},
+		{element.Tags{"highway": "unknown"}, []Match{}},
+		{element.Tags{"highway": "pedestrian"},
+			[]Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}},
+
+		// exclude_tags area=yes
+		{element.Tags{"highway": "pedestrian", "area": "yes"}, []Match{}},
+
+		{element.Tags{"barrier": "hedge"},
+			[]Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}},
+		{element.Tags{"barrier": "hedge", "area": "yes"}, []Match{}},
+
+		{element.Tags{"aeroway": "runway"}, []Match{}},
+		{element.Tags{"aeroway": "runway", "area": "no"},
+			[]Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}},
+
+		{element.Tags{"highway": "secondary", "railway": "tram"},
+			[]Match{
+				{"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil},
+				{"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}},
+		},
+		{element.Tags{"highway": "footway", "landuse": "park", "barrier": "hedge"},
+			// landusages not a linestring table
+			[]Match{
+				{"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil},
+				{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}},
+		},
+	}
+
 	elem := element.Way{}
 	// fake closed way for area matching
 	elem.Refs = []int64{1, 2, 3, 4, 1}
 	if !elem.IsClosed() {
 		t.Fatal("way not closed")
 	}
-	ls := mapping.LineStringMatcher()
+	m := mapping.LineStringMatcher()
-
+	for i, test := range tests {
-	elem.Tags = element.Tags{"unknown": "baz"}
+		elem.Tags = test.tags
-	matchesEqual(t, []Match{}, ls.MatchWay(&elem))
+		actual := m.MatchWay(&elem)
-
+		if !matchesEqual(actual, test.matches) {
-	elem.Tags = element.Tags{"highway": "unknown"}
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
-	matchesEqual(t, []Match{}, ls.MatchWay(&elem))
+		}
-
+	}
-	elem.Tags = element.Tags{"highway": "pedestrian"}
-	matchesEqual(t, []Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem))
-
-	// exclude_tags area=yes
-	elem.Tags = element.Tags{"highway": "pedestrian", "area": "yes"}
-	matchesEqual(t, []Match{}, ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"barrier": "hedge"}
-	matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}, ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"}
-	matchesEqual(t, []Match{}, ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"aeroway": "runway", "area": "no"}
-	matchesEqual(t, []Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}, ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"aeroway": "runway"}
-	matchesEqual(t, []Match{}, ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"highway": "secondary", "railway": "tram"}
-	matchesEqual(t,
-		[]Match{
-			{"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil},
-			{"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}},
-		ls.MatchWay(&elem))
-
-	elem.Tags = element.Tags{"highway": "footway", "landuse": "park"}
-	// landusages not a linestring table
-	matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem))
 }
 
-func TestPolygonMatcher(t *testing.T) {
+func TestPolygonMatcher_MatchWay(t *testing.T) {
-	elem := element.Relation{}
+	tests := []struct {
-	polys := mapping.PolygonMatcher()
+		tags    element.Tags
+		matches []Match
+	}{
+		{element.Tags{}, []Match{}},
+		{element.Tags{"unknown": "baz"}, []Match{}},
+		{element.Tags{"landuse": "unknown"}, []Match{}},
+		{element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}},
+		{element.Tags{"building": "yes"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}},
+		{element.Tags{"building": "residential"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}},
+		// line type requires area=yes
+		{element.Tags{"barrier": "hedge"}, []Match{}},
+		{element.Tags{"barrier": "hedge", "area": "yes"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}},
 
-	elem.Tags = element.Tags{"unknown": "baz"}
+		{element.Tags{"building": "shop"}, []Match{
-	matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
+			{"building", "shop", DestTable{Name: "buildings"}, nil},
+			{"building", "shop", DestTable{Name: "amenity_areas"}, nil},
+		}},
 
-	elem.Tags = element.Tags{"landuse": "unknowns"}
+		{element.Tags{"aeroway": "apron", "landuse": "farm"}, []Match{
-	matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"building": "yes"}
-	matchesEqual(t, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem))
-	elem.Tags = element.Tags{"building": "residential"}
-	matchesEqual(t, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"barrier": "hedge"}
-	matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"}
-	matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"building": "shop"}
-	matchesEqual(t, []Match{
-		{"building", "shop", DestTable{Name: "buildings"}, nil},
-		{"building", "shop", DestTable{Name: "amenity_areas"}, nil}},
-		polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"landuse": "farm"}
-	matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"landuse": "farm", "highway": "secondary"}
-	matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"landuse": "farm", "aeroway": "apron"}
-	matchesEqual(t,
-		[]Match{
 			{"aeroway", "apron", DestTable{Name: "transport_areas"}, nil},
-			{"landuse", "farm", DestTable{Name: "landusages"}, nil}},
+			{"landuse", "farm", DestTable{Name: "landusages"}, nil},
-		polys.MatchRelation(&elem))
+		}},
 
-	elem.Tags = element.Tags{"highway": "footway"} // linear by default
+		{element.Tags{"landuse": "farm", "highway": "secondary"}, []Match{
-	matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
+			{"landuse", "farm", DestTable{Name: "landusages"}, nil},
+		}},
 
-	elem.Tags = element.Tags{"highway": "footway", "area": "yes"}
+		{element.Tags{"highway": "footway"}, []Match{}},
-	matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
+		{element.Tags{"highway": "footway", "area": "yes"}, []Match{
+			{"highway", "footway", DestTable{Name: "landusages"}, nil},
+		}},
 
-	elem.Tags = element.Tags{"boundary": "administrative", "admin_level": "8"}
+		{element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}},
-	matchesEqual(t, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}, polys.MatchRelation(&elem))
+
+		/*
+			landusages mapping has the following order,
+			check that XxxMatcher always uses the first
+
+			amenity:
+			- university
+			landuse:
+			- forest
+			leisure:
+			- park
+			landuse:
+			- park
+		*/
+
+		{element.Tags{"landuse": "forest", "leisure": "park"}, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}},
+		{element.Tags{"landuse": "park", "leisure": "park"}, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}},
+		{element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"}, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}},
+	}
+
+	elem := element.Way{}
+	// fake closed way for area matching
+	elem.Refs = []int64{1, 2, 3, 4, 1}
+	if !elem.IsClosed() {
+		t.Fatal("way not closed")
+	}
+	m := mapping.PolygonMatcher()
+	for i, test := range tests {
+		elem.Tags = test.tags
+		actual := m.MatchWay(&elem)
+		if !matchesEqual(actual, test.matches) {
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
+		}
+	}
+
+	elem.Refs = nil
+	elem.Tags = element.Tags{"building": "yes"}
+	actual := m.MatchWay(&elem)
+	if !matchesEqual([]Match{}, actual) {
+		t.Error("open way matched as polygon")
+	}
 }
 
-func TestMatcherMappingOrder(t *testing.T) {
+func TestPolygonMatcher_MatchRelation(t *testing.T) {
+	// check that only relations with type=multipolygon/boundary are matched as polygon
+
+	tests := []struct {
+		tags    element.Tags
+		matches []Match
+	}{
+		{element.Tags{}, []Match{}},
+		{element.Tags{"unknown": "baz"}, []Match{}},
+		{element.Tags{"landuse": "unknown"}, []Match{}},
+		{element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}},
+		{element.Tags{"building": "yes"}, []Match{}},
+		{element.Tags{"building": "yes", "type": "multipolygon"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}},
+		{element.Tags{"building": "residential", "type": "multipolygon"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}},
+		// line type requires area=yes
+		{element.Tags{"barrier": "hedge", "type": "multipolygon"}, []Match{}},
+		{element.Tags{"barrier": "hedge", "area": "yes", "type": "multipolygon"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}},
+
+		{element.Tags{"building": "shop", "type": "multipolygon"}, []Match{
+			{"building", "shop", DestTable{Name: "buildings"}, nil},
+			{"building", "shop", DestTable{Name: "amenity_areas"}, nil},
+		}},
+
+		{element.Tags{"aeroway": "apron", "landuse": "farm", "type": "multipolygon"}, []Match{
+			{"aeroway", "apron", DestTable{Name: "transport_areas"}, nil},
+			{"landuse", "farm", DestTable{Name: "landusages"}, nil},
+		}},
+
+		{element.Tags{"landuse": "farm", "highway": "secondary", "type": "multipolygon"}, []Match{
+			{"landuse", "farm", DestTable{Name: "landusages"}, nil},
+		}},
+
+		{element.Tags{"highway": "footway", "type": "multipolygon"}, []Match{}},
+		{element.Tags{"highway": "footway", "area": "yes", "type": "multipolygon"}, []Match{
+			{"highway", "footway", DestTable{Name: "landusages"}, nil},
+		}},
+
+		{element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{}},
+		{element.Tags{"boundary": "administrative", "admin_level": "8", "type": "boundary"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}},
+	}
+
 	elem := element.Relation{}
-	polys := mapping.PolygonMatcher()
+	m := mapping.PolygonMatcher()
-
+	for i, test := range tests {
-	/*
+		elem.Tags = test.tags
-		landusages mapping has the following order,
+		actual := m.MatchRelation(&elem)
-		check that XxxMatcher always uses the first
+		if !matchesEqual(actual, test.matches) {
-
+			t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
-		amenity:
+		}
-		- university
-		landuse:
-		- forest
-		leisure:
-		- park
-		landuse:
-		- park
-	*/
-
-	elem.Tags = element.Tags{"landuse": "forest", "leisure": "park"}
-	matchesEqual(t, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"landuse": "park", "leisure": "park"}
-	matchesEqual(t, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-
-	elem.Tags = element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"}
-	matchesEqual(t, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
-}
-
-func TestFilterNodes(t *testing.T) {
-	var tags element.Tags
-
-	// test name only
-	tags = make(element.Tags)
-	tags["name"] = "foo"
-
-	points := mapping.NodeTagFilter()
-	if points.Filter(&tags) != false {
-		t.Fatal("Filter result not false")
-	}
-	if len(tags) != 0 {
-		t.Fatal("Filter result not empty")
-	}
-
-	// test name + unmapped tags
-	tags = make(element.Tags)
-	tags["name"] = "foo"
-	tags["boring"] = "true"
-
-	if points.Filter(&tags) != false {
-		t.Fatal("Filter result not false")
-	}
-	if len(tags) != 0 {
-		t.Fatal("Filter result not empty")
-	}
-
-	// test fields only, but no mapping
-	tags = make(element.Tags)
-	tags["population"] = "0"
-	tags["name"] = "foo"
-	tags["boring"] = "true"
-
-	if points.Filter(&tags) != false {
-		t.Fatal("Filter result true", tags)
-	}
-	if len(tags) != 0 {
-		t.Fatal("Filter result not empty", tags)
-	}
-
-	// ... not with mapped tag (place)
-	tags = make(element.Tags)
-	tags["population"] = "0"
-	tags["name"] = "foo"
-	tags["boring"] = "true"
-	tags["place"] = "village"
-
-	if points.Filter(&tags) != true {
-		t.Fatal("Filter result true", tags)
-	}
-	if len(tags) != 3 && tags["population"] == "0" && tags["name"] == "foo" && tags["place"] == "village" {
-		t.Fatal("Filter result not expected", tags)
 	}
 }
 
@@ -460,7 +350,7 @@ func TestExcludeFilter(t *testing.T) {
 	var tags element.Tags
 
 	// no matches
-	f = newExcludeFilter([]Key{})
+	f = newExcludeFilter([]config.Key{})
 	tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
 	f.Filter(&tags)
 	if !reflect.DeepEqual(tags, element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}) {
@@ -468,7 +358,7 @@ func TestExcludeFilter(t *testing.T) {
 	}
 
 	// match all
-	f = newExcludeFilter([]Key{"*"})
+	f = newExcludeFilter([]config.Key{"*"})
 	tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
 	f.Filter(&tags)
 	if !reflect.DeepEqual(tags, element.Tags{}) {
@@ -476,7 +366,7 @@ func TestExcludeFilter(t *testing.T) {
 	}
 
 	// fixed string and wildcard match
-	f = newExcludeFilter([]Key{"source", "tiger:*"})
+	f = newExcludeFilter([]config.Key{"source", "tiger:*"})
 	tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
 	f.Filter(&tags)
 	if !reflect.DeepEqual(tags, element.Tags{"source:foo": "1"}) {
@@ -495,9 +385,7 @@ func BenchmarkFilterNodes(b *testing.B) {
 		tags["boring"] = "true"
 
 		points := mapping.NodeTagFilter()
-		if points.Filter(&tags) != true {
+		points.Filter(&tags)
-			b.Fatal("Filter result true", tags)
-		}
 		if len(tags) != 2 && tags["population"] == "0" && tags["name"] == "foo" {
 			b.Fatal("Filter result not expected", tags)
 		}

mapping/mapping.go

@@ -0,0 +1,331 @@
+package mapping
+
+import (
+	"errors"
+	"io/ioutil"
+
+	"github.com/omniscale/imposm3/element"
+	"github.com/omniscale/imposm3/mapping/config"
+
+	"gopkg.in/yaml.v2"
+)
+
+type orderedDestTable struct {
+	DestTable
+	order int
+}
+
+type TagTableMapping map[Key]map[Value][]orderedDestTable
+
+func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) {
+	for key, vals := range mapping {
+		for _, v := range vals {
+			vals, ok := tt[Key(key)]
+			tbl := orderedDestTable{DestTable: table, order: v.Order}
+			if ok {
+				vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl)
+			} else {
+				tt[Key(key)] = make(map[Value][]orderedDestTable)
+				tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl)
+			}
+		}
+	}
+}
+
+func (tt TagTableMapping) asTagMap() tagMap {
+	result := make(tagMap)
+	for k, vals := range tt {
+		result[k] = make(map[Value]struct{})
+		for v := range vals {
+			result[k][v] = struct{}{}
+		}
+	}
+	return result
+}
+
+type DestTable struct {
+	Name       string
+	SubMapping string
+}
+
+type TableType string
+
+func (tt *TableType) UnmarshalJSON(data []byte) error {
+	switch string(data) {
+	case "":
+		return errors.New("missing table type")
+	case `"point"`:
+		*tt = PointTable
+	case `"linestring"`:
+		*tt = LineStringTable
+	case `"polygon"`:
+		*tt = PolygonTable
+	case `"geometry"`:
+		*tt = GeometryTable
+	case `"relation"`:
+		*tt = RelationTable
+	case `"relation_member"`:
+		*tt = RelationMemberTable
+	default:
+		return errors.New("unknown type " + string(data))
+	}
+	return nil
+}
+
+const (
+	PolygonTable        TableType = "polygon"
+	LineStringTable     TableType = "linestring"
+	PointTable          TableType = "point"
+	GeometryTable       TableType = "geometry"
+	RelationTable       TableType = "relation"
+	RelationMemberTable TableType = "relation_member"
+)
+
+type Mapping struct {
+	Conf config.Mapping
+}
+
+func NewMapping(filename string) (*Mapping, error) {
+	f, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	mapping := Mapping{}
+	err = yaml.Unmarshal(f, &mapping.Conf)
+	if err != nil {
+		return nil, err
+	}
+
+	err = mapping.prepare()
+	if err != nil {
+		return nil, err
+	}
+	return &mapping, nil
+}
+
+func (m *Mapping) prepare() error {
+	for name, t := range m.Conf.Tables {
+		t.Name = name
+		if t.OldFields != nil {
+			// todo deprecate 'fields'
+			t.Columns = t.OldFields
+		}
+	}
+
+	for name, t := range m.Conf.GeneralizedTables {
+		t.Name = name
+	}
+	return nil
+}
+
+func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
+	for name, t := range m.Conf.Tables {
+		if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
+			continue
+		}
+		mappings.addFromMapping(t.Mapping, DestTable{Name: name})
+
+		for subMappingName, subMapping := range t.Mappings {
+			mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
+		}
+
+		switch tableType {
+		case PointTable:
+			mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
+		case LineStringTable:
+			mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
+		case PolygonTable:
+			mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
+		}
+	}
+}
+
+func (m *Mapping) tables(tableType TableType) map[string]*rowBuilder {
+	result := make(map[string]*rowBuilder)
+	for name, t := range m.Conf.Tables {
+		if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable {
+			result[name] = makeRowBuilder(t)
+		}
+	}
+	return result
+}
+
+func makeRowBuilder(tbl *config.Table) *rowBuilder {
+	result := rowBuilder{}
+
+	for _, mappingColumn := range tbl.Columns {
+		column := valueBuilder{}
+		column.key = Key(mappingColumn.Key)
+
+		columnType := MakeColumnType(mappingColumn)
+		if columnType != nil {
+			column.colType = *columnType
+		} else {
+			log.Warn("unhandled type: ", mappingColumn.Type)
+		}
+		result.columns = append(result.columns, column)
+	}
+	return &result
+}
+
+func MakeColumnType(c *config.Column) *ColumnType {
+	if columnType, ok := AvailableColumnTypes[c.Type]; ok {
+		if columnType.MakeFunc != nil {
+			makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
+			if err != nil {
+				log.Print(err)
+				return nil
+			}
+			columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
+		}
+		columnType.FromMember = c.FromMember
+		return &columnType
+	}
+	return nil
+}
+
+func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
+	for _, t := range m.Conf.Tables {
+		if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable {
+			continue
+		}
+
+		for _, col := range t.Columns {
+			if col.Key != "" {
+				tags[Key(col.Key)] = true
+			}
+			for _, k := range col.Keys {
+				tags[Key(k)] = true
+			}
+		}
+
+		if t.Filters != nil && t.Filters.ExcludeTags != nil {
+			for _, keyVal := range *t.Filters.ExcludeTags {
+				tags[Key(keyVal[0])] = true
+			}
+		}
+
+		if tableType == PolygonTable || tableType == RelationTable || tableType == RelationMemberTable {
+			if t.RelationTypes != nil {
+				tags["type"] = true
+			}
+		}
+	}
+	for _, k := range m.Conf.Tags.Include {
+		tags[Key(k)] = true
+	}
+
+	// always include area tag for closed-way handling
+	tags["area"] = true
+}
+
+type elementFilter func(tags element.Tags, key Key, closed bool) bool
+
+type tableElementFilters map[string][]elementFilter
+
+func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) {
+	var areaTags map[Key]struct{}
+	var linearTags map[Key]struct{}
+	if m.Conf.Areas.AreaTags != nil {
+		areaTags = make(map[Key]struct{})
+		for _, tag := range m.Conf.Areas.AreaTags {
+			areaTags[Key(tag)] = struct{}{}
+		}
+	}
+	if m.Conf.Areas.LinearTags != nil {
+		linearTags = make(map[Key]struct{})
+		for _, tag := range m.Conf.Areas.LinearTags {
+			linearTags[Key(tag)] = struct{}{}
+		}
+	}
+
+	for name, t := range m.Conf.Tables {
+		if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
+			continue
+		}
+		if TableType(t.Type) == LineStringTable && areaTags != nil {
+			f := func(tags element.Tags, key Key, closed bool) bool {
+				if closed {
+					if tags["area"] == "yes" {
+						return false
+					}
+					if tags["area"] != "no" {
+						if _, ok := areaTags[key]; ok {
+							return false
+						}
+					}
+				}
+				return true
+			}
+			filters[name] = append(filters[name], f)
+		}
+		if TableType(t.Type) == PolygonTable && linearTags != nil {
+			f := func(tags element.Tags, key Key, closed bool) bool {
+				if closed && tags["area"] == "no" {
+					return false
+				}
+				if tags["area"] != "yes" {
+					if _, ok := linearTags[key]; ok {
+						return false
+					}
+				}
+				return true
+			}
+			filters[name] = append(filters[name], f)
+		}
+	}
+}
+
+func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
+	for name, t := range m.Conf.Tables {
+		if t.RelationTypes != nil {
+			relTypes := t.RelationTypes // copy loop var for closure
+			f := func(tags element.Tags, key Key, closed bool) bool {
+				if v, ok := tags["type"]; ok {
+					for _, rtype := range relTypes {
+						if v == rtype {
+							return true
+						}
+					}
+				}
+				return false
+			}
+			filters[name] = append(filters[name], f)
+		} else {
+			if TableType(t.Type) == PolygonTable {
+				// standard mulipolygon handling (boundary and land_area are for backwards compatibility)
+				f := func(tags element.Tags, key Key, closed bool) bool {
+					if v, ok := tags["type"]; ok {
+						if v == "multipolygon" || v == "boundary" || v == "land_area" {
+							return true
+						}
+					}
+					return false
+				}
+				filters[name] = append(filters[name], f)
+			}
+		}
+	}
+}
+
+func (m *Mapping) addFilters(filters tableElementFilters) {
+	for name, t := range m.Conf.Tables {
+		if t.Filters == nil {
+			continue
+		}
+		if t.Filters.ExcludeTags != nil {
+			for _, filterKeyVal := range *t.Filters.ExcludeTags {
+				f := func(tags element.Tags, key Key, closed bool) bool {
+					if v, ok := tags[filterKeyVal[0]]; ok {
+						if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
+							return false
+						}
+					}
+					return true
+				}
+				filters[name] = append(filters[name], f)
+			}
+		}
+	}
+}

mapping/matcher.go

@@ -6,9 +6,11 @@ import (
 )
 
 func (m *Mapping) PointMatcher() NodeMatcher {
-	mappings := make(TagTables)
+	mappings := make(TagTableMapping)
 	m.mappings(PointTable, mappings)
-	filters := m.ElementFilters()
+	filters := make(tableElementFilters)
+	m.addFilters(filters)
+	m.addTypedFilters(PointTable, filters)
 	return &tagMatcher{
 		mappings:   mappings,
 		tables:     m.tables(PointTable),
@@ -18,9 +20,11 @@ func (m *Mapping) PointMatcher() NodeMatcher {
 }
 
 func (m *Mapping) LineStringMatcher() WayMatcher {
-	mappings := make(TagTables)
+	mappings := make(TagTableMapping)
 	m.mappings(LineStringTable, mappings)
-	filters := m.ElementFilters()
+	filters := make(tableElementFilters)
+	m.addFilters(filters)
+	m.addTypedFilters(LineStringTable, filters)
 	return &tagMatcher{
 		mappings:   mappings,
 		tables:     m.tables(LineStringTable),
@@ -30,48 +34,57 @@ func (m *Mapping) LineStringMatcher() WayMatcher {
 }
 
 func (m *Mapping) PolygonMatcher() RelWayMatcher {
-	mappings := make(TagTables)
+	mappings := make(TagTableMapping)
 	m.mappings(PolygonTable, mappings)
-	filters := m.ElementFilters()
+	filters := make(tableElementFilters)
+	m.addFilters(filters)
+	m.addTypedFilters(PolygonTable, filters)
+	relFilters := make(tableElementFilters)
+	m.addRelationFilters(PolygonTable, relFilters)
 	return &tagMatcher{
 		mappings:   mappings,
 		tables:     m.tables(PolygonTable),
 		filters:    filters,
+		relFilters: relFilters,
 		matchAreas: true,
 	}
 }
 
 func (m *Mapping) RelationMatcher() RelationMatcher {
-	mappings := make(TagTables)
+	mappings := make(TagTableMapping)
 	m.mappings(RelationTable, mappings)
-	filters := m.ElementFilters()
+	filters := make(tableElementFilters)
+	m.addFilters(filters)
+	m.addTypedFilters(PolygonTable, filters)
+	m.addTypedFilters(RelationTable, filters)
+	relFilters := make(tableElementFilters)
+	m.addRelationFilters(RelationTable, relFilters)
 	return &tagMatcher{
 		mappings:   mappings,
 		tables:     m.tables(RelationTable),
 		filters:    filters,
+		relFilters: relFilters,
 		matchAreas: true,
 	}
 }
 
 func (m *Mapping) RelationMemberMatcher() RelationMatcher {
-	mappings := make(TagTables)
+	mappings := make(TagTableMapping)
 	m.mappings(RelationMemberTable, mappings)
-	filters := m.ElementFilters()
+	filters := make(tableElementFilters)
+	m.addFilters(filters)
+	m.addTypedFilters(RelationMemberTable, filters)
+	relFilters := make(tableElementFilters)
+	m.addRelationFilters(RelationMemberTable, relFilters)
 	return &tagMatcher{
 		mappings:   mappings,
 		tables:     m.tables(RelationMemberTable),
 		filters:    filters,
+		relFilters: relFilters,
 		matchAreas: true,
 	}
 }
 
-type Match struct {
-	Key         string
-	Value       string
-	Table       DestTable
-	tableFields *TableFields
-}
-
 type NodeMatcher interface {
 	MatchNode(node *element.Node) []Match
 }
@@ -89,23 +102,31 @@ type RelWayMatcher interface {
 	RelationMatcher
 }
 
-type tagMatcher struct {
+type Match struct {
-	mappings   TagTables
+	Key     string
-	tables     map[string]*TableFields
+	Value   string
-	filters    map[string][]ElementFilter
+	Table   DestTable
-	matchAreas bool
+	builder *rowBuilder
 }
 
 func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} {
-	return m.tableFields.MakeRow(elem, geom, *m)
+	return m.builder.MakeRow(elem, geom, *m)
 }
 
 func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry) []interface{} {
-	return m.tableFields.MakeMemberRow(rel, member, geom, *m)
+	return m.builder.MakeMemberRow(rel, member, geom, *m)
+}
+
+type tagMatcher struct {
+	mappings   TagTableMapping
+	tables     map[string]*rowBuilder
+	filters    tableElementFilters
+	relFilters tableElementFilters
+	matchAreas bool
 }
 
 func (tm *tagMatcher) MatchNode(node *element.Node) []Match {
-	return tm.match(node.Tags, false)
+	return tm.match(node.Tags, false, false)
 }
 
 func (tm *tagMatcher) MatchWay(way *element.Way) []Match {
@@ -114,22 +135,22 @@ func (tm *tagMatcher) MatchWay(way *element.Way) []Match {
 			if way.Tags["area"] == "no" {
 				return nil
 			}
-			return tm.match(way.Tags, true)
+			return tm.match(way.Tags, true, false)
 		}
 	} else { // match way as linestring
 		if way.IsClosed() {
 			if way.Tags["area"] == "yes" {
 				return nil
 			}
-			return tm.match(way.Tags, true)
+			return tm.match(way.Tags, true, false)
 		}
-		return tm.match(way.Tags, false)
+		return tm.match(way.Tags, false, false)
 	}
 	return nil
 }
 
 func (tm *tagMatcher) MatchRelation(rel *element.Relation) []Match {
-	return tm.match(rel.Tags, true)
+	return tm.match(rel.Tags, true, true)
 }
 
 type orderedMatch struct {
@@ -137,17 +158,17 @@ type orderedMatch struct {
 	order int
 }
 
-func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
+func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Match {
 	tables := make(map[DestTable]orderedMatch)
 
-	addTables := func(k, v string, tbls []OrderedDestTable) {
+	addTables := func(k, v string, tbls []orderedDestTable) {
 		for _, t := range tbls {
 			this := orderedMatch{
 				Match: Match{
-					Key:         k,
+					Key:     k,
-					Value:       v,
+					Value:   v,
-					Table:       t.DestTable,
+					Table:   t.DestTable,
-					tableFields: tm.tables[t.Name],
+					builder: tm.tables[t.Name],
 				},
 				order: t.order,
 			}
@@ -187,6 +208,18 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
 				}
 			}
 		}
+		if relation && !filteredOut {
+			filters, ok := tm.relFilters[t.Name]
+			if ok {
+				for _, filter := range filters {
+					if !filter(tags, Key(match.Key), closed) {
+						filteredOut = true
+						break
+					}
+				}
+			}
+		}
+
 		if !filteredOut {
 			matches = append(matches, match.Match)
 		}
@@ -194,6 +227,54 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
 	return matches
 }
 
+type valueBuilder struct {
+	key     Key
+	colType ColumnType
+}
+
+func (v *valueBuilder) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
+	if v.colType.Func != nil {
+		return v.colType.Func(elem.Tags[string(v.key)], elem, geom, match)
+	}
+	return nil
+}
+
+func (v *valueBuilder) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} {
+	if v.colType.Func != nil {
+		if v.colType.FromMember {
+			if member.Elem == nil {
+				return nil
+			}
+			return v.colType.Func(member.Elem.Tags[string(v.key)], member.Elem, geom, match)
+		}
+		return v.colType.Func(rel.Tags[string(v.key)], &rel.OSMElem, geom, match)
+	}
+	if v.colType.MemberFunc != nil {
+		return v.colType.MemberFunc(rel, member, match)
+	}
+	return nil
+}
+
+type rowBuilder struct {
+	columns []valueBuilder
+}
+
+func (r *rowBuilder) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} {
+	var row []interface{}
+	for _, column := range r.columns {
+		row = append(row, column.Value(elem, geom, match))
+	}
+	return row
+}
+
+func (r *rowBuilder) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} {
+	var row []interface{}
+	for _, column := range r.columns {
+		row = append(row, column.MemberValue(rel, member, geom, match))
+	}
+	return row
+}
+
 // SelectRelationPolygons returns a slice of all members that are already
 // imported as part of the relation.
 // Outer members are "imported" if they share the same destination table. Inner members

mapping/matcher_test.go

@@ -43,7 +43,7 @@ func TestSelectRelationPolygonsSimple(t *testing.T) {
 		t.Fatal(err)
 	}
 	r := element.Relation{}
-	r.Tags = element.Tags{"landuse": "park"}
+	r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
 	r.Members = []element.Member{
 		makeMember(0, element.Tags{"landuse": "forest"}),
 		makeMember(1, element.Tags{"landuse": "park"}),
@@ -68,7 +68,7 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
 		t.Fatal(err)
 	}
 	r := element.Relation{}
-	r.Tags = element.Tags{"landuse": "park"}
+	r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
 	r.Members = []element.Member{
 		makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}),
 		makeMember(1, element.Tags{"landuse": "forest"}),
@@ -91,7 +91,7 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) {
 		t.Fatal(err)
 	}
 	r := element.Relation{}
-	r.Tags = element.Tags{"landuse": "park"}
+	r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
 	r.Members = []element.Member{
 		makeMember(0, element.Tags{"landuse": "park"}),
 		makeMember(1, element.Tags{"natural": "forest"}),
@@ -117,7 +117,7 @@ func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
 		t.Fatal(err)
 	}
 	r := element.Relation{}
-	r.Tags = element.Tags{"landuse": "forest", "natural": "scrub"}
+	r.Tags = element.Tags{"landuse": "forest", "natural": "scrub", "type": "multipolygon"}
 	r.Members = []element.Member{
 		makeMember(0, element.Tags{"natural": "scrub"}),
 		makeMember(1, element.Tags{"landuse": "forest"}),
@@ -139,7 +139,7 @@ func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) {
 		t.Fatal(err)
 	}
 	r := element.Relation{}
-	r.Tags = element.Tags{"waterway": "riverbank"}
+	r.Tags = element.Tags{"waterway": "riverbank", "type": "multipolygon"}
 	r.Members = []element.Member{
 		makeMemberRole(0, element.Tags{"waterway": "riverbank", "natural": "water"}, "outer"),
 		makeMemberRole(1, element.Tags{"natural": "water"}, "inner"),

mapping/test_mapping.json

@@ -43,7 +43,7 @@
     },
     "tables": {
         "landusages": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -209,7 +209,7 @@
             }
         },
         "buildings": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -240,7 +240,7 @@
         },
         "amenity_areas": {
             "_comment": "for testing duplicate inserts with __any__ and exact match",
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -270,7 +270,7 @@
             }
         },
         "places": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -333,7 +333,7 @@
             }
         },
         "transport_areas": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -370,7 +370,7 @@
             }
         },
         "admin": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -405,7 +405,7 @@
             }
         },
         "aeroways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -436,7 +436,7 @@
             }
         },
         "waterways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -473,7 +473,7 @@
             }
         },
         "barrierways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -515,7 +515,7 @@
             }
         },
         "transport_points": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -566,7 +566,7 @@
             }
         },
         "amenities": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -603,7 +603,7 @@
             }
         },
         "barrierpoints": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -648,7 +648,7 @@
             }
         },
         "housenumbers_interpolated": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -698,7 +698,7 @@
             }
         },
         "roads": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -818,7 +818,7 @@
             }
         },
         "housenumbers": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",

mapping/test_mapping.yml

@@ -35,7 +35,7 @@ generalized_tables:
     tolerance: 50.0
 tables:
   admin:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -53,7 +53,7 @@ tables:
       - administrative
     type: polygon
   aeroways:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -69,7 +69,7 @@ tables:
       - taxiway
     type: linestring
   amenities:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -92,7 +92,7 @@ tables:
     type: point
   amenity_areas:
     _comment: for testing duplicate inserts with __any__ and exact match
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -107,7 +107,7 @@ tables:
       - shop
     type: polygon
   barrierpoints:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -137,7 +137,7 @@ tables:
       - stile
     type: point
   barrierways:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -164,7 +164,7 @@ tables:
       - wire_fence
     type: linestring
   buildings:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -179,7 +179,7 @@ tables:
       - __any__
     type: polygon
   housenumbers:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -203,7 +203,7 @@ tables:
       - __any__
     type: point
   housenumbers_interpolated:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -230,7 +230,7 @@ tables:
       - __any__
     type: linestring
   landusages:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -363,7 +363,7 @@ tables:
       - riverbank
     type: polygon
   places:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -404,7 +404,7 @@ tables:
       - locality
     type: point
   roads:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -524,7 +524,7 @@ tables:
           - groyne
     type: linestring
   transport_areas:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -545,7 +545,7 @@ tables:
       - platform
     type: polygon
   transport_points:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -577,7 +577,7 @@ tables:
       - subway_entrance
     type: point
   waterways:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

test/any_any_mapping.json

@@ -8,7 +8,7 @@
     },
     "tables": {
         "all": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id"
@@ -28,7 +28,7 @@
             }
         },
         "amenities": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id"

test/complete_db.osc

@@ -35,15 +35,6 @@
       <tag k="name" v="way 11001"/>
       <tag k="natural" v="water"/>
     </way>
-    <way id="12001" version="2" timestamp="2011-11-11T00:11:11Z">
-      <nd ref="12001"/>
-      <nd ref="12002"/>
-      <nd ref="12003"/>
-      <nd ref="12004"/>
-      <nd ref="12001"/>
-      <tag k="name" v="way 12001"/>
-      <tag k="natural" v="water"/>
-    </way>
     <relation id="13001" version="2" timestamp="2011-11-11T00:11:11Z">
       <member type="way" ref="13001" role="outer"/>
       <tag k="natural" v="water"/>
@@ -71,6 +62,7 @@
       <tag k="natural" v="water"/>
     </way>
     <relation id="14001" version="2" timestamp="2011-11-11T00:11:11Z">
+      <!-- now a new style relation -->
       <member type="way" ref="14001" role="outer"/>
       <member type="way" ref="14011" role="inner"/>
       <tag k="type" v="multipolygon"/>

test/complete_db.osm

@@ -267,8 +267,19 @@
   <relation id="1001" version="1" timestamp="2011-11-11T00:11:11Z">
     <member type="way" ref="1001" role="outer"/>
     <member type="way" ref="1002" role="inner"/>
+    <tag k="landuse" v="wood"/>
     <tag k="type" v="multipolygon"/>
   </relation>
+  <relation id="1011" version="1" timestamp="2011-11-11T00:11:11Z">
+    <member type="way" ref="1001" role="outer"/>
+    <member type="way" ref="1002" role="inner"/>
+    <tag k="type" v="multipolygon"/>
+  </relation>
+  <relation id="1021" version="1" timestamp="2011-11-11T00:11:11Z">
+    <member type="way" ref="1001" role="outer"/>
+    <member type="way" ref="1002" role="inner"/>
+    <tag k="landuse" v="wood"/>
+  </relation>
   <relation id="2001" version="1" timestamp="2011-11-11T00:11:11Z">
     <member type="way" ref="2001" role="outer"/>
     <member type="way" ref="2002" role="inner"/>
@@ -524,7 +535,6 @@
     <nd ref="9201"/>
     <nd ref="9202"/>
     <nd ref="9203"/>
-    <tag k="landuse" v="park"/>
     <tag k="highway" v="secondary"/>
     <tag k="name" v="9209"/>
   </way>
@@ -540,37 +550,10 @@
     <member type="way" ref="9209" role="outer"/>
     <member type="way" ref="9210" role="outer"/>
     <tag k="type" v="multipolygon"/>
-  </relation>
-
-  <!-- test multipolygon ways were inserted (same as 92xx, but different tagging) -->
-  <node id="9301" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
-  <node id="9302" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
-  <node id="9303" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="82"/>
-  <node id="9304" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="80"/>
-  <way id="9309" version="1" timestamp="2011-11-11T00:11:11Z">
-    <nd ref="9301"/>
-    <nd ref="9302"/>
-    <nd ref="9303"/>
-    <tag k="landuse" v="park"/>
-    <tag k="highway" v="secondary"/>
-    <tag k="name" v="9309"/>
-  </way>
-  <way id="9310" version="1" timestamp="2011-11-11T00:11:11Z">
-    <nd ref="9303"/>
-    <nd ref="9304"/>
-    <nd ref="9301"/>
-    <tag k="highway" v="residential"/>
-    <tag k="name" v="9310"/>
-  </way>
-  <relation id="9301" version="1" timestamp="2011-11-11T00:11:11Z">
-    <member type="way" ref="9309" role="outer"/>
-    <member type="way" ref="9310" role="outer"/>
-    <tag k="type" v="multipolygon"/>
     <tag k="landuse" v="park"/>
   </relation>
 
 
-
   <!-- test multipolygon way was inserted -->
   <node id="8001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
   <node id="8002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
@@ -606,25 +589,6 @@
     <tag k="landuse" v="park"/>
   </way>
 
-  <!-- test for changed tags in way belonging to relation -->
-  <node id="12001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="85"/>
-  <node id="12002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="86"/>
-  <node id="12003" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="86"/>
-  <node id="12004" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="85"/>
-  <way id="12001" version="1" timestamp="2011-11-11T00:11:11Z">
-    <nd ref="12001"/>
-    <nd ref="12002"/>
-    <nd ref="12003"/>
-    <nd ref="12004"/>
-    <nd ref="12001"/>
-    <tag k="name" v="way 12001"/>
-    <tag k="landuse" v="park"/>
-  </way>
-  <relation id="12001" version="1" timestamp="2011-11-11T00:11:11Z">
-    <member type="way" ref="12001" role="outer"/>
-    <tag k="type" v="multipolygon"/>
-  </relation>
-
   <!-- test for changed tags in relation -->
   <node id="13001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="87"/>
   <node id="13002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="88"/>
@@ -672,6 +636,7 @@
     <tag k="name" v="way 14011"/>
   </way>
   <relation id="14001" version="1" timestamp="2011-11-11T00:11:11Z">
+    <!-- old style relation not inserted -->
     <member type="way" ref="14001" role="outer"/>
     <member type="way" ref="14011" role="inner"/>
     <tag k="type" v="multipolygon"/>
@@ -694,7 +659,6 @@
     <nd ref="15004"/>
     <nd ref="15001"/>
     <tag k="name" v="way 15001"/>
-    <tag k="landuse" v="park"/>
   </way>
   <way id="15011" version="1" timestamp="2011-11-11T00:11:11Z">
     <nd ref="15011"/>
@@ -708,6 +672,7 @@
     <member type="way" ref="15001" role="outer"/>
     <member type="way" ref="15011" role="inner"/>
     <tag k="type" v="multipolygon"/>
+    <tag k="landuse" v="park"/>
   </relation>
 
 
@@ -947,7 +912,7 @@
     <tag k="landuse" v="park"/>
   </relation>
 
-  <!-- test removing of relation (r:50121) without tags -->
+  <!-- test old-style relation does not affect way -->
   <node id="50101" version="1" timestamp="2011-11-11T00:11:11Z" lat="42" lon="10"/>
   <node id="50102" version="1" timestamp="2011-11-11T00:11:11Z" lat="42" lon="11"/>
   <node id="50103" version="1" timestamp="2011-11-11T00:11:11Z" lat="44" lon="10"/>
@@ -1031,10 +996,10 @@
     <nd ref="52103"/>
     <nd ref="52104"/>
     <nd ref="52101"/>
-    <tag k="building" v="yes"/>
   </way>
   <relation id="52121" version="1" timestamp="2011-11-11T00:11:11Z">
     <member type="way" ref="52111" role="outer"/>
+    <tag k="building" v="yes"/>
     <tag k="type" v="multipolygon"/>
   </relation>
 

test/complete_db_mapping.json

@@ -54,7 +54,7 @@
     },
     "tables": {
         "landusages": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -191,7 +191,7 @@
             }
         },
         "buildings": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -228,7 +228,7 @@
             }
         },
         "places": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -290,7 +290,7 @@
             }
         },
         "transport_areas": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -327,7 +327,7 @@
             }
         },
         "admin": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -362,7 +362,7 @@
             }
         },
         "aeroways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -393,7 +393,7 @@
             }
         },
         "waterways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -430,7 +430,7 @@
             }
         },
         "barrierways": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -472,7 +472,7 @@
             }
         },
         "transport_points": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -523,7 +523,7 @@
             }
         },
         "amenities": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -560,7 +560,7 @@
             }
         },
         "barrierpoints": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -605,7 +605,7 @@
             }
         },
         "housenumbers_interpolated": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -655,7 +655,7 @@
             }
         },
         "roads": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -784,7 +784,7 @@
             }
         },
         "housenumbers": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",
@@ -829,7 +829,7 @@
             }
         },
         "waterareas": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",

test/completedb_test.go

@@ -61,37 +61,38 @@ func TestComplete_Deploy(t *testing.T) {
 	}
 }
 
+func TestComplete_OnlyNewStyleMultipolgon(t *testing.T) {
+	assertRecords(t, []checkElem{
+		{"osm_landusages", -1001, "wood", nil},
+		{"osm_landusages", -1011, Missing, nil},
+		{"osm_landusages", -1021, Missing, nil},
+	})
+}
+
 func TestComplete_LandusageToWaterarea1(t *testing.T) {
 	// Parks inserted into landusages
 	cache := ts.cache(t)
 	defer cache.Close()
 	assertCachedWay(t, cache, 11001)
-	assertCachedWay(t, cache, 12001)
 	assertCachedWay(t, cache, 13001)
 
 	assertRecords(t, []checkElem{
 		{"osm_waterareas", 11001, Missing, nil},
-		{"osm_waterareas", -12001, Missing, nil},
 		{"osm_waterareas", -13001, Missing, nil},
 
 		{"osm_waterareas_gen0", 11001, Missing, nil},
-		{"osm_waterareas_gen0", -12001, Missing, nil},
 		{"osm_waterareas_gen0", -13001, Missing, nil},
 
 		{"osm_waterareas_gen1", 11001, Missing, nil},
-		{"osm_waterareas_gen1", -12001, Missing, nil},
 		{"osm_waterareas_gen1", -13001, Missing, nil},
 
 		{"osm_landusages", 11001, "park", nil},
-		{"osm_landusages", -12001, "park", nil},
 		{"osm_landusages", -13001, "park", nil},
 
 		{"osm_landusages_gen0", 11001, "park", nil},
-		{"osm_landusages_gen0", -12001, "park", nil},
 		{"osm_landusages_gen0", -13001, "park", nil},
 
 		{"osm_landusages_gen1", 11001, "park", nil},
-		{"osm_landusages_gen1", -12001, "park", nil},
 		{"osm_landusages_gen1", -13001, "park", nil},
 	})
 }
@@ -106,7 +107,8 @@ func TestComplete_ChangedHoleTags1(t *testing.T) {
 	assertRecords(t, []checkElem{
 		{"osm_waterareas", 14011, Missing, nil},
 		{"osm_waterareas", -14011, Missing, nil},
-		{"osm_landusages", -14001, "park", nil},
+		{"osm_landusages", 14001, "park", nil},
+		{"osm_landusages", -14001, Missing, nil},
 	})
 }
 
@@ -218,19 +220,15 @@ func TestComplete_RelationWayNotInserted(t *testing.T) {
 func TestComplete_RelationWaysInserted(t *testing.T) {
 	// Outer ways of multipolygon are inserted.
 	assertRecords(t, []checkElem{
-		{"osm_landusages", -9201, "park", map[string]string{"name": "9209"}},
+		// no name on relation
+		{"osm_landusages", -9201, "park", map[string]string{"name": ""}},
 		{"osm_landusages", 9201, Missing, nil},
+		{"osm_landusages", 9209, Missing, nil},
+		{"osm_landusages", 9210, Missing, nil},
 		// outer ways of multipolygon stand for their own
 		{"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}},
 		{"osm_roads", 9210, "residential", map[string]string{"name": "9210"}},
-
-		// no name on relation
-		{"osm_landusages", -9301, "park", map[string]string{"name": ""}},
-		// outer ways of multipolygon stand for their own
-		{"osm_roads", 9309, "secondary", map[string]string{"name": "9309"}},
-		{"osm_roads", 9310, "residential", map[string]string{"name": "9310"}},
 	})
-
 }
 
 func TestComplete_RelationWayInserted(t *testing.T) {
@@ -283,12 +281,12 @@ func TestComplete_RelationBeforeRemove(t *testing.T) {
 	})
 }
 
-func TestComplete_RelationWithoutTags(t *testing.T) {
+func TestComplete_OldStyleRelationIsIgnored(t *testing.T) {
-	// Relation without tags is inserted.
+	// Relation without tags is not inserted.
 
 	assertRecords(t, []checkElem{
-		{"osm_buildings", 50111, Missing, nil},
+		{"osm_buildings", 50111, "yes", nil},
-		{"osm_buildings", -50121, "yes", nil},
+		{"osm_buildings", -50121, Missing, nil},
 	})
 }
 
@@ -483,27 +481,21 @@ func TestComplete_LandusageToWaterarea2(t *testing.T) {
 
 	assertRecords(t, []checkElem{
 		{"osm_waterareas", 11001, "water", nil},
-		{"osm_waterareas", -12001, "water", nil},
 		{"osm_waterareas", -13001, "water", nil},
 
 		{"osm_waterareas_gen0", 11001, "water", nil},
-		{"osm_waterareas_gen0", -12001, "water", nil},
 		{"osm_waterareas_gen0", -13001, "water", nil},
 
 		{"osm_waterareas_gen1", 11001, "water", nil},
-		{"osm_waterareas_gen1", -12001, "water", nil},
 		{"osm_waterareas_gen1", -13001, "water", nil},
 
 		{"osm_landusages", 11001, Missing, nil},
-		{"osm_landusages", -12001, Missing, nil},
 		{"osm_landusages", -13001, Missing, nil},
 
 		{"osm_landusages_gen0", 11001, Missing, nil},
-		{"osm_landusages_gen0", -12001, Missing, nil},
 		{"osm_landusages_gen0", -13001, Missing, nil},
 
 		{"osm_landusages_gen1", 11001, Missing, nil},
-		{"osm_landusages_gen1", -12001, Missing, nil},
 		{"osm_landusages_gen1", -13001, Missing, nil},
 	})
 }
@@ -518,6 +510,11 @@ func TestComplete_ChangedHoleTags2(t *testing.T) {
 
 	assertGeomArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779)
 	assertGeomArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182)
+
+	assertRecords(t, []checkElem{
+		{"osm_waterareas", -14011, Missing, nil},
+		{"osm_landusages", -14001, "park", nil},
+	})
 }
 
 func TestComplete_SplitOuterMultipolygonWay2(t *testing.T) {

test/expire_tiles_mapping.yml

@@ -1,7 +1,7 @@
 tables:
   roads:
     type: linestring
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: type
@@ -16,7 +16,7 @@ tables:
 
   pois:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: type
@@ -31,7 +31,7 @@ tables:
 
   buildings:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: type

test/route_relation.osm

@@ -153,5 +153,15 @@
   <tag k="type" v="route"/>
  </relation>
 
+<!-- non-route type is not imported -->
+ <node id="130101" version="1" timestamp="2015-12-31T23:59:99Z" lat="53.0" lon="8.200">
+    <tag k="name" v="Stop"/>
+ </node>
+
+ <relation id="130901" version="23" timestamp="2015-06-02T04:13:19Z">
+  <member type="node" ref="130101" role="stop"/>
+  <tag k="route" v="bus"/>
+  <tag k="type" v="bus_route"/> <!-- invalid type -->
+ </relation>
 
 </osm>

test/route_relation_mapping.yml

@@ -1,13 +1,7 @@
-tags:
-  load_all: true
-  exclude:
-  - created_by
-  - source
-
 tables:
   master_routes:
     type: relation_member
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: member
@@ -27,11 +21,12 @@ tables:
     - key: name
       name: name
       type: string
+    relation_types: [route_master]
     mapping:
       route_master: [bus]
   route_members:
     type: relation_member
-    fields:
+    columns:
     - name: osm_id
       type: id
     - key: ref
@@ -54,11 +49,12 @@ tables:
       key: name
       type: string
       from_member: true
+    relation_types: [route]
     mapping:
       route: [bus, tram, rail]
   routes:
     type: relation
-    fields:
+    columns:
     - name: osm_id
       type: id
     - key: ref
@@ -66,5 +62,7 @@ tables:
       type: string
     - name: tags
       type: hstore_tags
+    relation_types: [route, route_master]
     mapping:
+      route_master: [bus, tram, rail]
       route: [bus, tram, rail]

test/route_relation_test.go

@@ -58,6 +58,23 @@ func TestRouteRelation_RelationData(t *testing.T) {
 	if r.tags["name"] != "Bus 301: A => B" {
 		t.Error(r)
 	}
+
+	// check tags of master relation
+	r = ts.queryTags(t, "osm_routes", -100911)
+	if r.tags["name"] != "Bus 301" {
+		t.Error(r)
+	}
+}
+
+func TestRouteRelation_MemberUpdatedByNode1(t *testing.T) {
+	// check that member is updated after node was modified
+	rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101")
+	if len(rows) != 1 {
+		t.Fatal(rows)
+	}
+	if rows[0]["name"] != "Stop" {
+		t.Error(rows[0])
+	}
 }
 
 func TestRouteRelation_MemberGeomUpdated1(t *testing.T) {
@@ -131,7 +148,7 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) {
 
 }
 
-func TestRouteRelation_MemberUpdatedByNode(t *testing.T) {
+func TestRouteRelation_MemberUpdatedByNode2(t *testing.T) {
 	// check that member is updated after node was modified
 	rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101")
 	if len(rows) != 1 {

test/single_table_mapping.json

@@ -9,7 +9,7 @@
     "use_single_id_space": true,
     "tables": {
         "all": {
-            "fields": [
+            "columns": [
                 {
                     "type": "id",
                     "name": "osm_id",

test/single_table_test.go

@@ -159,7 +159,7 @@ func TestSingleTable_DuplicateIds1(t *testing.T) {
 	}
 
 	assertHstore(t, []checkElem{
-		{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
+		{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}},
 	})
 	assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
 }
@@ -185,7 +185,7 @@ func TestSingleTable_DuplicateIds2(t *testing.T) {
 	}
 
 	assertHstore(t, []checkElem{
-		{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
+		{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}},
 	})
 	assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
 }

update/process.go

@@ -119,7 +119,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
 		ProductionSchema: config.BaseOptions.Schemas.Production,
 		BackupSchema:     config.BaseOptions.Schemas.Backup,
 	}
-	db, err := database.Open(dbConf, tagmapping)
+	db, err := database.Open(dbConf, &tagmapping.Conf)
 	if err != nil {
 		return errors.New("database open: " + err.Error())
 	}
@@ -144,7 +144,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
 		delDb,
 		osmCache,
 		diffCache,
-		tagmapping.SingleIdSpace,
+		tagmapping.Conf.SingleIdSpace,
 		tagmapping.PointMatcher(),
 		tagmapping.LineStringMatcher(),
 		tagmapping.PolygonMatcher(),
@@ -162,7 +162,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
 	nodes := make(chan *element.Node)
 
 	relWriter := writer.NewRelationWriter(osmCache, diffCache,
-		tagmapping.SingleIdSpace,
+		tagmapping.Conf.SingleIdSpace,
 		relations,
 		db, progress,
 		tagmapping.PolygonMatcher(),
@@ -174,7 +174,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
 	relWriter.Start()
 
 	wayWriter := writer.NewWayWriter(osmCache, diffCache,
-		tagmapping.SingleIdSpace,
+		tagmapping.Conf.SingleIdSpace,
 		ways, db,
 		progress,
 		tagmapping.PolygonMatcher(),
@@ -350,11 +350,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
 		}
 		// insert new relation
 		progress.AddRelations(1)
-		// filter out unsupported relation types, otherwise they might
+		relations <- rel
-		// get inserted with the tags from an outer way
-		if relTagFilter.Filter(&rel.Tags) {
-			relations <- rel
-		}
 	}
 
 	for wayId, _ := range wayIds {

writer/relations.go

@@ -134,8 +134,12 @@ NextRel:
 }
 
 func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool {
-	// prepare relation first (build rings and compute actual
+	matches := rw.polygonMatcher.MatchRelation(r)
-	// relation tags)
+	if matches == nil {
+		return false
+	}
+
+	// prepare relation (build rings)
 	prepedRel, err := geomp.PrepareRelation(r, rw.srid, rw.maxGap)
 	if err != nil {
 		if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
@@ -144,12 +148,6 @@ func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geo
 		return false
 	}
 
-	// check for matches befor building the geometry
-	matches := rw.polygonMatcher.MatchRelation(r)
-	if matches == nil {
-		return false
-	}
-
 	// build the multipolygon
 	geom, err := prepedRel.Build()
 	if geom.Geom != nil {