vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #146 from omniscale/remove-old-style-multipolygon-support 

Remove old style multipolygon support
master
Oliver Tonnhofer 2017-05-15 11:21:24 +02:00 committed by GitHub
parent 42b147c118 b3c8e350ba
commit 119beee4e5
34 changed files with 1149 additions and 1241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
database/database.go
View File

@ -7,6 +7,7 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
type Config struct {
@ -67,17 +68,17 @@ type Optimizer interface {
Optimize() error
}
var databases map[string]func(Config, *mapping.Mapping) (DB, error)
var databases map[string]func(Config, *config.Mapping) (DB, error)
func init() {
databases = make(map[string]func(Config, *mapping.Mapping) (DB, error))
databases = make(map[string]func(Config, *config.Mapping) (DB, error))
}
func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) {
func Register(name string, f func(Config, *config.Mapping) (DB, error)) {
databases[name] = f
}
func Open(conf Config, m *mapping.Mapping) (DB, error) {
func Open(conf Config, m *config.Mapping) (DB, error) {
parts := strings.SplitN(conf.ConnectionParams, ":", 2)
connectionType := parts[0]
@ -108,7 +109,7 @@ func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geo
return nil
}
func newNullDb(conf Config, m *mapping.Mapping) (DB, error) {
func newNullDb(conf Config, m *config.Mapping) (DB, error) {
return &nullDb{}, nil
}

0
database/postgis/fields.go → database/postgis/columns.go
View File

3
database/postgis/postgis.go
View File

@ -14,6 +14,7 @@ import (
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/logging"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
var log = logging.NewLogger("PostGIS")
@ -599,7 +600,7 @@ func (pg *PostGIS) Close() error {
return pg.Db.Close()
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
func New(conf database.Config, m *config.Mapping) (database.DB, error) {
db := &PostGIS{}
db.Tables = make(map[string]*TableSpec)

24
database/postgis/spec.go
View File

@ -5,11 +5,12 @@ import (
"strings"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
type ColumnSpec struct {
Name string
FieldType mapping.FieldType
FieldType mapping.ColumnType
Type ColumnType
}
type TableSpec struct {
@ -124,12 +125,11 @@ func (spec *TableSpec) DeleteSQL() string {
)
}
func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
func NewTableSpec(pg *PostGIS, t *config.Table) *TableSpec {
var geomType string
switch t.Type {
case mapping.RelationMemberTable:
if mapping.TableType(t.Type) == mapping.RelationMemberTable {
geomType = "geometry"
default:
} else {
geomType = string(t.Type)
}
@ -140,23 +140,23 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
GeometryType: geomType,
Srid: pg.Config.Srid,
}
for _, field := range t.Fields {
fieldType := field.FieldType()
if fieldType == nil {
for _, column := range t.Columns {
columnType := mapping.MakeColumnType(column)
if columnType == nil {
continue
}
pgType, ok := pgTypes[fieldType.GoType]
pgType, ok := pgTypes[columnType.GoType]
if !ok {
log.Errorf("unhandled field type %v, using string type", fieldType)
log.Errorf("unhandled column type %v, using string type", columnType)
pgType = pgTypes["string"]
}
col := ColumnSpec{field.Name, *fieldType, pgType}
col := ColumnSpec{column.Name, *columnType, pgType}
spec.Columns = append(spec.Columns, col)
}
return &spec
}
func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
func NewGeneralizedTableSpec(pg *PostGIS, t *config.GeneralizedTable) *GeneralizedTableSpec {
spec := GeneralizedTableSpec{
Name: t.Name,
FullName: pg.Prefix + t.Name,

30
docs/mapping.rst
View File

@ -38,10 +38,28 @@ To import all polygons with `tourism=zoo`, `natural=wood` or `natural=land` into
…
``relation_types``
~~~~~~~~~~~~~~~~~~
``relation_types`` restricts which relation types should be imported. It is a list with `type` values, e.g. ``[route, master_route]``.
For tables of type ``relation`` and ``relation_member``: Only import relations which have this type value. You still need to have a mapping.
For tables of type ``polygon``: Only build multi-polygons for relations which have this type value. You still need to have a mapping. Defaults to ``[multipolygon, boundary, land_area]``.
.. code-block:: yaml
tables:
routes:
type: relation
relation_types: [route]
mapping:
route: [bus]
``columns``
~~~~~~~~~~~
``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionaly ``key``, ``args`` and ``from_member``.
``columns`` is a list of columns that Imposm should create for this table. Each column is a YAML object with a ``type`` and a ``name`` and optionally ``key``, ``args`` and ``from_member``.
``name``
^^^^^^^^^
@ -349,6 +367,16 @@ To load all tags except ``created_by``, ``source``, and ``tiger:county``, ``tige
To load specific data about amenities for inclusion into an `hstore_tags` column:
.. code-block:: yaml
tags:
include: [operator, opening_hours, wheelchair, website, phone, cuisine]
.. _Areas:
Areas

19
docs/relations.rst
View File

@ -42,21 +42,7 @@ It will also insert relations of the type ``multipolygon`` with a ``building`` t
The roles are ignored by Imposm as not all holes are correctly tagged as ``inner``. Imposm uses geometry operations to verify if a member of a multipolygon is a hole, or if it is a separate polygon.
For compatibility, multipolygon relations without tags will use the tags from the (longest) outer way. Imposm will insert the following relation as well::
<way id="18101" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="1001"/>
...
<nd ref="1001"/>
<tag k="building" v="yes"/>
</way>
<relation id="18901" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="18101" role="outer"/>
<member type="way" ref="18102" role="outer"/>
<tag k="type" v="multipolygon"/>
</relation>
Old-style multipolygon relations with tags on the outer way, instead of the relation are no longer supported.
Other relations
@ -72,7 +58,6 @@ These relations can not be mapped to `simple` linestrings or polygons as they ca
The Imposm table types ``relation`` and ``relation_member`` allow you to import all relevant data for these relations.
.. note:: ``relation`` and ``relation_member`` require :ref:`load_all<tags>` to have access to all keys.
``relation_member``
^^^^^^^^^^^^^^^^^^^
@ -109,6 +94,7 @@ You can use the following mapping::
- key: ref
name: ref
type: string
relation_type: [route]
mapping:
route: [bus]
@ -171,6 +157,7 @@ The following mapping imports the bus route relation from above::
- name: network
key: network
type: string
relation_type: [route]
mapping:
route: [bus]

26
geom/multipolygon.go
View File

@ -16,15 +16,12 @@ type PreparedRelation struct {
// PrepareRelation is the first step in building a (multi-)polygon of a Relation.
// It builds rings from all ways and returns an error if there are unclosed rings.
// It also merges the Relation.Tags with the Tags of the outer way.
func PrepareRelation(rel *element.Relation, srid int, maxRingGap float64) (PreparedRelation, error) {
rings, err := buildRings(rel, maxRingGap)
if err != nil {
return PreparedRelation{}, err
}
rel.Tags = relationTags(rel.Tags, rings[0].ways[0].Tags)
return PreparedRelation{rings, rel, srid}, nil
}
@ -224,29 +221,6 @@ func buildRelGeometry(g *geos.Geos, rel *element.Relation, rings []*ring) (*geos
return result, nil
}
func relationTags(relTags, wayTags element.Tags) element.Tags {
result := make(element.Tags)
for k, v := range relTags {
if k == "name" || k == "type" {
continue
}
result[k] = v
}
if len(result) == 0 {
// relation does not have tags? use way tags
for k, v := range wayTags {
result[k] = v
}
} else {
// add back name (if present)
if name, ok := relTags["name"]; ok {
result["name"] = name
}
}
return result
}
// ringIsHole returns true if rings[idx] is a hole, False if it is a
// shell (also if hole in a hole, etc)
func ringIsHole(rings []*ring, idx int) bool {

19
geom/multipolygon_test.go
View File

@ -94,7 +94,7 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) {
})
rel := element.Relation{
OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "rel"}}}
OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"name": "Relation", "natural": "forest", "type": "multipolygon"}}}
rel.Members = []element.Member{
{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@ -107,10 +107,11 @@ func TestMultiPolygonWithHoleAndRelName(t *testing.T) {
g := geos.NewGeos()
defer g.Finish()
if len(rel.Tags) != 2 {
if len(rel.Tags) != 3 {
t.Fatal("wrong rel tags", rel.Tags)
}
if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Blackwood" {
// name from way is ignored
if rel.Tags["natural"] != "forest" || rel.Tags["name"] != "Relation" {
t.Fatal("wrong rel tags", rel.Tags)
}
@ -147,7 +148,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) {
})
rel := element.Relation{
OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest"}}}
OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
rel.Members = []element.Member{
{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@ -161,7 +162,7 @@ func TestMultiPolygonWithMultipleHoles(t *testing.T) {
g := geos.NewGeos()
defer g.Finish()
if len(rel.Tags) != 1 {
if len(rel.Tags) != 2 {
t.Fatal("wrong rel tags", rel.Tags)
}
if rel.Tags["landusage"] != "forest" {
@ -214,7 +215,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) {
{1, 4, 4},
})
rel := element.Relation{OSMElem: element.OSMElem{Id: 1}}
rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
rel.Members = []element.Member{
{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@ -230,7 +231,7 @@ func TestMultiPolygonWithNeastedHoles(t *testing.T) {
g := geos.NewGeos()
defer g.Finish()
if len(rel.Tags) != 1 {
if len(rel.Tags) != 2 {
t.Fatal("wrong rel tags", rel.Tags)
}
if rel.Tags["landusage"] != "forest" {
@ -261,7 +262,7 @@ func TestPolygonFromThreeWays(t *testing.T) {
{1, 0, 0},
})
rel := element.Relation{OSMElem: element.OSMElem{Id: 1}}
rel := element.Relation{OSMElem: element.OSMElem{Id: 1, Tags: element.Tags{"landusage": "forest", "type": "multipolygon"}}}
rel.Members = []element.Member{
{Id: 1, Type: element.WAY, Role: "outer", Way: &w1},
{Id: 2, Type: element.WAY, Role: "inner", Way: &w2},
@ -275,7 +276,7 @@ func TestPolygonFromThreeWays(t *testing.T) {
g := geos.NewGeos()
defer g.Finish()
if len(rel.Tags) != 1 {
if len(rel.Tags) != 2 {
t.Fatal("wrong rel tags", rel.Tags)
}
if rel.Tags["landusage"] != "forest" {

6
import_/import.go
View File

@ -67,7 +67,7 @@ func Import() {
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err = database.Open(conf, tagmapping)
db, err = database.Open(conf, &tagmapping.Conf)
if err != nil {
log.Fatal(err)
}
@ -181,7 +181,7 @@ func Import() {
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
@ -196,7 +196,7 @@ func Import() {
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),

131
mapping/fields.go → mapping/columns.go
View File

@ -11,14 +11,15 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/logging"
"github.com/omniscale/imposm3/mapping/config"
)
var log = logging.NewLogger("mapping")
var AvailableFieldTypes map[string]FieldType
var AvailableColumnTypes map[string]ColumnType
func init() {
AvailableFieldTypes = map[string]FieldType{
AvailableColumnTypes = map[string]ColumnType{
"bool": {"bool", "bool", Bool, nil, nil, false},
"boolint": {"boolint", "int8", BoolInt, nil, nil, false},
"id": {"id", "int64", Id, nil, nil, false},
@ -47,94 +48,12 @@ func init() {
type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{}
type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{}
type MakeMakeValue func(string, FieldType, Field) (MakeValue, error)
type MakeMakeValue func(string, ColumnType, config.Column) (MakeValue, error)
type Key string
type Value string
type FieldSpec struct {
Key Key
Type FieldType
}
func (f *FieldSpec) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
if f.Type.Func != nil {
return f.Type.Func(elem.Tags[string(f.Key)], elem, geom, match)
}
return nil
}
func (f *FieldSpec) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} {
if f.Type.Func != nil {
if f.Type.FromMember {
if member.Elem == nil {
return nil
}
return f.Type.Func(member.Elem.Tags[string(f.Key)], member.Elem, geom, match)
}
return f.Type.Func(rel.Tags[string(f.Key)], &rel.OSMElem, geom, match)
}
if f.Type.MemberFunc != nil {
return f.Type.MemberFunc(rel, member, match)
}
return nil
}
type TableFields struct {
fields []FieldSpec
}
func (t *TableFields) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} {
var row []interface{}
for _, field := range t.fields {
row = append(row, field.Value(elem, geom, match))
}
return row
}
func (t *TableFields) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} {
var row []interface{}
for _, field := range t.fields {
row = append(row, field.MemberValue(rel, member, geom, match))
}
return row
}
func (field *Field) FieldType() *FieldType {
if fieldType, ok := AvailableFieldTypes[field.Type]; ok {
if fieldType.MakeFunc != nil {
makeValue, err := fieldType.MakeFunc(field.Name, fieldType, *field)
if err != nil {
log.Print(err)
return nil
}
fieldType = FieldType{fieldType.Name, fieldType.GoType, makeValue, nil, nil, fieldType.FromMember}
}
fieldType.FromMember = field.FromMember
return &fieldType
}
return nil
}
func (t *Table) TableFields() *TableFields {
result := TableFields{}
for _, mappingField := range t.Fields {
field := FieldSpec{}
field.Key = mappingField.Key
fieldType := mappingField.FieldType()
if fieldType != nil {
field.Type = *fieldType
} else {
log.Warn("unhandled type: ", mappingField.Type)
}
result.fields = append(result.fields, field)
}
return &result
}
type FieldType struct {
type ColumnType struct {
Name string
GoType string
Func MakeValue
@ -216,7 +135,7 @@ func Geometry(val string, elem *element.OSMElem, geom *geom.Geometry, match Matc
return string(geom.Wkb)
}
func MakePseudoArea(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
func MakePseudoArea(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
log.Print("warn: pseudoarea type is deprecated and will be removed. See area and webmercarea type.")
return Area, nil
}
@ -248,14 +167,14 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M
var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
func MakeHStoreString(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
var includeAll bool
var err error
var include map[string]int
if _, ok := field.Args["include"]; !ok {
if _, ok := column.Args["include"]; !ok {
includeAll = true
} else {
include, err = decodeEnumArg(field, "include")
include, err = decodeEnumArg(column, "include")
if err != nil {
return nil, err
}
@ -273,18 +192,18 @@ func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeV
return hstoreString, nil
}
func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
if _, ok := field.Args["ranks"]; !ok {
func MakeWayZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
if _, ok := column.Args["ranks"]; !ok {
return DefaultWayZOrder, nil
}
ranks, err := decodeEnumArg(field, "ranks")
ranks, err := decodeEnumArg(column, "ranks")
if err != nil {
return nil, err
}
levelOffset := len(ranks)
defaultRank := 0
if val, ok := field.Args["default"].(float64); ok {
if val, ok := column.Args["default"].(float64); ok {
defaultRank = int(val)
}
@ -361,9 +280,9 @@ func DefaultWayZOrder(val string, elem *element.OSMElem, geom *geom.Geometry, ma
return z
}
func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
func MakeZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
log.Print("warn: zorder type is deprecated and will be removed. See enumerate type.")
_rankList, ok := field.Args["ranks"]
_rankList, ok := column.Args["ranks"]
if !ok {
return nil, errors.New("missing ranks in args for zorder")
}
@ -374,7 +293,7 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue,
}
var key string
_key, ok := field.Args["key"]
_key, ok := column.Args["key"]
if ok {
key, ok = _key.(string)
if !ok {
@ -408,13 +327,13 @@ func MakeZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue,
return zOrder, nil
}
func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
values, err := decodeEnumArg(field, "values")
func MakeEnumerate(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
values, err := decodeEnumArg(column, "values")
if err != nil {
return nil, err
}
enumerate := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
if field.Key != "" {
if column.Key != "" {
if r, ok := values[val]; ok {
return r
}
@ -429,15 +348,15 @@ func MakeEnumerate(fieldName string, fieldType FieldType, field Field) (MakeValu
return enumerate, nil
}
func decodeEnumArg(field Field, key string) (map[string]int, error) {
_valuesList, ok := field.Args[key]
func decodeEnumArg(column config.Column, key string) (map[string]int, error) {
_valuesList, ok := column.Args[key]
if !ok {
return nil, fmt.Errorf("missing '%v' in args for %s", key, field.Type)
return nil, fmt.Errorf("missing '%v' in args for %s", key, column.Type)
}
valuesList, ok := _valuesList.([]interface{})
if !ok {
return nil, fmt.Errorf("'%v' in args for %s not a list", key, field.Type)
return nil, fmt.Errorf("'%v' in args for %s not a list", key, column.Type)
}
values := make(map[string]int)
@ -452,8 +371,8 @@ func decodeEnumArg(field Field, key string) (map[string]int, error) {
return values, nil
}
func MakeSuffixReplace(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
_changes, ok := field.Args["suffixes"]
func MakeSuffixReplace(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
_changes, ok := column.Args["suffixes"]
if !ok {
return nil, errors.New("missing suffixes in args for string_suffixreplace")
}

35
mapping/fields_test.go → mapping/columns_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/geom/geos"
"github.com/omniscale/imposm3/mapping/config"
)
func TestBool(t *testing.T) {
@ -74,8 +75,8 @@ func TestZOrder(t *testing.T) {
match := Match{}
zOrder, err := MakeZOrder("z_order",
AvailableFieldTypes["z_order"],
Field{
AvailableColumnTypes["z_order"],
config.Column{
Name: "z_order",
Key: "",
Type: "z_order",
@ -113,8 +114,8 @@ func TestEnumerate_Match(t *testing.T) {
// test enumerate by matched mapping key
zOrder, err := MakeEnumerate("enumerate",
AvailableFieldTypes["enumerate"],
Field{
AvailableColumnTypes["enumerate"],
config.Column{
Name: "enumerate",
Key: "",
Type: "enumerate",
@ -148,8 +149,8 @@ func TestEnumerate_Key(t *testing.T) {
// test enumerate by key
zOrder, err := MakeEnumerate("enumerate",
AvailableFieldTypes["enumerate"],
Field{
AvailableColumnTypes["enumerate"],
config.Column{
Name: "enumerate",
Key: "fips",
Type: "enumerate",
@ -182,8 +183,8 @@ func TestEnumerate_Key(t *testing.T) {
func TestWayZOrder(t *testing.T) {
zOrder, err := MakeWayZOrder("z_order",
AvailableFieldTypes["wayzorder"],
Field{
AvailableColumnTypes["wayzorder"],
config.Column{
Name: "zorder",
Type: "wayzorder",
Args: map[string]interface{}{
@ -233,7 +234,7 @@ func TestWayZOrder(t *testing.T) {
}
}
func TestAreaFields(t *testing.T) {
func TestAreaColumn(t *testing.T) {
tests := []struct {
wkt string
expected float32
@ -277,10 +278,10 @@ func TestAreaFields(t *testing.T) {
}
func TestMakeSuffixReplace(t *testing.T) {
field := Field{
column := config.Column{
Name: "name", Key: "name", Type: "string_suffixreplace",
Args: map[string]interface{}{"suffixes": map[interface{}]interface{}{"Straße": "Str.", "straße": "str."}}}
suffixReplace, err := MakeSuffixReplace("name", FieldType{}, field)
suffixReplace, err := MakeSuffixReplace("name", ColumnType{}, column)
if err != nil {
t.Fatal(err)
@ -298,27 +299,27 @@ func TestMakeSuffixReplace(t *testing.T) {
}
func TestHstoreString(t *testing.T) {
field := Field{
column := config.Column{
Name: "tags",
Type: "hstore_tags",
}
hstoreAll, err := MakeHStoreString("tags", FieldType{}, field)
hstoreAll, err := MakeHStoreString("tags", ColumnType{}, column)
if err != nil {
t.Fatal(err)
}
field = Field{
column = config.Column{
Name: "tags",
Type: "hstore_tags",
Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}},
}
hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field)
hstoreInclude, err := MakeHStoreString("tags", ColumnType{}, column)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
field MakeValue
column MakeValue
tags element.Tags
expected interface{}
}{
@ -331,7 +332,7 @@ func TestHstoreString(t *testing.T) {
{hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`},
{hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`},
} {
actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{})
actual := test.column("", &element.OSMElem{Tags: test.tags}, nil, Match{})
if actual.(string) != test.expected {
t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags)
}

347
mapping/config.go
View File

@ -1,347 +0,0 @@
package mapping
import (
"errors"
"fmt"
"io/ioutil"
"github.com/omniscale/imposm3/element"
"gopkg.in/yaml.v2"
)
type Field struct {
Name string `yaml:"name"`
Key Key `yaml:"key"`
Keys []Key `yaml:"keys"`
Type string `yaml:"type"`
Args map[string]interface{} `yaml:"args"`
FromMember bool `yaml:"from_member"`
}
type Table struct {
Name string
Type TableType `yaml:"type"`
Mapping KeyValues `yaml:"mapping"`
Mappings map[string]SubMapping `yaml:"mappings"`
TypeMappings TypeMappings `yaml:"type_mappings"`
Fields []*Field `yaml:"columns"` // TODO rename Fields internaly to Columns
OldFields []*Field `yaml:"fields"`
Filters *Filters `yaml:"filters"`
}
type GeneralizedTable struct {
Name string
SourceTableName string `yaml:"source"`
Tolerance float64 `yaml:"tolerance"`
SqlFilter string `yaml:"sql_filter"`
}
type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Tables map[string]*Table
type GeneralizedTables map[string]*GeneralizedTable
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Areas struct {
AreaTags []Key `yaml:"area_tags"`
LinearTags []Key `yaml:"linear_tags"`
}
type Tags struct {
LoadAll bool `yaml:"load_all"`
Exclude []Key `yaml:"exclude"`
Include []Key `yaml:"include"`
}
type orderedValue struct {
value Value
order int
}
type KeyValues map[Key][]orderedValue
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
if *kv == nil {
*kv = make(map[Key][]orderedValue)
}
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
order := 0
for _, item := range slice {
k, ok := item.Key.(string)
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
values, ok := item.Value.([]interface{})
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
for _, v := range values {
if v, ok := v.(string); ok {
(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order})
} else {
return fmt.Errorf("mapping value '%s' not a string", v)
}
order += 1
}
}
return nil
}
type SubMapping struct {
Mapping KeyValues
}
type TypeMappings struct {
Points KeyValues `yaml:"points"`
LineStrings KeyValues `yaml:"linestrings"`
Polygons KeyValues `yaml:"polygons"`
}
type ElementFilter func(tags element.Tags, key Key, closed bool) bool
type TagTables map[Key]map[Value][]OrderedDestTable
type DestTable struct {
Name string
SubMapping string
}
type OrderedDestTable struct {
DestTable
order int
}
type TableType string
func (tt *TableType) UnmarshalJSON(data []byte) error {
switch string(data) {
case "":
return errors.New("missing table type")
case `"point"`:
*tt = PointTable
case `"linestring"`:
*tt = LineStringTable
case `"polygon"`:
*tt = PolygonTable
case `"geometry"`:
*tt = GeometryTable
case `"relation"`:
*tt = RelationTable
case `"relation_member"`:
*tt = RelationMemberTable
default:
return errors.New("unknown type " + string(data))
}
return nil
}
const (
PolygonTable TableType = "polygon"
LineStringTable TableType = "linestring"
PointTable TableType = "point"
GeometryTable TableType = "geometry"
RelationTable TableType = "relation"
RelationMemberTable TableType = "relation_member"
)
func NewMapping(filename string) (*Mapping, error) {
f, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
mapping := Mapping{}
err = yaml.Unmarshal(f, &mapping)
if err != nil {
return nil, err
}
err = mapping.prepare()
if err != nil {
return nil, err
}
return &mapping, nil
}
func (t *Table) ExtraTags() map[Key]bool {
tags := make(map[Key]bool)
for _, field := range t.Fields {
if field.Key != "" {
tags[field.Key] = true
}
for _, k := range field.Keys {
tags[k] = true
}
}
return tags
}
func (m *Mapping) prepare() error {
for name, t := range m.Tables {
t.Name = name
if t.OldFields != nil {
// todo deprecate 'fields'
t.Fields = t.OldFields
}
}
for name, t := range m.GeneralizedTables {
t.Name = name
}
return nil
}
func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) {
for key, vals := range mapping {
for _, v := range vals {
vals, ok := tt[key]
tbl := OrderedDestTable{DestTable: table, order: v.order}
if ok {
vals[v.value] = append(vals[v.value], tbl)
} else {
tt[key] = make(map[Value][]OrderedDestTable)
tt[key][v.value] = append(tt[key][v.value], tbl)
}
}
}
}
func (m *Mapping) mappings(tableType TableType, mappings TagTables) {
for name, t := range m.Tables {
if t.Type != GeometryTable && t.Type != tableType {
continue
}
mappings.addFromMapping(t.Mapping, DestTable{Name: name})
for subMappingName, subMapping := range t.Mappings {
mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
}
switch tableType {
case PointTable:
mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
case LineStringTable:
mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
case PolygonTable:
mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
}
}
}
func (m *Mapping) tables(tableType TableType) map[string]*TableFields {
result := make(map[string]*TableFields)
for name, t := range m.Tables {
if t.Type == tableType || t.Type == "geometry" {
result[name] = t.TableFields()
}
}
return result
}
func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
for _, t := range m.Tables {
if t.Type != tableType && t.Type != "geometry" {
continue
}
for key, _ := range t.ExtraTags() {
tags[key] = true
}
if t.Filters != nil && t.Filters.ExcludeTags != nil {
for _, keyVal := range *t.Filters.ExcludeTags {
tags[Key(keyVal[0])] = true
}
}
}
for _, k := range m.Tags.Include {
tags[k] = true
}
// always include area tag for closed-way handling
tags["area"] = true
}
func (m *Mapping) ElementFilters() map[string][]ElementFilter {
result := make(map[string][]ElementFilter)
var areaTags map[Key]struct{}
var linearTags map[Key]struct{}
if m.Areas.AreaTags != nil {
areaTags = make(map[Key]struct{})
for _, tag := range m.Areas.AreaTags {
areaTags[tag] = struct{}{}
}
}
if m.Areas.LinearTags != nil {
linearTags = make(map[Key]struct{})
for _, tag := range m.Areas.LinearTags {
linearTags[tag] = struct{}{}
}
}
for name, t := range m.Tables {
if t.Type == LineStringTable && areaTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed {
if tags["area"] == "yes" {
return false
}
if tags["area"] != "no" {
if _, ok := areaTags[key]; ok {
return false
}
}
}
return true
}
result[name] = append(result[name], f)
}
if t.Type == PolygonTable && linearTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed && tags["area"] == "no" {
return false
}
if tags["area"] != "yes" {
if _, ok := linearTags[key]; ok {
return false
}
}
return true
}
result[name] = append(result[name], f)
}
if t.Filters == nil {
continue
}
if t.Filters.ExcludeTags != nil {
for _, filterKeyVal := range *t.Filters.ExcludeTags {
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[filterKeyVal[0]]; ok {
if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
return false
}
}
return true
}
result[name] = append(result[name], f)
}
}
}
return result
}

113
mapping/config/config.go Normal file
View File

@ -0,0 +1,113 @@
package config
import (
"fmt"
"gopkg.in/yaml.v2"
)
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Column struct {
Name string `yaml:"name"`
Key Key `yaml:"key"`
Keys []Key `yaml:"keys"`
Type string `yaml:"type"`
Args map[string]interface{} `yaml:"args"`
FromMember bool `yaml:"from_member"`
}
type Tables map[string]*Table
type Table struct {
Name string
Type string `yaml:"type"`
Mapping KeyValues `yaml:"mapping"`
Mappings map[string]SubMapping `yaml:"mappings"`
TypeMappings TypeMappings `yaml:"type_mappings"`
Columns []*Column `yaml:"columns"`
OldFields []*Column `yaml:"fields"`
Filters *Filters `yaml:"filters"`
RelationTypes []string `yaml:"relation_types"`
}
type GeneralizedTables map[string]*GeneralizedTable
type GeneralizedTable struct {
Name string
SourceTableName string `yaml:"source"`
Tolerance float64 `yaml:"tolerance"`
SqlFilter string `yaml:"sql_filter"`
}
type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Areas struct {
AreaTags []Key `yaml:"area_tags"`
LinearTags []Key `yaml:"linear_tags"`
}
type Tags struct {
LoadAll bool `yaml:"load_all"`
Exclude []Key `yaml:"exclude"`
Include []Key `yaml:"include"`
}
type Key string
type Value string
type orderedValue struct {
Value Value
Order int
}
type KeyValues map[Key][]orderedValue
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
if *kv == nil {
*kv = make(map[Key][]orderedValue)
}
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
order := 0
for _, item := range slice {
k, ok := item.Key.(string)
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
values, ok := item.Value.([]interface{})
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
for _, v := range values {
if v, ok := v.(string); ok {
(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{Value: Value(v), Order: order})
} else {
return fmt.Errorf("mapping value '%s' not a string", v)
}
order += 1
}
}
return nil
}
type SubMapping struct {
Mapping KeyValues
}
type TypeMappings struct {
Points KeyValues `yaml:"points"`
LineStrings KeyValues `yaml:"linestrings"`
Polygons KeyValues `yaml:"polygons"`
}

12
mapping/doc.go
View File

@ -1,4 +1,14 @@
/*
Package mapping provides functions for defining and executing the database schema.
Package mapping provides implements mapping and convertion between OSM elements and database tables, rows and columns.
The core logic of Imposm is accesible with the Mapping struct.
A Mapping creates filters and matchers based on mapping configuration (.yaml or .json file).
Filters are for initial filtering (during -read). They remove all tags that are not needed.
Matchers map OSM elements to zero or more destination tables. Each Match results can convert an OSM element
to a row with all mapped column values.
The matching is dependend on the element type (node, way, relation), the element tags and the destination
table type (point, linestring, polygon, relation, relation_member).
*/
package mapping

187
mapping/filter.go
View File

@ -5,67 +5,96 @@ import (
"strings"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
)
type TagFilterer interface {
Filter(tags *element.Tags)
}
func (m *Mapping) NodeTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
m.mappings("point", mappings)
mappings := make(TagTableMapping)
m.mappings(PointTable, mappings)
tags := make(map[Key]bool)
m.extraTags("point", tags)
return &TagFilter{mappings, tags}
m.extraTags(PointTable, tags)
m.extraTags(RelationMemberTable, tags)
return &tagFilter{mappings.asTagMap(), tags}
}
func (m *Mapping) WayTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
m.mappings("linestring", mappings)
m.mappings("polygon", mappings)
mappings := make(TagTableMapping)
m.mappings(LineStringTable, mappings)
m.mappings(PolygonTable, mappings)
tags := make(map[Key]bool)
m.extraTags("linestring", tags)
m.extraTags("polygon", tags)
return &TagFilter{mappings, tags}
m.extraTags(LineStringTable, tags)
m.extraTags(PolygonTable, tags)
m.extraTags(RelationMemberTable, tags)
return &tagFilter{mappings.asTagMap(), tags}
}
func (m *Mapping) RelationTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
m.mappings("linestring", mappings)
m.mappings("polygon", mappings)
mappings := make(TagTableMapping)
// do not filter out type tag for common relations
mappings["type"] = map[Value][]orderedDestTable{
"multipolygon": []orderedDestTable{},
"boundary": []orderedDestTable{},
"land_area": []orderedDestTable{},
}
m.mappings(LineStringTable, mappings)
m.mappings(PolygonTable, mappings)
m.mappings(RelationTable, mappings)
m.mappings(RelationMemberTable, mappings)
tags := make(map[Key]bool)
m.extraTags("linestring", tags)
m.extraTags("polygon", tags)
// do not filter out type tag
mappings["type"] = map[Value][]OrderedDestTable{
"multipolygon": []OrderedDestTable{},
"boundary": []OrderedDestTable{},
"land_area": []OrderedDestTable{},
}
return &RelationTagFilter{TagFilter{mappings, tags}}
m.extraTags(LineStringTable, tags)
m.extraTags(PolygonTable, tags)
m.extraTags(RelationTable, tags)
m.extraTags(RelationMemberTable, tags)
return &tagFilter{mappings.asTagMap(), tags}
}
type TagFilter struct {
mappings map[Key]map[Value][]OrderedDestTable
type tagMap map[Key]map[Value]struct{}
type tagFilter struct {
mappings tagMap
extraTags map[Key]bool
}
type RelationTagFilter struct {
TagFilter
func (f *tagFilter) Filter(tags *element.Tags) {
if tags == nil {
return
}
for k, v := range *tags {
values, ok := f.mappings[Key(k)]
if ok {
if _, ok := values["__any__"]; ok {
continue
} else if _, ok := values[Value(v)]; ok {
continue
} else if _, ok := f.extraTags[Key(k)]; !ok {
delete(*tags, k)
}
} else if _, ok := f.extraTags[Key(k)]; !ok {
delete(*tags, k)
}
}
}
type ExcludeFilter struct {
type excludeFilter struct {
keys map[Key]struct{}
matches []string
}
func newExcludeFilter(tags []Key) *ExcludeFilter {
f := ExcludeFilter{
func newExcludeFilter(tags []config.Key) *excludeFilter {
f := excludeFilter{
keys: make(map[Key]struct{}),
matches: make([]string, 0),
}
@ -73,14 +102,14 @@ func newExcludeFilter(tags []Key) *ExcludeFilter {
if strings.ContainsAny(string(t), "?*[") {
f.matches = append(f.matches, string(t))
} else {
f.keys[t] = struct{}{}
f.keys[Key(t)] = struct{}{}
}
}
return &f
}
func (f *ExcludeFilter) Filter(tags *element.Tags) bool {
for k, _ := range *tags {
func (f *excludeFilter) Filter(tags *element.Tags) {
for k := range *tags {
if _, ok := f.keys[Key(k)]; ok {
delete(*tags, k)
} else if f.matches != nil {
@ -92,86 +121,4 @@ func (f *ExcludeFilter) Filter(tags *element.Tags) bool {
}
}
}
return true
}
type TagFilterer interface {
Filter(tags *element.Tags) bool
}
func (f *TagFilter) Filter(tags *element.Tags) bool {
if tags == nil {
return false
}
foundMapping := false
for k, v := range *tags {
values, ok := f.mappings[Key(k)]
if ok {
if _, ok := values["__any__"]; ok {
foundMapping = true
continue
} else if _, ok := values[Value(v)]; ok {
foundMapping = true
continue
} else if _, ok := f.extraTags[Key(k)]; !ok {
delete(*tags, k)
}
} else if _, ok := f.extraTags[Key(k)]; !ok {
delete(*tags, k)
}
}
if foundMapping {
return true
} else {
*tags = nil
return false
}
}
func (f *RelationTagFilter) Filter(tags *element.Tags) bool {
if tags == nil {
return false
}
// TODO improve filtering for relation/relation_member mappings
// right now this only works with tags.load_all:true
if t, ok := (*tags)["type"]; ok {
if t != "multipolygon" && t != "boundary" && t != "land_area" {
*tags = nil
return false
}
if t == "boundary" {
if _, ok := (*tags)["boundary"]; !ok {
// a lot of the boundary relations are not multipolygon
// only import with boundary tags (e.g. boundary=administrative)
*tags = nil
return false
}
}
} else {
*tags = nil
return false
}
tagCount := len(*tags)
f.TagFilter.Filter(tags)
// we removed tags...
if len(*tags) < tagCount {
expectedTags := 0
if _, ok := (*tags)["name"]; ok {
expectedTags += 1
}
if _, ok := (*tags)["type"]; ok {
expectedTags += 1
}
if len(*tags) == expectedTags {
// but no tags except name and type are left
// remove all, otherwise tags from longest
// way/ring would be used during MP building
*tags = nil
return false
}
}
// always return true here since we found a matching type
return true
}

632
mapping/filter_test.go
View File

@ -5,6 +5,7 @@ import (
"testing"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
)
var mapping *Mapping
@ -17,28 +18,29 @@ func init() {
}
}
func stringMapEquals(t *testing.T, expected, actual map[string]string) {
func stringMapEqual(expected, actual map[string]string) bool {
if len(expected) != len(actual) {
t.Fatalf("different length in %v and %v\n", expected, actual)
return false
}
for k, v := range expected {
if actualV, ok := actual[k]; ok {
if actualV != v {
t.Fatalf("%s != %s in %v and %v\n", v, actualV, expected, actual)
return false
}
} else {
t.Fatalf("%s not in %v\n", k, actual)
return false
}
}
return true
}
func matchesEqual(t *testing.T, expected []Match, actual []Match) {
func matchesEqual(expected []Match, actual []Match) bool {
expectedMatches := make(map[DestTable]Match)
actualMatches := make(map[DestTable]Match)
if len(expected) != len(actual) {
t.Fatalf("different length in %v and %v\n", expected, actual)
return false
}
for _, match := range expected {
@ -53,405 +55,293 @@ func matchesEqual(t *testing.T, expected []Match, actual []Match) {
if expectedMatch.Table != actualMatch.Table ||
expectedMatch.Key != actualMatch.Key ||
expectedMatch.Value != actualMatch.Value {
t.Fatalf("match differ %v != %v", expectedMatch, actualMatch)
return false
}
} else {
t.Fatalf("%s not in %v", name, actualMatches)
return false
}
}
return true
}
func TestTagFilterNodes(t *testing.T) {
tests := []struct {
tags element.Tags
expected element.Tags
}{
{tags: element.Tags{}, expected: element.Tags{}},
{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "place": "unknown"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "place": "unknown", "population": "1000"}, expected: element.Tags{"name": "foo", "population": "1000"}},
{tags: element.Tags{"name": "foo", "place": "village"}, expected: element.Tags{"name": "foo", "place": "village"}},
{tags: element.Tags{"name": "foo", "place": "village", "population": "1000"}, expected: element.Tags{"name": "foo", "place": "village", "population": "1000"}},
{tags: element.Tags{"name": "foo", "place": "village", "unknown": "foo"}, expected: element.Tags{"name": "foo", "place": "village"}},
{tags: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, expected: element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}},
}
nodes := mapping.NodeTagFilter()
for i, test := range tests {
nodes.Filter(&test.tags)
if !stringMapEqual(test.tags, test.expected) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
}
}
}
func TestTagFilterNodes(t *testing.T) {
var tags element.Tags
nodes := mapping.NodeTagFilter()
tags = element.Tags{"name": "foo"}
if nodes.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "unknown": "baz"}
if nodes.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "place": "unknown"}
if nodes.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "place": "village"}
if nodes.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags)
tags = element.Tags{"name": "foo", "place": "village", "population": "1000"}
if nodes.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "population": "1000"}, tags)
tags = element.Tags{"name": "foo", "place": "village", "highway": "unknown"}
if nodes.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "place": "village"}, tags)
tags = element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}
if nodes.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "place": "village", "highway": "bus_stop"}, tags)
}
func TestTagFilterWays(t *testing.T) {
var tags element.Tags
tests := []struct {
tags element.Tags
expected element.Tags
}{
{tags: element.Tags{}, expected: element.Tags{}},
{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "highway": "unknown"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}},
{tags: element.Tags{"name": "foo", "building": "whatever"}, expected: element.Tags{"name": "foo", "building": "whatever"}},
{tags: element.Tags{"name": "foo", "highway": "track", "unknown": "foo"}, expected: element.Tags{"name": "foo", "highway": "track"}},
{tags: element.Tags{"name": "foo", "place": "village", "highway": "track"}, expected: element.Tags{"name": "foo", "highway": "track"}},
{tags: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, expected: element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}},
}
ways := mapping.WayTagFilter()
tags = element.Tags{"name": "foo"}
if ways.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
for i, test := range tests {
ways.Filter(&test.tags)
if !stringMapEqual(test.tags, test.expected) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
}
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "unknown": "baz"}
if ways.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "highway": "unknown"}
if ways.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "highway": "track"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags)
tags = element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "highway": "track", "oneway": "yes", "tunnel": "1"}, tags)
tags = element.Tags{"name": "foo", "place": "village", "highway": "track"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "highway": "track"}, tags)
tags = element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "railway": "tram", "highway": "secondary"}, tags)
// with __any__ value
tags = element.Tags{"name": "foo", "building": "yes"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "building": "yes"}, tags)
tags = element.Tags{"name": "foo", "building": "whatever"}
if ways.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "building": "whatever"}, tags)
}
func TestTagFilterRelations(t *testing.T) {
var tags element.Tags
tests := []struct {
tags element.Tags
expected element.Tags
}{
{tags: element.Tags{}, expected: element.Tags{}},
{tags: element.Tags{"name": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "unknown": "foo"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "landuse": "unknown"}, expected: element.Tags{"name": "foo"}},
{tags: element.Tags{"name": "foo", "landuse": "farm"}, expected: element.Tags{"name": "foo", "landuse": "farm"}},
{tags: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}},
{tags: element.Tags{"name": "foo", "type": "multipolygon"}, expected: element.Tags{"name": "foo", "type": "multipolygon"}},
{tags: element.Tags{"name": "foo", "type": "boundary"}, expected: element.Tags{"name": "foo", "type": "boundary"}},
{tags: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}, expected: element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}},
}
relations := mapping.RelationTagFilter()
tags = element.Tags{"name": "foo"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
for i, test := range tests {
relations.Filter(&test.tags)
if !stringMapEqual(test.tags, test.expected) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, test.tags, test.expected)
}
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "unknown": "baz"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "landuse": "unknown"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "landuse": "farm"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}
if relations.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "type": "multipolygon"}, tags)
// skip multipolygon with filtered tags, otherwise tags from
// longest way would be used
tags = element.Tags{"name": "foo", "landuse": "unknown", "type": "multipolygon"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "landuse": "park", "type": "multipolygon"}
if relations.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "type": "multipolygon", "landuse": "park"}, tags)
tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"}
if relations.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "multipolygon"}, tags)
// boundary relation for boundary
tags = element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"}
if relations.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "landuse": "farm", "boundary": "administrative", "type": "boundary"}, tags)
// boundary relation for non boundary
tags = element.Tags{"name": "foo", "landuse": "farm", "type": "boundary"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
/* skip boundary with filtered tags, otherwise tags from longest way would
be used */
tags = element.Tags{"name": "foo", "boundary": "unknown", "type": "boundary"}
if relations.Filter(&tags) != false {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{}, tags)
tags = element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"}
if relations.Filter(&tags) != true {
t.Fatal("unexpected filter response for", tags)
}
stringMapEquals(t, element.Tags{"name": "foo", "boundary": "administrative", "type": "boundary"}, tags)
}
func TestPointMatcher(t *testing.T) {
elem := element.Node{}
points := mapping.PointMatcher()
elem.Tags = element.Tags{"unknown": "baz"}
matchesEqual(t, []Match{}, points.MatchNode(&elem))
elem.Tags = element.Tags{"place": "unknown"}
matchesEqual(t, []Match{}, points.MatchNode(&elem))
elem.Tags = element.Tags{"place": "city"}
matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem))
elem.Tags = element.Tags{"place": "city", "highway": "unknown"}
matchesEqual(t, []Match{{"place", "city", DestTable{Name: "places"}, nil}}, points.MatchNode(&elem))
elem.Tags = element.Tags{"place": "city", "highway": "bus_stop"}
matchesEqual(t,
[]Match{
tests := []struct {
tags element.Tags
matches []Match
}{
{element.Tags{"unknown": "baz"}, []Match{}},
{element.Tags{"place": "unknown"}, []Match{}},
{element.Tags{"place": "city"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}},
{element.Tags{"place": "city", "highway": "unknown"}, []Match{{"place", "city", DestTable{Name: "places"}, nil}}},
{element.Tags{"place": "city", "highway": "bus_stop"}, []Match{
{"place", "city", DestTable{Name: "places"}, nil},
{"highway", "bus_stop", DestTable{Name: "transport_points"}, nil}},
points.MatchNode(&elem))
},
}
elem := element.Node{}
m := mapping.PointMatcher()
for i, test := range tests {
elem.Tags = test.tags
actual := m.MatchNode(&elem)
if !matchesEqual(actual, test.matches) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
}
}
}
func TestLineStringMatcher(t *testing.T) {
tests := []struct {
tags element.Tags
matches []Match
}{
{element.Tags{"unknown": "baz"}, []Match{}},
{element.Tags{"highway": "unknown"}, []Match{}},
{element.Tags{"highway": "pedestrian"},
[]Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}},
// exclude_tags area=yes
{element.Tags{"highway": "pedestrian", "area": "yes"}, []Match{}},
{element.Tags{"barrier": "hedge"},
[]Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}},
{element.Tags{"barrier": "hedge", "area": "yes"}, []Match{}},
{element.Tags{"aeroway": "runway"}, []Match{}},
{element.Tags{"aeroway": "runway", "area": "no"},
[]Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}},
{element.Tags{"highway": "secondary", "railway": "tram"},
[]Match{
{"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil},
{"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}},
},
{element.Tags{"highway": "footway", "landuse": "park", "barrier": "hedge"},
// landusages not a linestring table
[]Match{
{"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil},
{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}},
},
}
elem := element.Way{}
// fake closed way for area matching
elem.Refs = []int64{1, 2, 3, 4, 1}
if !elem.IsClosed() {
t.Fatal("way not closed")
}
ls := mapping.LineStringMatcher()
elem.Tags = element.Tags{"unknown": "baz"}
matchesEqual(t, []Match{}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"highway": "unknown"}
matchesEqual(t, []Match{}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"highway": "pedestrian"}
matchesEqual(t, []Match{{"highway", "pedestrian", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem))
// exclude_tags area=yes
elem.Tags = element.Tags{"highway": "pedestrian", "area": "yes"}
matchesEqual(t, []Match{}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"barrier": "hedge"}
matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "barrierways"}, nil}}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"}
matchesEqual(t, []Match{}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"aeroway": "runway", "area": "no"}
matchesEqual(t, []Match{{"aeroway", "runway", DestTable{Name: "aeroways"}, nil}}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"aeroway": "runway"}
matchesEqual(t, []Match{}, ls.MatchWay(&elem))
elem.Tags = element.Tags{"highway": "secondary", "railway": "tram"}
matchesEqual(t,
[]Match{
{"highway", "secondary", DestTable{Name: "roads", SubMapping: "roads"}, nil},
{"railway", "tram", DestTable{Name: "roads", SubMapping: "railway"}, nil}},
ls.MatchWay(&elem))
elem.Tags = element.Tags{"highway": "footway", "landuse": "park"}
// landusages not a linestring table
matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "roads", SubMapping: "roads"}, nil}}, ls.MatchWay(&elem))
m := mapping.LineStringMatcher()
for i, test := range tests {
elem.Tags = test.tags
actual := m.MatchWay(&elem)
if !matchesEqual(actual, test.matches) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
}
}
}
func TestPolygonMatcher(t *testing.T) {
elem := element.Relation{}
polys := mapping.PolygonMatcher()
func TestPolygonMatcher_MatchWay(t *testing.T) {
tests := []struct {
tags element.Tags
matches []Match
}{
{element.Tags{}, []Match{}},
{element.Tags{"unknown": "baz"}, []Match{}},
{element.Tags{"landuse": "unknown"}, []Match{}},
{element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}},
{element.Tags{"building": "yes"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}},
{element.Tags{"building": "residential"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}},
// line type requires area=yes
{element.Tags{"barrier": "hedge"}, []Match{}},
{element.Tags{"barrier": "hedge", "area": "yes"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}},
elem.Tags = element.Tags{"unknown": "baz"}
matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
{element.Tags{"building": "shop"}, []Match{
{"building", "shop", DestTable{Name: "buildings"}, nil},
{"building", "shop", DestTable{Name: "amenity_areas"}, nil},
}},
elem.Tags = element.Tags{"landuse": "unknowns"}
matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"building": "yes"}
matchesEqual(t, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"building": "residential"}
matchesEqual(t, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"barrier": "hedge"}
matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"barrier": "hedge", "area": "yes"}
matchesEqual(t, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"building": "shop"}
matchesEqual(t, []Match{
{"building", "shop", DestTable{Name: "buildings"}, nil},
{"building", "shop", DestTable{Name: "amenity_areas"}, nil}},
polys.MatchRelation(&elem))
elem.Tags = element.Tags{"landuse": "farm"}
matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"landuse": "farm", "highway": "secondary"}
matchesEqual(t, []Match{{"landuse", "farm", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"landuse": "farm", "aeroway": "apron"}
matchesEqual(t,
[]Match{
{element.Tags{"aeroway": "apron", "landuse": "farm"}, []Match{
{"aeroway", "apron", DestTable{Name: "transport_areas"}, nil},
{"landuse", "farm", DestTable{Name: "landusages"}, nil}},
polys.MatchRelation(&elem))
{"landuse", "farm", DestTable{Name: "landusages"}, nil},
}},
elem.Tags = element.Tags{"highway": "footway"} // linear by default
matchesEqual(t, []Match{}, polys.MatchRelation(&elem))
{element.Tags{"landuse": "farm", "highway": "secondary"}, []Match{
{"landuse", "farm", DestTable{Name: "landusages"}, nil},
}},
elem.Tags = element.Tags{"highway": "footway", "area": "yes"}
matchesEqual(t, []Match{{"highway", "footway", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
{element.Tags{"highway": "footway"}, []Match{}},
{element.Tags{"highway": "footway", "area": "yes"}, []Match{
{"highway", "footway", DestTable{Name: "landusages"}, nil},
}},
elem.Tags = element.Tags{"boundary": "administrative", "admin_level": "8"}
matchesEqual(t, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}, polys.MatchRelation(&elem))
{element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}},
/*
landusages mapping has the following order,
check that XxxMatcher always uses the first
amenity:
- university
landuse:
- forest
leisure:
- park
landuse:
- park
*/
{element.Tags{"landuse": "forest", "leisure": "park"}, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}},
{element.Tags{"landuse": "park", "leisure": "park"}, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}},
{element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"}, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}},
}
elem := element.Way{}
// fake closed way for area matching
elem.Refs = []int64{1, 2, 3, 4, 1}
if !elem.IsClosed() {
t.Fatal("way not closed")
}
m := mapping.PolygonMatcher()
for i, test := range tests {
elem.Tags = test.tags
actual := m.MatchWay(&elem)
if !matchesEqual(actual, test.matches) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
}
}
elem.Refs = nil
elem.Tags = element.Tags{"building": "yes"}
actual := m.MatchWay(&elem)
if !matchesEqual([]Match{}, actual) {
t.Error("open way matched as polygon")
}
}
func TestMatcherMappingOrder(t *testing.T) {
func TestPolygonMatcher_MatchRelation(t *testing.T) {
// check that only relations with type=multipolygon/boundary are matched as polygon
tests := []struct {
tags element.Tags
matches []Match
}{
{element.Tags{}, []Match{}},
{element.Tags{"unknown": "baz"}, []Match{}},
{element.Tags{"landuse": "unknown"}, []Match{}},
{element.Tags{"landuse": "unknown", "type": "multipolygon"}, []Match{}},
{element.Tags{"building": "yes"}, []Match{}},
{element.Tags{"building": "yes", "type": "multipolygon"}, []Match{{"building", "yes", DestTable{Name: "buildings"}, nil}}},
{element.Tags{"building": "residential", "type": "multipolygon"}, []Match{{"building", "residential", DestTable{Name: "buildings"}, nil}}},
// line type requires area=yes
{element.Tags{"barrier": "hedge", "type": "multipolygon"}, []Match{}},
{element.Tags{"barrier": "hedge", "area": "yes", "type": "multipolygon"}, []Match{{"barrier", "hedge", DestTable{Name: "landusages"}, nil}}},
{element.Tags{"building": "shop", "type": "multipolygon"}, []Match{
{"building", "shop", DestTable{Name: "buildings"}, nil},
{"building", "shop", DestTable{Name: "amenity_areas"}, nil},
}},
{element.Tags{"aeroway": "apron", "landuse": "farm", "type": "multipolygon"}, []Match{
{"aeroway", "apron", DestTable{Name: "transport_areas"}, nil},
{"landuse", "farm", DestTable{Name: "landusages"}, nil},
}},
{element.Tags{"landuse": "farm", "highway": "secondary", "type": "multipolygon"}, []Match{
{"landuse", "farm", DestTable{Name: "landusages"}, nil},
}},
{element.Tags{"highway": "footway", "type": "multipolygon"}, []Match{}},
{element.Tags{"highway": "footway", "area": "yes", "type": "multipolygon"}, []Match{
{"highway", "footway", DestTable{Name: "landusages"}, nil},
}},
{element.Tags{"boundary": "administrative", "admin_level": "8"}, []Match{}},
{element.Tags{"boundary": "administrative", "admin_level": "8", "type": "boundary"}, []Match{{"boundary", "administrative", DestTable{Name: "admin"}, nil}}},
}
elem := element.Relation{}
polys := mapping.PolygonMatcher()
/*
landusages mapping has the following order,
check that XxxMatcher always uses the first
amenity:
- university
landuse:
- forest
leisure:
- park
landuse:
- park
*/
elem.Tags = element.Tags{"landuse": "forest", "leisure": "park"}
matchesEqual(t, []Match{{"landuse", "forest", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"landuse": "park", "leisure": "park"}
matchesEqual(t, []Match{{"leisure", "park", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
elem.Tags = element.Tags{"landuse": "park", "leisure": "park", "amenity": "university"}
matchesEqual(t, []Match{{"amenity", "university", DestTable{Name: "landusages"}, nil}}, polys.MatchRelation(&elem))
}
func TestFilterNodes(t *testing.T) {
var tags element.Tags
// test name only
tags = make(element.Tags)
tags["name"] = "foo"
points := mapping.NodeTagFilter()
if points.Filter(&tags) != false {
t.Fatal("Filter result not false")
}
if len(tags) != 0 {
t.Fatal("Filter result not empty")
}
// test name + unmapped tags
tags = make(element.Tags)
tags["name"] = "foo"
tags["boring"] = "true"
if points.Filter(&tags) != false {
t.Fatal("Filter result not false")
}
if len(tags) != 0 {
t.Fatal("Filter result not empty")
}
// test fields only, but no mapping
tags = make(element.Tags)
tags["population"] = "0"
tags["name"] = "foo"
tags["boring"] = "true"
if points.Filter(&tags) != false {
t.Fatal("Filter result true", tags)
}
if len(tags) != 0 {
t.Fatal("Filter result not empty", tags)
}
// ... not with mapped tag (place)
tags = make(element.Tags)
tags["population"] = "0"
tags["name"] = "foo"
tags["boring"] = "true"
tags["place"] = "village"
if points.Filter(&tags) != true {
t.Fatal("Filter result true", tags)
}
if len(tags) != 3 && tags["population"] == "0" && tags["name"] == "foo" && tags["place"] == "village" {
t.Fatal("Filter result not expected", tags)
m := mapping.PolygonMatcher()
for i, test := range tests {
elem.Tags = test.tags
actual := m.MatchRelation(&elem)
if !matchesEqual(actual, test.matches) {
t.Errorf("unexpected result for case %d: %v != %v", i+1, actual, test.matches)
}
}
}
@ -460,7 +350,7 @@ func TestExcludeFilter(t *testing.T) {
var tags element.Tags
// no matches
f = newExcludeFilter([]Key{})
f = newExcludeFilter([]config.Key{})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}) {
@ -468,7 +358,7 @@ func TestExcludeFilter(t *testing.T) {
}
// match all
f = newExcludeFilter([]Key{"*"})
f = newExcludeFilter([]config.Key{"*"})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{}) {
@ -476,7 +366,7 @@ func TestExcludeFilter(t *testing.T) {
}
// fixed string and wildcard match
f = newExcludeFilter([]Key{"source", "tiger:*"})
f = newExcludeFilter([]config.Key{"source", "tiger:*"})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{"source:foo": "1"}) {
@ -495,9 +385,7 @@ func BenchmarkFilterNodes(b *testing.B) {
tags["boring"] = "true"
points := mapping.NodeTagFilter()
if points.Filter(&tags) != true {
b.Fatal("Filter result true", tags)
}
points.Filter(&tags)
if len(tags) != 2 && tags["population"] == "0" && tags["name"] == "foo" {
b.Fatal("Filter result not expected", tags)
}

331
mapping/mapping.go Normal file
View File

@ -0,0 +1,331 @@
package mapping
import (
"errors"
"io/ioutil"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
"gopkg.in/yaml.v2"
)
type orderedDestTable struct {
DestTable
order int
}
type TagTableMapping map[Key]map[Value][]orderedDestTable
func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) {
for key, vals := range mapping {
for _, v := range vals {
vals, ok := tt[Key(key)]
tbl := orderedDestTable{DestTable: table, order: v.Order}
if ok {
vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl)
} else {
tt[Key(key)] = make(map[Value][]orderedDestTable)
tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl)
}
}
}
}
func (tt TagTableMapping) asTagMap() tagMap {
result := make(tagMap)
for k, vals := range tt {
result[k] = make(map[Value]struct{})
for v := range vals {
result[k][v] = struct{}{}
}
}
return result
}
type DestTable struct {
Name string
SubMapping string
}
type TableType string
func (tt *TableType) UnmarshalJSON(data []byte) error {
switch string(data) {
case "":
return errors.New("missing table type")
case `"point"`:
*tt = PointTable
case `"linestring"`:
*tt = LineStringTable
case `"polygon"`:
*tt = PolygonTable
case `"geometry"`:
*tt = GeometryTable
case `"relation"`:
*tt = RelationTable
case `"relation_member"`:
*tt = RelationMemberTable
default:
return errors.New("unknown type " + string(data))
}
return nil
}
const (
PolygonTable TableType = "polygon"
LineStringTable TableType = "linestring"
PointTable TableType = "point"
GeometryTable TableType = "geometry"
RelationTable TableType = "relation"
RelationMemberTable TableType = "relation_member"
)
type Mapping struct {
Conf config.Mapping
}
func NewMapping(filename string) (*Mapping, error) {
f, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
mapping := Mapping{}
err = yaml.Unmarshal(f, &mapping.Conf)
if err != nil {
return nil, err
}
err = mapping.prepare()
if err != nil {
return nil, err
}
return &mapping, nil
}
func (m *Mapping) prepare() error {
for name, t := range m.Conf.Tables {
t.Name = name
if t.OldFields != nil {
// todo deprecate 'fields'
t.Columns = t.OldFields
}
}
for name, t := range m.Conf.GeneralizedTables {
t.Name = name
}
return nil
}
func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
for name, t := range m.Conf.Tables {
if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
continue
}
mappings.addFromMapping(t.Mapping, DestTable{Name: name})
for subMappingName, subMapping := range t.Mappings {
mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
}
switch tableType {
case PointTable:
mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
case LineStringTable:
mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
case PolygonTable:
mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
}
}
}
func (m *Mapping) tables(tableType TableType) map[string]*rowBuilder {
result := make(map[string]*rowBuilder)
for name, t := range m.Conf.Tables {
if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable {
result[name] = makeRowBuilder(t)
}
}
return result
}
func makeRowBuilder(tbl *config.Table) *rowBuilder {
result := rowBuilder{}
for _, mappingColumn := range tbl.Columns {
column := valueBuilder{}
column.key = Key(mappingColumn.Key)
columnType := MakeColumnType(mappingColumn)
if columnType != nil {
column.colType = *columnType
} else {
log.Warn("unhandled type: ", mappingColumn.Type)
}
result.columns = append(result.columns, column)
}
return &result
}
func MakeColumnType(c *config.Column) *ColumnType {
if columnType, ok := AvailableColumnTypes[c.Type]; ok {
if columnType.MakeFunc != nil {
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
if err != nil {
log.Print(err)
return nil
}
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
}
columnType.FromMember = c.FromMember
return &columnType
}
return nil
}
func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
for _, t := range m.Conf.Tables {
if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable {
continue
}
for _, col := range t.Columns {
if col.Key != "" {
tags[Key(col.Key)] = true
}
for _, k := range col.Keys {
tags[Key(k)] = true
}
}
if t.Filters != nil && t.Filters.ExcludeTags != nil {
for _, keyVal := range *t.Filters.ExcludeTags {
tags[Key(keyVal[0])] = true
}
}
if tableType == PolygonTable || tableType == RelationTable || tableType == RelationMemberTable {
if t.RelationTypes != nil {
tags["type"] = true
}
}
}
for _, k := range m.Conf.Tags.Include {
tags[Key(k)] = true
}
// always include area tag for closed-way handling
tags["area"] = true
}
type elementFilter func(tags element.Tags, key Key, closed bool) bool
type tableElementFilters map[string][]elementFilter
func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) {
var areaTags map[Key]struct{}
var linearTags map[Key]struct{}
if m.Conf.Areas.AreaTags != nil {
areaTags = make(map[Key]struct{})
for _, tag := range m.Conf.Areas.AreaTags {
areaTags[Key(tag)] = struct{}{}
}
}
if m.Conf.Areas.LinearTags != nil {
linearTags = make(map[Key]struct{})
for _, tag := range m.Conf.Areas.LinearTags {
linearTags[Key(tag)] = struct{}{}
}
}
for name, t := range m.Conf.Tables {
if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
continue
}
if TableType(t.Type) == LineStringTable && areaTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed {
if tags["area"] == "yes" {
return false
}
if tags["area"] != "no" {
if _, ok := areaTags[key]; ok {
return false
}
}
}
return true
}
filters[name] = append(filters[name], f)
}
if TableType(t.Type) == PolygonTable && linearTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed && tags["area"] == "no" {
return false
}
if tags["area"] != "yes" {
if _, ok := linearTags[key]; ok {
return false
}
}
return true
}
filters[name] = append(filters[name], f)
}
}
}
func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
for name, t := range m.Conf.Tables {
if t.RelationTypes != nil {
relTypes := t.RelationTypes // copy loop var for closure
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags["type"]; ok {
for _, rtype := range relTypes {
if v == rtype {
return true
}
}
}
return false
}
filters[name] = append(filters[name], f)
} else {
if TableType(t.Type) == PolygonTable {
// standard mulipolygon handling (boundary and land_area are for backwards compatibility)
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags["type"]; ok {
if v == "multipolygon" || v == "boundary" || v == "land_area" {
return true
}
}
return false
}
filters[name] = append(filters[name], f)
}
}
}
}
func (m *Mapping) addFilters(filters tableElementFilters) {
for name, t := range m.Conf.Tables {
if t.Filters == nil {
continue
}
if t.Filters.ExcludeTags != nil {
for _, filterKeyVal := range *t.Filters.ExcludeTags {
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[filterKeyVal[0]]; ok {
if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
return false
}
}
return true
}
filters[name] = append(filters[name], f)
}
}
}
}

151
mapping/matcher.go
View File

@ -6,9 +6,11 @@ import (
)
func (m *Mapping) PointMatcher() NodeMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(PointTable, mappings)
filters := m.ElementFilters()
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PointTable, filters)
return &tagMatcher{
mappings: mappings,
tables: m.tables(PointTable),
@ -18,9 +20,11 @@ func (m *Mapping) PointMatcher() NodeMatcher {
}
func (m *Mapping) LineStringMatcher() WayMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(LineStringTable, mappings)
filters := m.ElementFilters()
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(LineStringTable, filters)
return &tagMatcher{
mappings: mappings,
tables: m.tables(LineStringTable),
@ -30,48 +34,57 @@ func (m *Mapping) LineStringMatcher() WayMatcher {
}
func (m *Mapping) PolygonMatcher() RelWayMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(PolygonTable, mappings)
filters := m.ElementFilters()
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PolygonTable, filters)
relFilters := make(tableElementFilters)
m.addRelationFilters(PolygonTable, relFilters)
return &tagMatcher{
mappings: mappings,
tables: m.tables(PolygonTable),
filters: filters,
relFilters: relFilters,
matchAreas: true,
}
}
func (m *Mapping) RelationMatcher() RelationMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(RelationTable, mappings)
filters := m.ElementFilters()
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PolygonTable, filters)
m.addTypedFilters(RelationTable, filters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationTable, relFilters)
return &tagMatcher{
mappings: mappings,
tables: m.tables(RelationTable),
filters: filters,
relFilters: relFilters,
matchAreas: true,
}
}
func (m *Mapping) RelationMemberMatcher() RelationMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(RelationMemberTable, mappings)
filters := m.ElementFilters()
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(RelationMemberTable, filters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationMemberTable, relFilters)
return &tagMatcher{
mappings: mappings,
tables: m.tables(RelationMemberTable),
filters: filters,
relFilters: relFilters,
matchAreas: true,
}
}
type Match struct {
Key string
Value string
Table DestTable
tableFields *TableFields
}
type NodeMatcher interface {
MatchNode(node *element.Node) []Match
}
@ -89,23 +102,31 @@ type RelWayMatcher interface {
RelationMatcher
}
type tagMatcher struct {
mappings TagTables
tables map[string]*TableFields
filters map[string][]ElementFilter
matchAreas bool
type Match struct {
Key string
Value string
Table DestTable
builder *rowBuilder
}
func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} {
return m.tableFields.MakeRow(elem, geom, *m)
return m.builder.MakeRow(elem, geom, *m)
}
func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry) []interface{} {
return m.tableFields.MakeMemberRow(rel, member, geom, *m)
return m.builder.MakeMemberRow(rel, member, geom, *m)
}
type tagMatcher struct {
mappings TagTableMapping
tables map[string]*rowBuilder
filters tableElementFilters
relFilters tableElementFilters
matchAreas bool
}
func (tm *tagMatcher) MatchNode(node *element.Node) []Match {
return tm.match(node.Tags, false)
return tm.match(node.Tags, false, false)
}
func (tm *tagMatcher) MatchWay(way *element.Way) []Match {
@ -114,22 +135,22 @@ func (tm *tagMatcher) MatchWay(way *element.Way) []Match {
if way.Tags["area"] == "no" {
return nil
}
return tm.match(way.Tags, true)
return tm.match(way.Tags, true, false)
}
} else { // match way as linestring
if way.IsClosed() {
if way.Tags["area"] == "yes" {
return nil
}
return tm.match(way.Tags, true)
return tm.match(way.Tags, true, false)
}
return tm.match(way.Tags, false)
return tm.match(way.Tags, false, false)
}
return nil
}
func (tm *tagMatcher) MatchRelation(rel *element.Relation) []Match {
return tm.match(rel.Tags, true)
return tm.match(rel.Tags, true, true)
}
type orderedMatch struct {
@ -137,17 +158,17 @@ type orderedMatch struct {
order int
}
func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Match {
tables := make(map[DestTable]orderedMatch)
addTables := func(k, v string, tbls []OrderedDestTable) {
addTables := func(k, v string, tbls []orderedDestTable) {
for _, t := range tbls {
this := orderedMatch{
Match: Match{
Key: k,
Value: v,
Table: t.DestTable,
tableFields: tm.tables[t.Name],
Key: k,
Value: v,
Table: t.DestTable,
builder: tm.tables[t.Name],
},
order: t.order,
}
@ -187,6 +208,18 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
}
}
}
if relation && !filteredOut {
filters, ok := tm.relFilters[t.Name]
if ok {
for _, filter := range filters {
if !filter(tags, Key(match.Key), closed) {
filteredOut = true
break
}
}
}
}
if !filteredOut {
matches = append(matches, match.Match)
}
@ -194,6 +227,54 @@ func (tm *tagMatcher) match(tags element.Tags, closed bool) []Match {
return matches
}
type valueBuilder struct {
key Key
colType ColumnType
}
func (v *valueBuilder) Value(elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
if v.colType.Func != nil {
return v.colType.Func(elem.Tags[string(v.key)], elem, geom, match)
}
return nil
}
func (v *valueBuilder) MemberValue(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) interface{} {
if v.colType.Func != nil {
if v.colType.FromMember {
if member.Elem == nil {
return nil
}
return v.colType.Func(member.Elem.Tags[string(v.key)], member.Elem, geom, match)
}
return v.colType.Func(rel.Tags[string(v.key)], &rel.OSMElem, geom, match)
}
if v.colType.MemberFunc != nil {
return v.colType.MemberFunc(rel, member, match)
}
return nil
}
type rowBuilder struct {
columns []valueBuilder
}
func (r *rowBuilder) MakeRow(elem *element.OSMElem, geom *geom.Geometry, match Match) []interface{} {
var row []interface{}
for _, column := range r.columns {
row = append(row, column.Value(elem, geom, match))
}
return row
}
func (r *rowBuilder) MakeMemberRow(rel *element.Relation, member *element.Member, geom *geom.Geometry, match Match) []interface{} {
var row []interface{}
for _, column := range r.columns {
row = append(row, column.MemberValue(rel, member, geom, match))
}
return row
}
// SelectRelationPolygons returns a slice of all members that are already
// imported as part of the relation.
// Outer members are "imported" if they share the same destination table. Inner members

10
mapping/matcher_test.go
View File

@ -43,7 +43,7 @@ func TestSelectRelationPolygonsSimple(t *testing.T) {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park"}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "forest"}),
makeMember(1, element.Tags{"landuse": "park"}),
@ -68,7 +68,7 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park"}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "park", "layer": "2", "name": "foo"}),
makeMember(1, element.Tags{"landuse": "forest"}),
@ -91,7 +91,7 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "park"}
r.Tags = element.Tags{"landuse": "park", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"landuse": "park"}),
makeMember(1, element.Tags{"natural": "forest"}),
@ -117,7 +117,7 @@ func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"landuse": "forest", "natural": "scrub"}
r.Tags = element.Tags{"landuse": "forest", "natural": "scrub", "type": "multipolygon"}
r.Members = []element.Member{
makeMember(0, element.Tags{"natural": "scrub"}),
makeMember(1, element.Tags{"landuse": "forest"}),
@ -139,7 +139,7 @@ func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) {
t.Fatal(err)
}
r := element.Relation{}
r.Tags = element.Tags{"waterway": "riverbank"}
r.Tags = element.Tags{"waterway": "riverbank", "type": "multipolygon"}
r.Members = []element.Member{
makeMemberRole(0, element.Tags{"waterway": "riverbank", "natural": "water"}, "outer"),
makeMemberRole(1, element.Tags{"natural": "water"}, "inner"),

30
mapping/test_mapping.json
View File

@ -43,7 +43,7 @@
},
"tables": {
"landusages": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -209,7 +209,7 @@
}
},
"buildings": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -240,7 +240,7 @@
},
"amenity_areas": {
"_comment": "for testing duplicate inserts with __any__ and exact match",
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -270,7 +270,7 @@
}
},
"places": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -333,7 +333,7 @@
}
},
"transport_areas": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -370,7 +370,7 @@
}
},
"admin": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -405,7 +405,7 @@
}
},
"aeroways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -436,7 +436,7 @@
}
},
"waterways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -473,7 +473,7 @@
}
},
"barrierways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -515,7 +515,7 @@
}
},
"transport_points": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -566,7 +566,7 @@
}
},
"amenities": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -603,7 +603,7 @@
}
},
"barrierpoints": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -648,7 +648,7 @@
}
},
"housenumbers_interpolated": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -698,7 +698,7 @@
}
},
"roads": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -818,7 +818,7 @@
}
},
"housenumbers": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",

30
mapping/test_mapping.yml
View File

@ -35,7 +35,7 @@ generalized_tables:
tolerance: 50.0
tables:
admin:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -53,7 +53,7 @@ tables:
- administrative
type: polygon
aeroways:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -69,7 +69,7 @@ tables:
- taxiway
type: linestring
amenities:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -92,7 +92,7 @@ tables:
type: point
amenity_areas:
_comment: for testing duplicate inserts with __any__ and exact match
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -107,7 +107,7 @@ tables:
- shop
type: polygon
barrierpoints:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -137,7 +137,7 @@ tables:
- stile
type: point
barrierways:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -164,7 +164,7 @@ tables:
- wire_fence
type: linestring
buildings:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -179,7 +179,7 @@ tables:
- __any__
type: polygon
housenumbers:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -203,7 +203,7 @@ tables:
- __any__
type: point
housenumbers_interpolated:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -230,7 +230,7 @@ tables:
- __any__
type: linestring
landusages:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -363,7 +363,7 @@ tables:
- riverbank
type: polygon
places:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -404,7 +404,7 @@ tables:
- locality
type: point
roads:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -524,7 +524,7 @@ tables:
- groyne
type: linestring
transport_areas:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -545,7 +545,7 @@ tables:
- platform
type: polygon
transport_points:
fields:
columns:
- name: osm_id
type: id
- name: geometry
@ -577,7 +577,7 @@ tables:
- subway_entrance
type: point
waterways:
fields:
columns:
- name: osm_id
type: id
- name: geometry

4
test/any_any_mapping.json
View File

@ -8,7 +8,7 @@
},
"tables": {
"all": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id"
@ -28,7 +28,7 @@
}
},
"amenities": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id"

10
test/complete_db.osc
View File

@ -35,15 +35,6 @@
<tag k="name" v="way 11001"/>
<tag k="natural" v="water"/>
</way>
<way id="12001" version="2" timestamp="2011-11-11T00:11:11Z">
<nd ref="12001"/>
<nd ref="12002"/>
<nd ref="12003"/>
<nd ref="12004"/>
<nd ref="12001"/>
<tag k="name" v="way 12001"/>
<tag k="natural" v="water"/>
</way>
<relation id="13001" version="2" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="13001" role="outer"/>
<tag k="natural" v="water"/>
@ -71,6 +62,7 @@
<tag k="natural" v="water"/>
</way>
<relation id="14001" version="2" timestamp="2011-11-11T00:11:11Z">
<!-- now a new style relation -->
<member type="way" ref="14001" role="outer"/>
<member type="way" ref="14011" role="inner"/>
<tag k="type" v="multipolygon"/>

65
test/complete_db.osm
View File

@ -267,8 +267,19 @@
<relation id="1001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="1001" role="outer"/>
<member type="way" ref="1002" role="inner"/>
<tag k="landuse" v="wood"/>
<tag k="type" v="multipolygon"/>
</relation>
<relation id="1011" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="1001" role="outer"/>
<member type="way" ref="1002" role="inner"/>
<tag k="type" v="multipolygon"/>
</relation>
<relation id="1021" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="1001" role="outer"/>
<member type="way" ref="1002" role="inner"/>
<tag k="landuse" v="wood"/>
</relation>
<relation id="2001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="2001" role="outer"/>
<member type="way" ref="2002" role="inner"/>
@ -524,7 +535,6 @@
<nd ref="9201"/>
<nd ref="9202"/>
<nd ref="9203"/>
<tag k="landuse" v="park"/>
<tag k="highway" v="secondary"/>
<tag k="name" v="9209"/>
</way>
@ -540,37 +550,10 @@
<member type="way" ref="9209" role="outer"/>
<member type="way" ref="9210" role="outer"/>
<tag k="type" v="multipolygon"/>
</relation>
<!-- test multipolygon ways were inserted (same as 92xx, but different tagging) -->
<node id="9301" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
<node id="9302" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
<node id="9303" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="82"/>
<node id="9304" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="80"/>
<way id="9309" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="9301"/>
<nd ref="9302"/>
<nd ref="9303"/>
<tag k="landuse" v="park"/>
<tag k="highway" v="secondary"/>
<tag k="name" v="9309"/>
</way>
<way id="9310" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="9303"/>
<nd ref="9304"/>
<nd ref="9301"/>
<tag k="highway" v="residential"/>
<tag k="name" v="9310"/>
</way>
<relation id="9301" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="9309" role="outer"/>
<member type="way" ref="9310" role="outer"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="park"/>
</relation>
<!-- test multipolygon way was inserted -->
<node id="8001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="80"/>
<node id="8002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="82"/>
@ -606,25 +589,6 @@
<tag k="landuse" v="park"/>
</way>
<!-- test for changed tags in way belonging to relation -->
<node id="12001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="85"/>
<node id="12002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="86"/>
<node id="12003" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="86"/>
<node id="12004" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="85"/>
<way id="12001" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="12001"/>
<nd ref="12002"/>
<nd ref="12003"/>
<nd ref="12004"/>
<nd ref="12001"/>
<tag k="name" v="way 12001"/>
<tag k="landuse" v="park"/>
</way>
<relation id="12001" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="12001" role="outer"/>
<tag k="type" v="multipolygon"/>
</relation>
<!-- test for changed tags in relation -->
<node id="13001" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="87"/>
<node id="13002" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="88"/>
@ -672,6 +636,7 @@
<tag k="name" v="way 14011"/>
</way>
<relation id="14001" version="1" timestamp="2011-11-11T00:11:11Z">
<!-- old style relation not inserted -->
<member type="way" ref="14001" role="outer"/>
<member type="way" ref="14011" role="inner"/>
<tag k="type" v="multipolygon"/>
@ -694,7 +659,6 @@
<nd ref="15004"/>
<nd ref="15001"/>
<tag k="name" v="way 15001"/>
<tag k="landuse" v="park"/>
</way>
<way id="15011" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="15011"/>
@ -708,6 +672,7 @@
<member type="way" ref="15001" role="outer"/>
<member type="way" ref="15011" role="inner"/>
<tag k="type" v="multipolygon"/>
<tag k="landuse" v="park"/>
</relation>
@ -947,7 +912,7 @@
<tag k="landuse" v="park"/>
</relation>
<!-- test removing of relation (r:50121) without tags -->
<!-- test old-style relation does not affect way -->
<node id="50101" version="1" timestamp="2011-11-11T00:11:11Z" lat="42" lon="10"/>
<node id="50102" version="1" timestamp="2011-11-11T00:11:11Z" lat="42" lon="11"/>
<node id="50103" version="1" timestamp="2011-11-11T00:11:11Z" lat="44" lon="10"/>
@ -1031,10 +996,10 @@
<nd ref="52103"/>
<nd ref="52104"/>
<nd ref="52101"/>
<tag k="building" v="yes"/>
</way>
<relation id="52121" version="1" timestamp="2011-11-11T00:11:11Z">
<member type="way" ref="52111" role="outer"/>
<tag k="building" v="yes"/>
<tag k="type" v="multipolygon"/>
</relation>

30
test/complete_db_mapping.json
View File

@ -54,7 +54,7 @@
},
"tables": {
"landusages": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -191,7 +191,7 @@
}
},
"buildings": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -228,7 +228,7 @@
}
},
"places": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -290,7 +290,7 @@
}
},
"transport_areas": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -327,7 +327,7 @@
}
},
"admin": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -362,7 +362,7 @@
}
},
"aeroways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -393,7 +393,7 @@
}
},
"waterways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -430,7 +430,7 @@
}
},
"barrierways": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -472,7 +472,7 @@
}
},
"transport_points": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -523,7 +523,7 @@
}
},
"amenities": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -560,7 +560,7 @@
}
},
"barrierpoints": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -605,7 +605,7 @@
}
},
"housenumbers_interpolated": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -655,7 +655,7 @@
}
},
"roads": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -784,7 +784,7 @@
}
},
"housenumbers": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",
@ -829,7 +829,7 @@
}
},
"waterareas": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",

49
test/completedb_test.go
View File

@ -61,37 +61,38 @@ func TestComplete_Deploy(t *testing.T) {
}
}
func TestComplete_OnlyNewStyleMultipolgon(t *testing.T) {
assertRecords(t, []checkElem{
{"osm_landusages", -1001, "wood", nil},
{"osm_landusages", -1011, Missing, nil},
{"osm_landusages", -1021, Missing, nil},
})
}
func TestComplete_LandusageToWaterarea1(t *testing.T) {
// Parks inserted into landusages
cache := ts.cache(t)
defer cache.Close()
assertCachedWay(t, cache, 11001)
assertCachedWay(t, cache, 12001)
assertCachedWay(t, cache, 13001)
assertRecords(t, []checkElem{
{"osm_waterareas", 11001, Missing, nil},
{"osm_waterareas", -12001, Missing, nil},
{"osm_waterareas", -13001, Missing, nil},
{"osm_waterareas_gen0", 11001, Missing, nil},
{"osm_waterareas_gen0", -12001, Missing, nil},
{"osm_waterareas_gen0", -13001, Missing, nil},
{"osm_waterareas_gen1", 11001, Missing, nil},
{"osm_waterareas_gen1", -12001, Missing, nil},
{"osm_waterareas_gen1", -13001, Missing, nil},
{"osm_landusages", 11001, "park", nil},
{"osm_landusages", -12001, "park", nil},
{"osm_landusages", -13001, "park", nil},
{"osm_landusages_gen0", 11001, "park", nil},
{"osm_landusages_gen0", -12001, "park", nil},
{"osm_landusages_gen0", -13001, "park", nil},
{"osm_landusages_gen1", 11001, "park", nil},
{"osm_landusages_gen1", -12001, "park", nil},
{"osm_landusages_gen1", -13001, "park", nil},
})
}
@ -106,7 +107,8 @@ func TestComplete_ChangedHoleTags1(t *testing.T) {
assertRecords(t, []checkElem{
{"osm_waterareas", 14011, Missing, nil},
{"osm_waterareas", -14011, Missing, nil},
{"osm_landusages", -14001, "park", nil},
{"osm_landusages", 14001, "park", nil},
{"osm_landusages", -14001, Missing, nil},
})
}
@ -218,19 +220,15 @@ func TestComplete_RelationWayNotInserted(t *testing.T) {
func TestComplete_RelationWaysInserted(t *testing.T) {
// Outer ways of multipolygon are inserted.
assertRecords(t, []checkElem{
{"osm_landusages", -9201, "park", map[string]string{"name": "9209"}},
// no name on relation
{"osm_landusages", -9201, "park", map[string]string{"name": ""}},
{"osm_landusages", 9201, Missing, nil},
{"osm_landusages", 9209, Missing, nil},
{"osm_landusages", 9210, Missing, nil},
// outer ways of multipolygon stand for their own
{"osm_roads", 9209, "secondary", map[string]string{"name": "9209"}},
{"osm_roads", 9210, "residential", map[string]string{"name": "9210"}},
// no name on relation
{"osm_landusages", -9301, "park", map[string]string{"name": ""}},
// outer ways of multipolygon stand for their own
{"osm_roads", 9309, "secondary", map[string]string{"name": "9309"}},
{"osm_roads", 9310, "residential", map[string]string{"name": "9310"}},
})
}
func TestComplete_RelationWayInserted(t *testing.T) {
@ -283,12 +281,12 @@ func TestComplete_RelationBeforeRemove(t *testing.T) {
})
}
func TestComplete_RelationWithoutTags(t *testing.T) {
// Relation without tags is inserted.
func TestComplete_OldStyleRelationIsIgnored(t *testing.T) {
// Relation without tags is not inserted.
assertRecords(t, []checkElem{
{"osm_buildings", 50111, Missing, nil},
{"osm_buildings", -50121, "yes", nil},
{"osm_buildings", 50111, "yes", nil},
{"osm_buildings", -50121, Missing, nil},
})
}
@ -483,27 +481,21 @@ func TestComplete_LandusageToWaterarea2(t *testing.T) {
assertRecords(t, []checkElem{
{"osm_waterareas", 11001, "water", nil},
{"osm_waterareas", -12001, "water", nil},
{"osm_waterareas", -13001, "water", nil},
{"osm_waterareas_gen0", 11001, "water", nil},
{"osm_waterareas_gen0", -12001, "water", nil},
{"osm_waterareas_gen0", -13001, "water", nil},
{"osm_waterareas_gen1", 11001, "water", nil},
{"osm_waterareas_gen1", -12001, "water", nil},
{"osm_waterareas_gen1", -13001, "water", nil},
{"osm_landusages", 11001, Missing, nil},
{"osm_landusages", -12001, Missing, nil},
{"osm_landusages", -13001, Missing, nil},
{"osm_landusages_gen0", 11001, Missing, nil},
{"osm_landusages_gen0", -12001, Missing, nil},
{"osm_landusages_gen0", -13001, Missing, nil},
{"osm_landusages_gen1", 11001, Missing, nil},
{"osm_landusages_gen1", -12001, Missing, nil},
{"osm_landusages_gen1", -13001, Missing, nil},
})
}
@ -518,6 +510,11 @@ func TestComplete_ChangedHoleTags2(t *testing.T) {
assertGeomArea(t, checkElem{"osm_waterareas", 14011, "water", nil}, 26672019779)
assertGeomArea(t, checkElem{"osm_landusages", -14001, "park", nil}, 10373697182)
assertRecords(t, []checkElem{
{"osm_waterareas", -14011, Missing, nil},
{"osm_landusages", -14001, "park", nil},
})
}
func TestComplete_SplitOuterMultipolygonWay2(t *testing.T) {

6
test/expire_tiles_mapping.yml
View File

@ -1,7 +1,7 @@
tables:
roads:
type: linestring
fields:
columns:
- name: osm_id
type: id
- name: type
@ -16,7 +16,7 @@ tables:
pois:
type: point
fields:
columns:
- name: osm_id
type: id
- name: type
@ -31,7 +31,7 @@ tables:
buildings:
type: polygon
fields:
columns:
- name: osm_id
type: id
- name: type

10
test/route_relation.osm
View File

@ -153,5 +153,15 @@
<tag k="type" v="route"/>
</relation>
<!-- non-route type is not imported -->
<node id="130101" version="1" timestamp="2015-12-31T23:59:99Z" lat="53.0" lon="8.200">
<tag k="name" v="Stop"/>
</node>
<relation id="130901" version="23" timestamp="2015-06-02T04:13:19Z">
<member type="node" ref="130101" role="stop"/>
<tag k="route" v="bus"/>
<tag k="type" v="bus_route"/> <!-- invalid type -->
</relation>
</osm>

16
test/route_relation_mapping.yml
View File

@ -1,13 +1,7 @@
tags:
load_all: true
exclude:
- created_by
- source
tables:
master_routes:
type: relation_member
fields:
columns:
- name: osm_id
type: id
- name: member
@ -27,11 +21,12 @@ tables:
- key: name
name: name
type: string
relation_types: [route_master]
mapping:
route_master: [bus]
route_members:
type: relation_member
fields:
columns:
- name: osm_id
type: id
- key: ref
@ -54,11 +49,12 @@ tables:
key: name
type: string
from_member: true
relation_types: [route]
mapping:
route: [bus, tram, rail]
routes:
type: relation
fields:
columns:
- name: osm_id
type: id
- key: ref
@ -66,5 +62,7 @@ tables:
type: string
- name: tags
type: hstore_tags
relation_types: [route, route_master]
mapping:
route_master: [bus, tram, rail]
route: [bus, tram, rail]

19
test/route_relation_test.go
View File

@ -58,6 +58,23 @@ func TestRouteRelation_RelationData(t *testing.T) {
if r.tags["name"] != "Bus 301: A => B" {
t.Error(r)
}
// check tags of master relation
r = ts.queryTags(t, "osm_routes", -100911)
if r.tags["name"] != "Bus 301" {
t.Error(r)
}
}
func TestRouteRelation_MemberUpdatedByNode1(t *testing.T) {
// check that member is updated after node was modified
rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101")
if len(rows) != 1 {
t.Fatal(rows)
}
if rows[0]["name"] != "Stop" {
t.Error(rows[0])
}
}
func TestRouteRelation_MemberGeomUpdated1(t *testing.T) {
@ -131,7 +148,7 @@ func TestRouteRelation_MemberGeomUpdated2(t *testing.T) {
}
func TestRouteRelation_MemberUpdatedByNode(t *testing.T) {
func TestRouteRelation_MemberUpdatedByNode2(t *testing.T) {
// check that member is updated after node was modified
rows := ts.queryDynamic(t, "osm_route_members", "osm_id = -110901 AND member = 110101")
if len(rows) != 1 {

2
test/single_table_mapping.json
View File

@ -9,7 +9,7 @@
"use_single_id_space": true,
"tables": {
"all": {
"fields": [
"columns": [
{
"type": "id",
"name": "osm_id",

4
test/single_table_test.go
View File

@ -159,7 +159,7 @@ func TestSingleTable_DuplicateIds1(t *testing.T) {
}
assertHstore(t, []checkElem{
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}},
})
assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
}
@ -185,7 +185,7 @@ func TestSingleTable_DuplicateIds2(t *testing.T) {
}
assertHstore(t, []checkElem{
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes"}},
{"osm_all", RelOffset - 31101, "*", map[string]string{"building": "yes", "type": "multipolygon"}},
})
assertGeomType(t, checkElem{"osm_all", RelOffset - 31101, "*", nil}, "Polygon")
}

14
update/process.go
View File

@ -119,7 +119,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err := database.Open(dbConf, tagmapping)
db, err := database.Open(dbConf, &tagmapping.Conf)
if err != nil {
return errors.New("database open: " + err.Error())
}
@ -144,7 +144,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
delDb,
osmCache,
diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
tagmapping.PointMatcher(),
tagmapping.LineStringMatcher(),
tagmapping.PolygonMatcher(),
@ -162,7 +162,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
@ -174,7 +174,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(),
@ -350,11 +350,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
}
// insert new relation
progress.AddRelations(1)
// filter out unsupported relation types, otherwise they might
// get inserted with the tags from an outer way
if relTagFilter.Filter(&rel.Tags) {
relations <- rel
}
relations <- rel
}
for wayId, _ := range wayIds {

14
writer/relations.go
View File

@ -134,8 +134,12 @@ NextRel:
}
func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geos) bool {
// prepare relation first (build rings and compute actual
// relation tags)
matches := rw.polygonMatcher.MatchRelation(r)
if matches == nil {
return false
}
// prepare relation (build rings)
prepedRel, err := geomp.PrepareRelation(r, rw.srid, rw.maxGap)
if err != nil {
if errl, ok := err.(ErrorLevel); !ok || errl.Level() > 0 {
@ -144,12 +148,6 @@ func handleMultiPolygon(rw *RelationWriter, r *element.Relation, geos *geosp.Geo
return false
}
// check for matches befor building the geometry
matches := rw.polygonMatcher.MatchRelation(r)
if matches == nil {
return false
}
// build the multipolygon
geom, err := prepedRel.Build()
if geom.Geom != nil {