2013-05-14 12:04:11 +04:00
|
|
|
package mapping
|
2013-05-13 15:58:44 +04:00
|
|
|
|
|
|
|
import (
|
2013-05-29 10:47:26 +04:00
|
|
|
"errors"
|
2015-09-25 14:36:23 +03:00
|
|
|
"fmt"
|
2015-08-11 20:22:01 +03:00
|
|
|
"io/ioutil"
|
2014-08-11 12:18:08 +04:00
|
|
|
|
|
|
|
"github.com/omniscale/imposm3/element"
|
2015-08-11 20:22:01 +03:00
|
|
|
|
|
|
|
"gopkg.in/yaml.v2"
|
2013-05-13 15:58:44 +04:00
|
|
|
)
|
|
|
|
|
|
|
|
type Field struct {
|
2016-01-05 11:40:30 +03:00
|
|
|
Name string `yaml:"name"`
|
|
|
|
Key Key `yaml:"key"`
|
|
|
|
Keys []Key `yaml:"keys"`
|
|
|
|
Type string `yaml:"type"`
|
|
|
|
Args map[string]interface{} `yaml:"args"`
|
|
|
|
FromMember bool `yaml:"from_member"`
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
type Table struct {
|
2014-06-19 13:51:15 +04:00
|
|
|
Name string
|
2015-08-11 20:22:01 +03:00
|
|
|
Type TableType `yaml:"type"`
|
2015-09-25 14:36:23 +03:00
|
|
|
Mapping KeyValues `yaml:"mapping"`
|
2015-08-11 20:22:01 +03:00
|
|
|
Mappings map[string]SubMapping `yaml:"mappings"`
|
|
|
|
TypeMappings TypeMappings `yaml:"type_mappings"`
|
|
|
|
Fields []*Field `yaml:"columns"` // TODO rename Fields internaly to Columns
|
|
|
|
OldFields []*Field `yaml:"fields"`
|
|
|
|
Filters *Filters `yaml:"filters"`
|
2013-05-17 17:07:03 +04:00
|
|
|
}
|
|
|
|
|
2013-05-22 13:48:34 +04:00
|
|
|
type GeneralizedTable struct {
|
|
|
|
Name string
|
2015-08-11 20:22:01 +03:00
|
|
|
SourceTableName string `yaml:"source"`
|
|
|
|
Tolerance float64 `yaml:"tolerance"`
|
|
|
|
SqlFilter string `yaml:"sql_filter"`
|
2013-05-22 13:48:34 +04:00
|
|
|
}
|
|
|
|
|
2013-05-17 17:07:03 +04:00
|
|
|
type Filters struct {
|
2015-08-11 20:22:01 +03:00
|
|
|
ExcludeTags *[][]string `yaml:"exclude_tags"`
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
|
2013-05-14 18:15:35 +04:00
|
|
|
type Tables map[string]*Table
|
2013-05-13 15:58:44 +04:00
|
|
|
|
2013-05-22 13:48:34 +04:00
|
|
|
type GeneralizedTables map[string]*GeneralizedTable
|
|
|
|
|
2013-05-13 15:58:44 +04:00
|
|
|
type Mapping struct {
|
2015-08-11 20:22:01 +03:00
|
|
|
Tables Tables `yaml:"tables"`
|
|
|
|
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
|
|
|
|
Tags Tags `yaml:"tags"`
|
2016-11-10 13:00:34 +03:00
|
|
|
Areas Areas `yaml:"areas"`
|
2014-08-11 12:18:08 +04:00
|
|
|
// SingleIdSpace mangles the overlapping node/way/relation IDs
|
|
|
|
// to be unique (nodes positive, ways negative, relations negative -1e17)
|
2015-09-25 14:36:23 +03:00
|
|
|
SingleIdSpace bool `yaml:"use_single_id_space"`
|
2014-06-19 13:51:15 +04:00
|
|
|
}
|
|
|
|
|
2016-11-10 13:00:34 +03:00
|
|
|
type Areas struct {
|
|
|
|
AreaTags []Key `yaml:"area_tags"`
|
|
|
|
LinearTags []Key `yaml:"linear_tags"`
|
|
|
|
}
|
|
|
|
|
2014-06-19 13:51:15 +04:00
|
|
|
type Tags struct {
|
2015-08-11 20:22:01 +03:00
|
|
|
LoadAll bool `yaml:"load_all"`
|
|
|
|
Exclude []Key `yaml:"exclude"`
|
2016-11-16 18:25:48 +03:00
|
|
|
Include []Key `yaml:"include"`
|
2014-06-19 13:51:15 +04:00
|
|
|
}
|
|
|
|
|
2015-09-25 14:36:23 +03:00
|
|
|
type orderedValue struct {
|
|
|
|
value Value
|
|
|
|
order int
|
|
|
|
}
|
|
|
|
type KeyValues map[Key][]orderedValue
|
|
|
|
|
|
|
|
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|
|
|
if *kv == nil {
|
|
|
|
*kv = make(map[Key][]orderedValue)
|
|
|
|
}
|
|
|
|
slice := yaml.MapSlice{}
|
|
|
|
err := unmarshal(&slice)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
order := 0
|
|
|
|
for _, item := range slice {
|
|
|
|
k, ok := item.Key.(string)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("mapping key '%s' not a string", k)
|
|
|
|
}
|
|
|
|
values, ok := item.Value.([]interface{})
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("mapping key '%s' not a string", k)
|
|
|
|
}
|
|
|
|
for _, v := range values {
|
|
|
|
if v, ok := v.(string); ok {
|
|
|
|
(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order})
|
|
|
|
} else {
|
|
|
|
return fmt.Errorf("mapping value '%s' not a string", v)
|
|
|
|
}
|
|
|
|
order += 1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2014-06-19 13:51:15 +04:00
|
|
|
type SubMapping struct {
|
2015-09-25 14:36:23 +03:00
|
|
|
Mapping KeyValues
|
2014-06-19 13:51:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
type TypeMappings struct {
|
2015-09-25 14:36:23 +03:00
|
|
|
Points KeyValues `yaml:"points"`
|
|
|
|
LineStrings KeyValues `yaml:"linestrings"`
|
|
|
|
Polygons KeyValues `yaml:"polygons"`
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
|
2016-11-10 13:00:34 +03:00
|
|
|
type ElementFilter func(tags element.Tags, key Key, closed bool) bool
|
2013-05-17 17:44:50 +04:00
|
|
|
|
2015-09-25 14:36:23 +03:00
|
|
|
type TagTables map[Key]map[Value][]OrderedDestTable
|
2013-05-27 13:24:22 +04:00
|
|
|
|
|
|
|
type DestTable struct {
|
|
|
|
Name string
|
|
|
|
SubMapping string
|
|
|
|
}
|
2013-05-17 17:44:50 +04:00
|
|
|
|
2015-09-25 14:36:23 +03:00
|
|
|
type OrderedDestTable struct {
|
|
|
|
DestTable
|
|
|
|
order int
|
|
|
|
}
|
|
|
|
|
2013-10-28 11:41:24 +04:00
|
|
|
type TableType string
|
|
|
|
|
2014-06-19 13:51:15 +04:00
|
|
|
func (tt *TableType) UnmarshalJSON(data []byte) error {
|
|
|
|
switch string(data) {
|
|
|
|
case "":
|
|
|
|
return errors.New("missing table type")
|
|
|
|
case `"point"`:
|
|
|
|
*tt = PointTable
|
|
|
|
case `"linestring"`:
|
|
|
|
*tt = LineStringTable
|
|
|
|
case `"polygon"`:
|
|
|
|
*tt = PolygonTable
|
|
|
|
case `"geometry"`:
|
|
|
|
*tt = GeometryTable
|
2016-01-04 13:19:28 +03:00
|
|
|
case `"relation"`:
|
|
|
|
*tt = RelationTable
|
|
|
|
case `"relation_member"`:
|
|
|
|
*tt = RelationMemberTable
|
2014-06-19 13:51:15 +04:00
|
|
|
default:
|
|
|
|
return errors.New("unknown type " + string(data))
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2013-10-28 11:41:24 +04:00
|
|
|
const (
|
2016-01-04 13:19:28 +03:00
|
|
|
PolygonTable TableType = "polygon"
|
|
|
|
LineStringTable TableType = "linestring"
|
|
|
|
PointTable TableType = "point"
|
|
|
|
GeometryTable TableType = "geometry"
|
|
|
|
RelationTable TableType = "relation"
|
|
|
|
RelationMemberTable TableType = "relation_member"
|
2013-10-28 11:41:24 +04:00
|
|
|
)
|
|
|
|
|
2013-05-17 17:44:50 +04:00
|
|
|
func NewMapping(filename string) (*Mapping, error) {
|
2015-08-11 20:22:01 +03:00
|
|
|
f, err := ioutil.ReadFile(filename)
|
2013-05-17 17:44:50 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
mapping := Mapping{}
|
2015-08-11 20:22:01 +03:00
|
|
|
err = yaml.Unmarshal(f, &mapping)
|
2013-05-17 17:44:50 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2013-05-29 10:47:26 +04:00
|
|
|
err = mapping.prepare()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-05-17 17:44:50 +04:00
|
|
|
return &mapping, nil
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
|
2014-04-30 18:33:07 +04:00
|
|
|
func (t *Table) ExtraTags() map[Key]bool {
|
|
|
|
tags := make(map[Key]bool)
|
2013-05-13 15:58:44 +04:00
|
|
|
for _, field := range t.Fields {
|
2013-05-15 10:15:33 +04:00
|
|
|
if field.Key != "" {
|
|
|
|
tags[field.Key] = true
|
|
|
|
}
|
2014-08-18 11:24:17 +04:00
|
|
|
for _, k := range field.Keys {
|
|
|
|
tags[k] = true
|
|
|
|
}
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
return tags
|
|
|
|
}
|
|
|
|
|
2013-05-29 10:47:26 +04:00
|
|
|
func (m *Mapping) prepare() error {
|
2013-05-14 18:15:35 +04:00
|
|
|
for name, t := range m.Tables {
|
|
|
|
t.Name = name
|
2014-06-24 15:54:10 +04:00
|
|
|
if t.OldFields != nil {
|
|
|
|
// todo deprecate 'fields'
|
|
|
|
t.Fields = t.OldFields
|
|
|
|
}
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
2014-06-19 13:51:15 +04:00
|
|
|
|
2013-05-22 13:48:34 +04:00
|
|
|
for name, t := range m.GeneralizedTables {
|
|
|
|
t.Name = name
|
|
|
|
}
|
2013-05-29 10:47:26 +04:00
|
|
|
return nil
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
|
2015-09-25 14:36:23 +03:00
|
|
|
func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) {
|
2013-05-27 13:24:22 +04:00
|
|
|
for key, vals := range mapping {
|
|
|
|
for _, v := range vals {
|
|
|
|
vals, ok := tt[key]
|
2015-09-25 14:36:23 +03:00
|
|
|
tbl := OrderedDestTable{DestTable: table, order: v.order}
|
2013-05-27 13:24:22 +04:00
|
|
|
if ok {
|
2015-09-25 14:36:23 +03:00
|
|
|
vals[v.value] = append(vals[v.value], tbl)
|
2013-05-27 13:24:22 +04:00
|
|
|
} else {
|
2015-09-25 14:36:23 +03:00
|
|
|
tt[key] = make(map[Value][]OrderedDestTable)
|
|
|
|
tt[key][v.value] = append(tt[key][v.value], tbl)
|
2013-05-27 13:24:22 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-28 11:41:24 +04:00
|
|
|
func (m *Mapping) mappings(tableType TableType, mappings TagTables) {
|
2013-05-13 15:58:44 +04:00
|
|
|
for name, t := range m.Tables {
|
2014-06-19 13:51:15 +04:00
|
|
|
if t.Type != GeometryTable && t.Type != tableType {
|
2013-05-13 15:58:44 +04:00
|
|
|
continue
|
|
|
|
}
|
2015-09-25 14:36:23 +03:00
|
|
|
mappings.addFromMapping(t.Mapping, DestTable{Name: name})
|
2013-05-27 13:24:22 +04:00
|
|
|
|
|
|
|
for subMappingName, subMapping := range t.Mappings {
|
2015-09-25 14:36:23 +03:00
|
|
|
mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
2014-06-19 13:51:15 +04:00
|
|
|
|
|
|
|
switch tableType {
|
|
|
|
case PointTable:
|
2015-09-25 14:36:23 +03:00
|
|
|
mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
|
2014-06-19 13:51:15 +04:00
|
|
|
case LineStringTable:
|
2015-09-25 14:36:23 +03:00
|
|
|
mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
|
2014-06-19 13:51:15 +04:00
|
|
|
case PolygonTable:
|
2015-09-25 14:36:23 +03:00
|
|
|
mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
|
2014-06-19 13:51:15 +04:00
|
|
|
}
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-28 11:41:24 +04:00
|
|
|
func (m *Mapping) tables(tableType TableType) map[string]*TableFields {
|
2013-05-14 18:15:35 +04:00
|
|
|
result := make(map[string]*TableFields)
|
|
|
|
for name, t := range m.Tables {
|
2014-06-19 13:51:15 +04:00
|
|
|
if t.Type == tableType || t.Type == "geometry" {
|
2013-05-14 18:15:35 +04:00
|
|
|
result[name] = t.TableFields()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
2014-04-30 18:33:07 +04:00
|
|
|
func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
|
2013-05-13 15:58:44 +04:00
|
|
|
for _, t := range m.Tables {
|
2015-11-21 20:14:26 +03:00
|
|
|
if t.Type != tableType && t.Type != "geometry" {
|
2013-05-13 15:58:44 +04:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
for key, _ := range t.ExtraTags() {
|
|
|
|
tags[key] = true
|
|
|
|
}
|
2013-05-17 17:07:03 +04:00
|
|
|
if t.Filters != nil && t.Filters.ExcludeTags != nil {
|
2013-05-27 13:24:22 +04:00
|
|
|
for _, keyVal := range *t.Filters.ExcludeTags {
|
2014-04-30 18:33:07 +04:00
|
|
|
tags[Key(keyVal[0])] = true
|
2013-05-17 17:07:03 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-11-16 18:25:48 +03:00
|
|
|
for _, k := range m.Tags.Include {
|
|
|
|
tags[k] = true
|
|
|
|
}
|
|
|
|
|
2016-11-10 13:00:34 +03:00
|
|
|
// always include area tag for closed-way handling
|
|
|
|
tags["area"] = true
|
2013-05-17 17:07:03 +04:00
|
|
|
}
|
|
|
|
|
2013-05-17 17:44:50 +04:00
|
|
|
func (m *Mapping) ElementFilters() map[string][]ElementFilter {
|
|
|
|
result := make(map[string][]ElementFilter)
|
2016-11-10 13:00:34 +03:00
|
|
|
|
|
|
|
var areaTags map[Key]struct{}
|
|
|
|
var linearTags map[Key]struct{}
|
|
|
|
if m.Areas.AreaTags != nil {
|
|
|
|
areaTags = make(map[Key]struct{})
|
|
|
|
for _, tag := range m.Areas.AreaTags {
|
|
|
|
areaTags[tag] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if m.Areas.LinearTags != nil {
|
|
|
|
linearTags = make(map[Key]struct{})
|
|
|
|
for _, tag := range m.Areas.LinearTags {
|
|
|
|
linearTags[tag] = struct{}{}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-05-17 17:07:03 +04:00
|
|
|
for name, t := range m.Tables {
|
2016-11-10 13:00:34 +03:00
|
|
|
if t.Type == LineStringTable && areaTags != nil {
|
|
|
|
f := func(tags element.Tags, key Key, closed bool) bool {
|
|
|
|
if closed {
|
|
|
|
if tags["area"] == "yes" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if tags["area"] != "no" {
|
|
|
|
if _, ok := areaTags[key]; ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
result[name] = append(result[name], f)
|
|
|
|
}
|
|
|
|
if t.Type == PolygonTable && linearTags != nil {
|
|
|
|
f := func(tags element.Tags, key Key, closed bool) bool {
|
|
|
|
if closed && tags["area"] == "no" {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if tags["area"] != "yes" {
|
|
|
|
if _, ok := linearTags[key]; ok {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
result[name] = append(result[name], f)
|
|
|
|
}
|
|
|
|
|
2013-05-17 17:07:03 +04:00
|
|
|
if t.Filters == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if t.Filters.ExcludeTags != nil {
|
2013-05-27 13:24:22 +04:00
|
|
|
for _, filterKeyVal := range *t.Filters.ExcludeTags {
|
2016-11-10 13:00:34 +03:00
|
|
|
f := func(tags element.Tags, key Key, closed bool) bool {
|
|
|
|
if v, ok := tags[filterKeyVal[0]]; ok {
|
2013-05-27 13:24:22 +04:00
|
|
|
if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
|
2013-05-17 17:07:03 +04:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
result[name] = append(result[name], f)
|
|
|
|
}
|
|
|
|
}
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|
2013-05-17 17:07:03 +04:00
|
|
|
return result
|
2013-05-13 15:58:44 +04:00
|
|
|
}
|