434 lines
11 KiB
Go
434 lines
11 KiB
Go
package mapping
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"regexp"
|
|
|
|
"github.com/omniscale/imposm3/element"
|
|
"github.com/omniscale/imposm3/mapping/config"
|
|
|
|
"github.com/pkg/errors"
|
|
"gopkg.in/yaml.v2"
|
|
)
|
|
|
|
type orderedDestTable struct {
|
|
DestTable
|
|
order int
|
|
}
|
|
|
|
type TagTableMapping map[Key]map[Value][]orderedDestTable
|
|
|
|
func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) {
|
|
for key, vals := range mapping {
|
|
for _, v := range vals {
|
|
vals, ok := tt[Key(key)]
|
|
tbl := orderedDestTable{DestTable: table, order: v.Order}
|
|
if ok {
|
|
vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl)
|
|
} else {
|
|
tt[Key(key)] = make(map[Value][]orderedDestTable)
|
|
tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (tt TagTableMapping) asTagMap() tagMap {
|
|
result := make(tagMap)
|
|
for k, vals := range tt {
|
|
result[k] = make(map[Value]struct{})
|
|
for v := range vals {
|
|
result[k][v] = struct{}{}
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
type DestTable struct {
|
|
Name string
|
|
SubMapping string
|
|
}
|
|
|
|
type TableType string
|
|
|
|
func (tt *TableType) UnmarshalJSON(data []byte) error {
|
|
switch string(data) {
|
|
case "":
|
|
return errors.New("missing table type")
|
|
case `"point"`:
|
|
*tt = PointTable
|
|
case `"linestring"`:
|
|
*tt = LineStringTable
|
|
case `"polygon"`:
|
|
*tt = PolygonTable
|
|
case `"geometry"`:
|
|
*tt = GeometryTable
|
|
case `"relation"`:
|
|
*tt = RelationTable
|
|
case `"relation_member"`:
|
|
*tt = RelationMemberTable
|
|
}
|
|
return errors.New("unknown type " + string(data))
|
|
}
|
|
|
|
const (
|
|
PolygonTable TableType = "polygon"
|
|
LineStringTable TableType = "linestring"
|
|
PointTable TableType = "point"
|
|
GeometryTable TableType = "geometry"
|
|
RelationTable TableType = "relation"
|
|
RelationMemberTable TableType = "relation_member"
|
|
)
|
|
|
|
type Mapping struct {
|
|
Conf config.Mapping
|
|
PointMatcher NodeMatcher
|
|
LineStringMatcher WayMatcher
|
|
PolygonMatcher RelWayMatcher
|
|
RelationMatcher RelationMatcher
|
|
RelationMemberMatcher RelationMatcher
|
|
}
|
|
|
|
func FromFile(filename string) (*Mapping, error) {
|
|
b, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return New(b)
|
|
}
|
|
|
|
func New(b []byte) (*Mapping, error) {
|
|
mapping := Mapping{}
|
|
err := yaml.Unmarshal(b, &mapping.Conf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = mapping.prepare()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = mapping.createMatcher()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &mapping, nil
|
|
}
|
|
|
|
func (m *Mapping) prepare() error {
|
|
for name, t := range m.Conf.Tables {
|
|
t.Name = name
|
|
if t.OldFields != nil {
|
|
// todo deprecate 'fields'
|
|
t.Columns = t.OldFields
|
|
}
|
|
if t.Type == "" {
|
|
return errors.Errorf("missing type for table %s", name)
|
|
}
|
|
|
|
if TableType(t.Type) == GeometryTable {
|
|
if t.Mapping != nil || t.Mappings != nil {
|
|
return errors.Errorf("table with type:geometry requires type_mapping for table %s", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
for name, t := range m.Conf.GeneralizedTables {
|
|
t.Name = name
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Mapping) createMatcher() error {
|
|
var err error
|
|
m.PointMatcher, err = m.pointMatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.LineStringMatcher, err = m.lineStringMatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.PolygonMatcher, err = m.polygonMatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.RelationMatcher, err = m.relationMatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m.RelationMemberMatcher, err = m.relationMemberMatcher()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
|
|
for name, t := range m.Conf.Tables {
|
|
if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
|
|
continue
|
|
}
|
|
mappings.addFromMapping(t.Mapping, DestTable{Name: name})
|
|
|
|
for subMappingName, subMapping := range t.Mappings {
|
|
mappings.addFromMapping(subMapping.Mapping, DestTable{Name: name, SubMapping: subMappingName})
|
|
}
|
|
|
|
switch tableType {
|
|
case PointTable:
|
|
mappings.addFromMapping(t.TypeMappings.Points, DestTable{Name: name})
|
|
case LineStringTable:
|
|
mappings.addFromMapping(t.TypeMappings.LineStrings, DestTable{Name: name})
|
|
case PolygonTable:
|
|
mappings.addFromMapping(t.TypeMappings.Polygons, DestTable{Name: name})
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *Mapping) tables(tableType TableType) (map[string]*rowBuilder, error) {
|
|
var err error
|
|
result := make(map[string]*rowBuilder)
|
|
for name, t := range m.Conf.Tables {
|
|
if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable {
|
|
result[name], err = makeRowBuilder(t)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "creating row builder for %s", name)
|
|
}
|
|
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func makeRowBuilder(tbl *config.Table) (*rowBuilder, error) {
|
|
result := rowBuilder{}
|
|
|
|
for _, mappingColumn := range tbl.Columns {
|
|
column := valueBuilder{}
|
|
column.key = Key(mappingColumn.Key)
|
|
|
|
columnType, err := MakeColumnType(mappingColumn)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "creating column %s", mappingColumn.Name)
|
|
}
|
|
column.colType = *columnType
|
|
result.columns = append(result.columns, column)
|
|
}
|
|
return &result, nil
|
|
}
|
|
|
|
func MakeColumnType(c *config.Column) (*ColumnType, error) {
|
|
columnType, ok := AvailableColumnTypes[c.Type]
|
|
if !ok {
|
|
return nil, errors.Errorf("unhandled type %s", c.Type)
|
|
}
|
|
|
|
if columnType.MakeFunc != nil {
|
|
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
|
|
}
|
|
columnType.FromMember = c.FromMember
|
|
return &columnType, nil
|
|
}
|
|
|
|
func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
|
|
for _, t := range m.Conf.Tables {
|
|
if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable {
|
|
continue
|
|
}
|
|
|
|
for _, col := range t.Columns {
|
|
if col.Key != "" {
|
|
tags[Key(col.Key)] = true
|
|
}
|
|
for _, k := range col.Keys {
|
|
tags[Key(k)] = true
|
|
}
|
|
}
|
|
|
|
if t.Filters != nil && t.Filters.ExcludeTags != nil {
|
|
for _, keyVal := range *t.Filters.ExcludeTags {
|
|
tags[Key(keyVal[0])] = true
|
|
}
|
|
}
|
|
|
|
if tableType == PolygonTable || tableType == RelationTable || tableType == RelationMemberTable {
|
|
if t.RelationTypes != nil {
|
|
tags["type"] = true
|
|
}
|
|
}
|
|
}
|
|
for _, k := range m.Conf.Tags.Include {
|
|
tags[Key(k)] = true
|
|
}
|
|
|
|
// always include area tag for closed-way handling
|
|
tags["area"] = true
|
|
}
|
|
|
|
type elementFilter func(tags element.Tags, key Key, closed bool) bool
|
|
|
|
type tableElementFilters map[string][]elementFilter
|
|
|
|
func (m *Mapping) getAreaTags() (map[Key]struct{}, map[Key]struct{}) {
|
|
var areaTags map[Key]struct{}
|
|
var linearTags map[Key]struct{}
|
|
if m.Conf.Areas.AreaTags != nil {
|
|
areaTags = make(map[Key]struct{})
|
|
for _, tag := range m.Conf.Areas.AreaTags {
|
|
areaTags[Key(tag)] = struct{}{}
|
|
}
|
|
}
|
|
if m.Conf.Areas.LinearTags != nil {
|
|
linearTags = make(map[Key]struct{})
|
|
for _, tag := range m.Conf.Areas.LinearTags {
|
|
linearTags[Key(tag)] = struct{}{}
|
|
}
|
|
}
|
|
return areaTags, linearTags
|
|
}
|
|
|
|
func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
|
|
for name, t := range m.Conf.Tables {
|
|
if t.RelationTypes != nil {
|
|
relTypes := t.RelationTypes // copy loop var for closure
|
|
f := func(tags element.Tags, key Key, closed bool) bool {
|
|
if v, ok := tags["type"]; ok {
|
|
for _, rtype := range relTypes {
|
|
if v == rtype {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
filters[name] = append(filters[name], f)
|
|
} else {
|
|
if TableType(t.Type) == PolygonTable {
|
|
// standard multipolygon handling (boundary and land_area are for backwards compatibility)
|
|
f := func(tags element.Tags, key Key, closed bool) bool {
|
|
if v, ok := tags["type"]; ok {
|
|
if v == "multipolygon" || v == "boundary" || v == "land_area" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
filters[name] = append(filters[name], f)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *Mapping) addFilters(filters tableElementFilters) {
|
|
for name, t := range m.Conf.Tables {
|
|
if t.Filters == nil {
|
|
continue
|
|
}
|
|
if t.Filters.ExcludeTags != nil {
|
|
log.Print("warn: exclude_tags filter is deprecated and will be removed. See require and reject filter.")
|
|
for _, filterKeyVal := range *t.Filters.ExcludeTags {
|
|
// Convert `exclude_tags`` filter to `reject` filter !
|
|
keyname := filterKeyVal[0]
|
|
vararr := []config.OrderedValue{
|
|
{
|
|
Value: config.Value(filterKeyVal[1]),
|
|
Order: 1,
|
|
},
|
|
}
|
|
filters[name] = append(filters[name], makeFiltersFunction(name, false, true, keyname, vararr))
|
|
|
|
}
|
|
}
|
|
|
|
if t.Filters.Require != nil {
|
|
for keyname, vararr := range t.Filters.Require {
|
|
filters[name] = append(filters[name], makeFiltersFunction(name, true, false, string(keyname), vararr))
|
|
}
|
|
}
|
|
|
|
if t.Filters.Reject != nil {
|
|
for keyname, vararr := range t.Filters.Reject {
|
|
filters[name] = append(filters[name], makeFiltersFunction(name, false, true, string(keyname), vararr))
|
|
}
|
|
}
|
|
|
|
if t.Filters.RequireRegexp != nil {
|
|
for keyname, regexp := range t.Filters.RequireRegexp {
|
|
filters[name] = append(filters[name], makeRegexpFiltersFunction(name, true, false, string(keyname), regexp))
|
|
}
|
|
}
|
|
|
|
if t.Filters.RejectRegexp != nil {
|
|
for keyname, regexp := range t.Filters.RejectRegexp {
|
|
filters[name] = append(filters[name], makeRegexpFiltersFunction(name, false, true, string(keyname), regexp))
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func findValueInOrderedValue(v config.Value, list []config.OrderedValue) bool {
|
|
for _, item := range list {
|
|
if item.Value == v {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func makeRegexpFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, vKeyname string, vRegexp string) func(tags element.Tags, key Key, closed bool) bool {
|
|
// Compile regular expression, if not valid regexp --> panic !
|
|
r := regexp.MustCompile(vRegexp)
|
|
return func(tags element.Tags, key Key, closed bool) bool {
|
|
if v, ok := tags[vKeyname]; ok {
|
|
if r.MatchString(v) {
|
|
return virtualTrue
|
|
}
|
|
}
|
|
return virtualFalse
|
|
}
|
|
}
|
|
|
|
func makeFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, vKeyname string, vVararr []config.OrderedValue) func(tags element.Tags, key Key, closed bool) bool {
|
|
|
|
if findValueInOrderedValue("__nil__", vVararr) { // check __nil__
|
|
log.Print("warn: Filter value '__nil__' is not supported ! (tablename:" + tablename + ")")
|
|
}
|
|
|
|
if findValueInOrderedValue("__any__", vVararr) { // check __any__
|
|
if len(vVararr) > 1 {
|
|
log.Print("warn: Multiple filter value with '__any__' keywords is not valid! (tablename:" + tablename + ")")
|
|
}
|
|
return func(tags element.Tags, key Key, closed bool) bool {
|
|
if _, ok := tags[vKeyname]; ok {
|
|
return virtualTrue
|
|
}
|
|
return virtualFalse
|
|
}
|
|
} else if len(vVararr) == 1 { // IF 1 parameter THEN we can generate optimal code
|
|
return func(tags element.Tags, key Key, closed bool) bool {
|
|
if v, ok := tags[vKeyname]; ok {
|
|
if config.Value(v) == vVararr[0].Value {
|
|
return virtualTrue
|
|
}
|
|
}
|
|
return virtualFalse
|
|
}
|
|
} else { // > 1 parameter - less optimal code
|
|
return func(tags element.Tags, key Key, closed bool) bool {
|
|
if v, ok := tags[vKeyname]; ok {
|
|
if findValueInOrderedValue(config.Value(v), vVararr) {
|
|
return virtualTrue
|
|
}
|
|
}
|
|
return virtualFalse
|
|
}
|
|
}
|
|
}
|