From a30442d254c5bf44ba7ca77ec16ca83ab1ea5562 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Thu, 11 May 2017 09:37:12 +0200 Subject: [PATCH] refactor code structure --- mapping/columns.go | 16 ------- mapping/config.go | 106 ++++++++++++++++++++++++++++----------------- mapping/filter.go | 106 +++++++++++++++++++++++---------------------- mapping/matcher.go | 54 +++++++++++------------ 4 files changed, 147 insertions(+), 135 deletions(-) diff --git a/mapping/columns.go b/mapping/columns.go index 26691ed..68efc6e 100644 --- a/mapping/columns.go +++ b/mapping/columns.go @@ -100,22 +100,6 @@ func (t *TableSpec) MakeMemberRow(rel *element.Relation, member *element.Member, return row } -func (c *Column) ColumnType() *ColumnType { - if columnType, ok := AvailableColumnTypes[c.Type]; ok { - if columnType.MakeFunc != nil { - makeValue, err := columnType.MakeFunc(c.Name, columnType, *c) - if err != nil { - log.Print(err) - return nil - } - columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember} - } - columnType.FromMember = c.FromMember - return &columnType - } - return nil -} - func (t *Table) TableSpec() *TableSpec { result := TableSpec{} diff --git a/mapping/config.go b/mapping/config.go index 4168f72..5aefd1f 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -10,6 +10,16 @@ import ( "gopkg.in/yaml.v2" ) +type Mapping struct { + Tables Tables `yaml:"tables"` + GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` + Tags Tags `yaml:"tags"` + Areas Areas `yaml:"areas"` + // SingleIdSpace mangles the overlapping node/way/relation IDs + // to be unique (nodes positive, ways negative, relations negative -1e17) + SingleIdSpace bool `yaml:"use_single_id_space"` +} + type Column struct { Name string `yaml:"name"` Key Key `yaml:"key"` @@ -19,6 +29,23 @@ type Column struct { FromMember bool `yaml:"from_member"` } +func (c *Column) ColumnType() *ColumnType { + if columnType, ok := AvailableColumnTypes[c.Type]; ok { + if columnType.MakeFunc != nil { + makeValue, err := columnType.MakeFunc(c.Name, columnType, *c) + if err != nil { + log.Print(err) + return nil + } + columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember} + } + columnType.FromMember = c.FromMember + return &columnType + } + return nil +} + +type Tables map[string]*Table type Table struct { Name string Type TableType `yaml:"type"` @@ -31,6 +58,7 @@ type Table struct { RelationTypes []string `yaml:"relation_types"` } +type GeneralizedTables map[string]*GeneralizedTable type GeneralizedTable struct { Name string SourceTableName string `yaml:"source"` @@ -42,20 +70,6 @@ type Filters struct { ExcludeTags *[][]string `yaml:"exclude_tags"` } -type Tables map[string]*Table - -type GeneralizedTables map[string]*GeneralizedTable - -type Mapping struct { - Tables Tables `yaml:"tables"` - GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` - Tags Tags `yaml:"tags"` - Areas Areas `yaml:"areas"` - // SingleIdSpace mangles the overlapping node/way/relation IDs - // to be unique (nodes positive, ways negative, relations negative -1e17) - SingleIdSpace bool `yaml:"use_single_id_space"` -} - type Areas struct { AreaTags []Key `yaml:"area_tags"` LinearTags []Key `yaml:"linear_tags"` @@ -71,6 +85,7 @@ type orderedValue struct { value Value order int } + type KeyValues map[Key][]orderedValue func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -116,18 +131,44 @@ type TypeMappings struct { type ElementFilter func(tags element.Tags, key Key, closed bool) bool -type TagTables map[Key]map[Value][]OrderedDestTable +type orderedDestTable struct { + DestTable + order int +} + +type TagTableMapping map[Key]map[Value][]orderedDestTable + +func (tt TagTableMapping) addFromMapping(mapping KeyValues, table DestTable) { + for key, vals := range mapping { + for _, v := range vals { + vals, ok := tt[key] + tbl := orderedDestTable{DestTable: table, order: v.order} + if ok { + vals[v.value] = append(vals[v.value], tbl) + } else { + tt[key] = make(map[Value][]orderedDestTable) + tt[key][v.value] = append(tt[key][v.value], tbl) + } + } + } +} + +func (tt TagTableMapping) asTagMap() tagMap { + result := make(tagMap) + for k, vals := range tt { + result[k] = make(map[Value]struct{}) + for v := range vals { + result[k][v] = struct{}{} + } + } + return result +} type DestTable struct { Name string SubMapping string } -type OrderedDestTable struct { - DestTable - order int -} - type TableType string func (tt *TableType) UnmarshalJSON(data []byte) error { @@ -195,22 +236,7 @@ func (m *Mapping) prepare() error { return nil } -func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) { - for key, vals := range mapping { - for _, v := range vals { - vals, ok := tt[key] - tbl := OrderedDestTable{DestTable: table, order: v.order} - if ok { - vals[v.value] = append(vals[v.value], tbl) - } else { - tt[key] = make(map[Value][]OrderedDestTable) - tt[key][v.value] = append(tt[key][v.value], tbl) - } - } - } -} - -func (m *Mapping) mappings(tableType TableType, mappings TagTables) { +func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) { for name, t := range m.Tables { if t.Type != GeometryTable && t.Type != tableType { continue @@ -277,9 +303,9 @@ func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) { tags["area"] = true } -type tableFilters map[string][]ElementFilter +type tableElementFilters map[string][]ElementFilter -func (m *Mapping) addTypedFilters(tableType TableType, filters tableFilters) { +func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) { var areaTags map[Key]struct{} var linearTags map[Key]struct{} if m.Areas.AreaTags != nil { @@ -332,7 +358,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableFilters) { } } -func (m *Mapping) addRelationFilters(tableType TableType, filters tableFilters) { +func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) { for name, t := range m.Tables { if t.RelationTypes != nil { relTypes := t.RelationTypes // copy loop var for closure @@ -364,7 +390,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableFilters) } } -func (m *Mapping) addFilters(filters tableFilters) { +func (m *Mapping) addFilters(filters tableElementFilters) { for name, t := range m.Tables { if t.Filters == nil { continue diff --git a/mapping/filter.go b/mapping/filter.go index 9cc52e6..fb31580 100644 --- a/mapping/filter.go +++ b/mapping/filter.go @@ -7,42 +7,46 @@ import ( "github.com/omniscale/imposm3/element" ) +type TagFilterer interface { + Filter(tags *element.Tags) +} + func (m *Mapping) NodeTagFilter() TagFilterer { if m.Tags.LoadAll { return newExcludeFilter(m.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) + mappings := make(TagTableMapping) m.mappings(PointTable, mappings) tags := make(map[Key]bool) m.extraTags(PointTable, tags) m.extraTags(RelationMemberTable, tags) - return &TagFilter{mappings, tags} + return &tagFilter{mappings.asTagMap(), tags} } func (m *Mapping) WayTagFilter() TagFilterer { if m.Tags.LoadAll { return newExcludeFilter(m.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) + mappings := make(TagTableMapping) m.mappings(LineStringTable, mappings) m.mappings(PolygonTable, mappings) tags := make(map[Key]bool) m.extraTags(LineStringTable, tags) m.extraTags(PolygonTable, tags) m.extraTags(RelationMemberTable, tags) - return &TagFilter{mappings, tags} + return &tagFilter{mappings.asTagMap(), tags} } func (m *Mapping) RelationTagFilter() TagFilterer { if m.Tags.LoadAll { return newExcludeFilter(m.Tags.Exclude) } - mappings := make(map[Key]map[Value][]OrderedDestTable) + mappings := make(TagTableMapping) // do not filter out type tag for common relations - mappings["type"] = map[Value][]OrderedDestTable{ - "multipolygon": []OrderedDestTable{}, - "boundary": []OrderedDestTable{}, - "land_area": []OrderedDestTable{}, + mappings["type"] = map[Value][]orderedDestTable{ + "multipolygon": []orderedDestTable{}, + "boundary": []orderedDestTable{}, + "land_area": []orderedDestTable{}, } m.mappings(LineStringTable, mappings) m.mappings(PolygonTable, mappings) @@ -53,54 +57,17 @@ func (m *Mapping) RelationTagFilter() TagFilterer { m.extraTags(PolygonTable, tags) m.extraTags(RelationTable, tags) m.extraTags(RelationMemberTable, tags) - return &TagFilter{mappings, tags} + return &tagFilter{mappings.asTagMap(), tags} } -type TagFilter struct { - mappings map[Key]map[Value][]OrderedDestTable +type tagMap map[Key]map[Value]struct{} + +type tagFilter struct { + mappings tagMap extraTags map[Key]bool } -type ExcludeFilter struct { - keys map[Key]struct{} - matches []string -} - -func newExcludeFilter(tags []Key) *ExcludeFilter { - f := ExcludeFilter{ - keys: make(map[Key]struct{}), - matches: make([]string, 0), - } - for _, t := range tags { - if strings.ContainsAny(string(t), "?*[") { - f.matches = append(f.matches, string(t)) - } else { - f.keys[t] = struct{}{} - } - } - return &f -} - -func (f *ExcludeFilter) Filter(tags *element.Tags) { - for k := range *tags { - if _, ok := f.keys[Key(k)]; ok { - delete(*tags, k) - } else if f.matches != nil { - for _, exkey := range f.matches { - if ok, _ := path.Match(exkey, k); ok { - delete(*tags, k) - break - } - } - } - } -} - -type TagFilterer interface { - Filter(tags *element.Tags) -} - -func (f *TagFilter) Filter(tags *element.Tags) { +func (f *tagFilter) Filter(tags *element.Tags) { if tags == nil { return } @@ -119,3 +86,38 @@ func (f *TagFilter) Filter(tags *element.Tags) { } } } + +type excludeFilter struct { + keys map[Key]struct{} + matches []string +} + +func newExcludeFilter(tags []Key) *excludeFilter { + f := excludeFilter{ + keys: make(map[Key]struct{}), + matches: make([]string, 0), + } + for _, t := range tags { + if strings.ContainsAny(string(t), "?*[") { + f.matches = append(f.matches, string(t)) + } else { + f.keys[t] = struct{}{} + } + } + return &f +} + +func (f *excludeFilter) Filter(tags *element.Tags) { + for k := range *tags { + if _, ok := f.keys[Key(k)]; ok { + delete(*tags, k) + } else if f.matches != nil { + for _, exkey := range f.matches { + if ok, _ := path.Match(exkey, k); ok { + delete(*tags, k) + break + } + } + } + } +} diff --git a/mapping/matcher.go b/mapping/matcher.go index f5191a2..309fb6d 100644 --- a/mapping/matcher.go +++ b/mapping/matcher.go @@ -6,9 +6,9 @@ import ( ) func (m *Mapping) PointMatcher() NodeMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(PointTable, mappings) - filters := make(tableFilters) + filters := make(tableElementFilters) m.addFilters(filters) m.addTypedFilters(PointTable, filters) return &tagMatcher{ @@ -20,9 +20,9 @@ func (m *Mapping) PointMatcher() NodeMatcher { } func (m *Mapping) LineStringMatcher() WayMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(LineStringTable, mappings) - filters := make(tableFilters) + filters := make(tableElementFilters) m.addFilters(filters) m.addTypedFilters(LineStringTable, filters) return &tagMatcher{ @@ -34,12 +34,12 @@ func (m *Mapping) LineStringMatcher() WayMatcher { } func (m *Mapping) PolygonMatcher() RelWayMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(PolygonTable, mappings) - filters := make(tableFilters) + filters := make(tableElementFilters) m.addFilters(filters) m.addTypedFilters(PolygonTable, filters) - relFilters := make(tableFilters) + relFilters := make(tableElementFilters) m.addRelationFilters(PolygonTable, relFilters) return &tagMatcher{ mappings: mappings, @@ -51,13 +51,13 @@ func (m *Mapping) PolygonMatcher() RelWayMatcher { } func (m *Mapping) RelationMatcher() RelationMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(RelationTable, mappings) - filters := make(tableFilters) + filters := make(tableElementFilters) m.addFilters(filters) m.addTypedFilters(PolygonTable, filters) m.addTypedFilters(RelationTable, filters) - relFilters := make(tableFilters) + relFilters := make(tableElementFilters) m.addRelationFilters(RelationTable, relFilters) return &tagMatcher{ mappings: mappings, @@ -69,12 +69,12 @@ func (m *Mapping) RelationMatcher() RelationMatcher { } func (m *Mapping) RelationMemberMatcher() RelationMatcher { - mappings := make(TagTables) + mappings := make(TagTableMapping) m.mappings(RelationMemberTable, mappings) - filters := make(tableFilters) + filters := make(tableElementFilters) m.addFilters(filters) m.addTypedFilters(RelationMemberTable, filters) - relFilters := make(tableFilters) + relFilters := make(tableElementFilters) m.addRelationFilters(RelationMemberTable, relFilters) return &tagMatcher{ mappings: mappings, @@ -85,13 +85,6 @@ func (m *Mapping) RelationMemberMatcher() RelationMatcher { } } -type Match struct { - Key string - Value string - Table DestTable - tableSpec *TableSpec -} - type NodeMatcher interface { MatchNode(node *element.Node) []Match } @@ -109,12 +102,11 @@ type RelWayMatcher interface { RelationMatcher } -type tagMatcher struct { - mappings TagTables - tables map[string]*TableSpec - filters map[string][]ElementFilter - relFilters map[string][]ElementFilter - matchAreas bool +type Match struct { + Key string + Value string + Table DestTable + tableSpec *TableSpec } func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} { @@ -125,6 +117,14 @@ func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *g return m.tableSpec.MakeMemberRow(rel, member, geom, *m) } +type tagMatcher struct { + mappings TagTableMapping + tables map[string]*TableSpec + filters tableElementFilters + relFilters tableElementFilters + matchAreas bool +} + func (tm *tagMatcher) MatchNode(node *element.Node) []Match { return tm.match(node.Tags, false, false) } @@ -161,7 +161,7 @@ type orderedMatch struct { func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Match { tables := make(map[DestTable]orderedMatch) - addTables := func(k, v string, tbls []OrderedDestTable) { + addTables := func(k, v string, tbls []orderedDestTable) { for _, t := range tbls { this := orderedMatch{ Match: Match{