vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactor code structure

master
Oliver Tonnhofer 2017-05-11 09:37:12 +02:00
parent ef89d91ae8
commit a30442d254
4 changed files with 147 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mapping/columns.go
View File

@ -100,22 +100,6 @@ func (t *TableSpec) MakeMemberRow(rel *element.Relation, member *element.Member,
return row
}
func (c *Column) ColumnType() *ColumnType {
if columnType, ok := AvailableColumnTypes[c.Type]; ok {
if columnType.MakeFunc != nil {
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
if err != nil {
log.Print(err)
return nil
}
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
}
columnType.FromMember = c.FromMember
return &columnType
}
return nil
}
func (t *Table) TableSpec() *TableSpec {
result := TableSpec{}

106
mapping/config.go
View File

@ -10,6 +10,16 @@ import (
"gopkg.in/yaml.v2"
)
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Column struct {
Name string `yaml:"name"`
Key Key `yaml:"key"`
@ -19,6 +29,23 @@ type Column struct {
FromMember bool `yaml:"from_member"`
}
func (c *Column) ColumnType() *ColumnType {
if columnType, ok := AvailableColumnTypes[c.Type]; ok {
if columnType.MakeFunc != nil {
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
if err != nil {
log.Print(err)
return nil
}
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
}
columnType.FromMember = c.FromMember
return &columnType
}
return nil
}
type Tables map[string]*Table
type Table struct {
Name string
Type TableType `yaml:"type"`
@ -31,6 +58,7 @@ type Table struct {
RelationTypes []string `yaml:"relation_types"`
}
type GeneralizedTables map[string]*GeneralizedTable
type GeneralizedTable struct {
Name string
SourceTableName string `yaml:"source"`
@ -42,20 +70,6 @@ type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Tables map[string]*Table
type GeneralizedTables map[string]*GeneralizedTable
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Areas struct {
AreaTags []Key `yaml:"area_tags"`
LinearTags []Key `yaml:"linear_tags"`
@ -71,6 +85,7 @@ type orderedValue struct {
value Value
order int
}
type KeyValues map[Key][]orderedValue
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
@ -116,18 +131,44 @@ type TypeMappings struct {
type ElementFilter func(tags element.Tags, key Key, closed bool) bool
type TagTables map[Key]map[Value][]OrderedDestTable
type orderedDestTable struct {
DestTable
order int
}
type TagTableMapping map[Key]map[Value][]orderedDestTable
func (tt TagTableMapping) addFromMapping(mapping KeyValues, table DestTable) {
for key, vals := range mapping {
for _, v := range vals {
vals, ok := tt[key]
tbl := orderedDestTable{DestTable: table, order: v.order}
if ok {
vals[v.value] = append(vals[v.value], tbl)
} else {
tt[key] = make(map[Value][]orderedDestTable)
tt[key][v.value] = append(tt[key][v.value], tbl)
}
}
}
}
func (tt TagTableMapping) asTagMap() tagMap {
result := make(tagMap)
for k, vals := range tt {
result[k] = make(map[Value]struct{})
for v := range vals {
result[k][v] = struct{}{}
}
}
return result
}
type DestTable struct {
Name string
SubMapping string
}
type OrderedDestTable struct {
DestTable
order int
}
type TableType string
func (tt *TableType) UnmarshalJSON(data []byte) error {
@ -195,22 +236,7 @@ func (m *Mapping) prepare() error {
return nil
}
func (tt TagTables) addFromMapping(mapping KeyValues, table DestTable) {
for key, vals := range mapping {
for _, v := range vals {
vals, ok := tt[key]
tbl := OrderedDestTable{DestTable: table, order: v.order}
if ok {
vals[v.value] = append(vals[v.value], tbl)
} else {
tt[key] = make(map[Value][]OrderedDestTable)
tt[key][v.value] = append(tt[key][v.value], tbl)
}
}
}
}
func (m *Mapping) mappings(tableType TableType, mappings TagTables) {
func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
for name, t := range m.Tables {
if t.Type != GeometryTable && t.Type != tableType {
continue
@ -277,9 +303,9 @@ func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
tags["area"] = true
}
type tableFilters map[string][]ElementFilter
type tableElementFilters map[string][]ElementFilter
func (m *Mapping) addTypedFilters(tableType TableType, filters tableFilters) {
func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) {
var areaTags map[Key]struct{}
var linearTags map[Key]struct{}
if m.Areas.AreaTags != nil {
@ -332,7 +358,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableFilters) {
}
}
func (m *Mapping) addRelationFilters(tableType TableType, filters tableFilters) {
func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
for name, t := range m.Tables {
if t.RelationTypes != nil {
relTypes := t.RelationTypes // copy loop var for closure
@ -364,7 +390,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableFilters)
}
}
func (m *Mapping) addFilters(filters tableFilters) {
func (m *Mapping) addFilters(filters tableElementFilters) {
for name, t := range m.Tables {
if t.Filters == nil {
continue

106
mapping/filter.go
View File

@ -7,42 +7,46 @@ import (
"github.com/omniscale/imposm3/element"
)
type TagFilterer interface {
Filter(tags *element.Tags)
}
func (m *Mapping) NodeTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
mappings := make(TagTableMapping)
m.mappings(PointTable, mappings)
tags := make(map[Key]bool)
m.extraTags(PointTable, tags)
m.extraTags(RelationMemberTable, tags)
return &TagFilter{mappings, tags}
return &tagFilter{mappings.asTagMap(), tags}
}
func (m *Mapping) WayTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
mappings := make(TagTableMapping)
m.mappings(LineStringTable, mappings)
m.mappings(PolygonTable, mappings)
tags := make(map[Key]bool)
m.extraTags(LineStringTable, tags)
m.extraTags(PolygonTable, tags)
m.extraTags(RelationMemberTable, tags)
return &TagFilter{mappings, tags}
return &tagFilter{mappings.asTagMap(), tags}
}
func (m *Mapping) RelationTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
}
mappings := make(map[Key]map[Value][]OrderedDestTable)
mappings := make(TagTableMapping)
// do not filter out type tag for common relations
mappings["type"] = map[Value][]OrderedDestTable{
"multipolygon": []OrderedDestTable{},
"boundary": []OrderedDestTable{},
"land_area": []OrderedDestTable{},
mappings["type"] = map[Value][]orderedDestTable{
"multipolygon": []orderedDestTable{},
"boundary": []orderedDestTable{},
"land_area": []orderedDestTable{},
}
m.mappings(LineStringTable, mappings)
m.mappings(PolygonTable, mappings)
@ -53,54 +57,17 @@ func (m *Mapping) RelationTagFilter() TagFilterer {
m.extraTags(PolygonTable, tags)
m.extraTags(RelationTable, tags)
m.extraTags(RelationMemberTable, tags)
return &TagFilter{mappings, tags}
return &tagFilter{mappings.asTagMap(), tags}
}
type TagFilter struct {
mappings map[Key]map[Value][]OrderedDestTable
type tagMap map[Key]map[Value]struct{}
type tagFilter struct {
mappings tagMap
extraTags map[Key]bool
}
type ExcludeFilter struct {
keys map[Key]struct{}
matches []string
}
func newExcludeFilter(tags []Key) *ExcludeFilter {
f := ExcludeFilter{
keys: make(map[Key]struct{}),
matches: make([]string, 0),
}
for _, t := range tags {
if strings.ContainsAny(string(t), "?*[") {
f.matches = append(f.matches, string(t))
} else {
f.keys[t] = struct{}{}
}
}
return &f
}
func (f *ExcludeFilter) Filter(tags *element.Tags) {
for k := range *tags {
if _, ok := f.keys[Key(k)]; ok {
delete(*tags, k)
} else if f.matches != nil {
for _, exkey := range f.matches {
if ok, _ := path.Match(exkey, k); ok {
delete(*tags, k)
break
}
}
}
}
}
type TagFilterer interface {
Filter(tags *element.Tags)
}
func (f *TagFilter) Filter(tags *element.Tags) {
func (f *tagFilter) Filter(tags *element.Tags) {
if tags == nil {
return
}
@ -119,3 +86,38 @@ func (f *TagFilter) Filter(tags *element.Tags) {
}
}
}
type excludeFilter struct {
keys map[Key]struct{}
matches []string
}
func newExcludeFilter(tags []Key) *excludeFilter {
f := excludeFilter{
keys: make(map[Key]struct{}),
matches: make([]string, 0),
}
for _, t := range tags {
if strings.ContainsAny(string(t), "?*[") {
f.matches = append(f.matches, string(t))
} else {
f.keys[t] = struct{}{}
}
}
return &f
}
func (f *excludeFilter) Filter(tags *element.Tags) {
for k := range *tags {
if _, ok := f.keys[Key(k)]; ok {
delete(*tags, k)
} else if f.matches != nil {
for _, exkey := range f.matches {
if ok, _ := path.Match(exkey, k); ok {
delete(*tags, k)
break
}
}
}
}
}

54
mapping/matcher.go
View File

@ -6,9 +6,9 @@ import (
)
func (m *Mapping) PointMatcher() NodeMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(PointTable, mappings)
filters := make(tableFilters)
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PointTable, filters)
return &tagMatcher{
@ -20,9 +20,9 @@ func (m *Mapping) PointMatcher() NodeMatcher {
}
func (m *Mapping) LineStringMatcher() WayMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(LineStringTable, mappings)
filters := make(tableFilters)
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(LineStringTable, filters)
return &tagMatcher{
@ -34,12 +34,12 @@ func (m *Mapping) LineStringMatcher() WayMatcher {
}
func (m *Mapping) PolygonMatcher() RelWayMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(PolygonTable, mappings)
filters := make(tableFilters)
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PolygonTable, filters)
relFilters := make(tableFilters)
relFilters := make(tableElementFilters)
m.addRelationFilters(PolygonTable, relFilters)
return &tagMatcher{
mappings: mappings,
@ -51,13 +51,13 @@ func (m *Mapping) PolygonMatcher() RelWayMatcher {
}
func (m *Mapping) RelationMatcher() RelationMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(RelationTable, mappings)
filters := make(tableFilters)
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(PolygonTable, filters)
m.addTypedFilters(RelationTable, filters)
relFilters := make(tableFilters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationTable, relFilters)
return &tagMatcher{
mappings: mappings,
@ -69,12 +69,12 @@ func (m *Mapping) RelationMatcher() RelationMatcher {
}
func (m *Mapping) RelationMemberMatcher() RelationMatcher {
mappings := make(TagTables)
mappings := make(TagTableMapping)
m.mappings(RelationMemberTable, mappings)
filters := make(tableFilters)
filters := make(tableElementFilters)
m.addFilters(filters)
m.addTypedFilters(RelationMemberTable, filters)
relFilters := make(tableFilters)
relFilters := make(tableElementFilters)
m.addRelationFilters(RelationMemberTable, relFilters)
return &tagMatcher{
mappings: mappings,
@ -85,13 +85,6 @@ func (m *Mapping) RelationMemberMatcher() RelationMatcher {
}
}
type Match struct {
Key string
Value string
Table DestTable
tableSpec *TableSpec
}
type NodeMatcher interface {
MatchNode(node *element.Node) []Match
}
@ -109,12 +102,11 @@ type RelWayMatcher interface {
RelationMatcher
}
type tagMatcher struct {
mappings TagTables
tables map[string]*TableSpec
filters map[string][]ElementFilter
relFilters map[string][]ElementFilter
matchAreas bool
type Match struct {
Key string
Value string
Table DestTable
tableSpec *TableSpec
}
func (m *Match) Row(elem *element.OSMElem, geom *geom.Geometry) []interface{} {
@ -125,6 +117,14 @@ func (m *Match) MemberRow(rel *element.Relation, member *element.Member, geom *g
return m.tableSpec.MakeMemberRow(rel, member, geom, *m)
}
type tagMatcher struct {
mappings TagTableMapping
tables map[string]*TableSpec
filters tableElementFilters
relFilters tableElementFilters
matchAreas bool
}
func (tm *tagMatcher) MatchNode(node *element.Node) []Match {
return tm.match(node.Tags, false, false)
}
@ -161,7 +161,7 @@ type orderedMatch struct {
func (tm *tagMatcher) match(tags element.Tags, closed bool, relation bool) []Match {
tables := make(map[DestTable]orderedMatch)
addTables := func(k, v string, tbls []OrderedDestTable) {
addTables := func(k, v string, tbls []orderedDestTable) {
for _, t := range tbls {
this := orderedMatch{
Match: Match{