diff --git a/mapping/config.go b/mapping/config.go index 0366328..5a9d62d 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "regexp" "github.com/omniscale/imposm3/element" @@ -38,7 +39,11 @@ type GeneralizedTable struct { } type Filters struct { - ExcludeTags *[][]string `yaml:"exclude_tags"` + ExcludeTags *[][]string `yaml:"exclude_tags"` + Reject KeyValues `yaml:"reject"` + Require KeyValues `yaml:"require"` + RejectRegexp KeyRegexpValue `yaml:"reject_regexp"` + RequireRegexp KeyRegexpValue `yaml:"require_regexp"` } type Tables map[string]*Table @@ -72,6 +77,8 @@ type orderedValue struct { } type KeyValues map[Key][]orderedValue +type KeyRegexpValue map[Key]string + func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error { if *kv == nil { *kv = make(map[Key][]orderedValue) @@ -330,7 +337,13 @@ func (m *Mapping) ElementFilters() map[string][]ElementFilter { continue } if t.Filters.ExcludeTags != nil { + if len(*t.Filters.ExcludeTags) > 1 { + log.Warnf("Multiple exclude_tags not supported! (tablename:" + name + ") Please use the filter:'require'/'reject' ") + } for _, filterKeyVal := range *t.Filters.ExcludeTags { + if filterKeyVal[1] == "__nil__" { + log.Warnf("exclude_tags __nil__ is not implemented! (tablename:" + name + ") Please use the filter: 'require'/'reject' ") + } f := func(tags element.Tags, key Key, closed bool) bool { if v, ok := tags[filterKeyVal[0]]; ok { if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] { @@ -342,6 +355,88 @@ func (m *Mapping) ElementFilters() map[string][]ElementFilter { result[name] = append(result[name], f) } } + + if t.Filters.Require != nil { + for keyname, vararr := range t.Filters.Require { + result[name] = append(result[name], makeFiltersFunction(name, true, false, string(keyname), vararr)) + } + } + + if t.Filters.Reject != nil { + for keyname, vararr := range t.Filters.Reject { + result[name] = append(result[name], makeFiltersFunction(name, false, true, string(keyname), vararr)) + } + } + + if t.Filters.RequireRegexp != nil { + for keyname, regexp := range t.Filters.RequireRegexp { + result[name] = append(result[name], makeRegexpFiltersFunction(name, true, false, string(keyname), regexp)) + } + } + + if t.Filters.RejectRegexp != nil { + for keyname, regexp := range t.Filters.RejectRegexp { + result[name] = append(result[name], makeRegexpFiltersFunction(name, false, true, string(keyname), regexp)) + } + } + } return result } + +func findValueInOrderedValue(v Value, list []orderedValue) bool { + for _, item := range list { + if item.value == v { + return true + } + } + return false +} + +func makeRegexpFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_regexp string) func(tags element.Tags, key Key, closed bool) bool { + // Compile regular expression + // if not valid regexp --> panic ! + + // log.Warnf("Regexp filter is experimental! (tablename:" + tablename + ")") + r := regexp.MustCompile(v_regexp) + return func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags[v_keyname]; ok { + if r.MatchString(v) { + return virtualTrue + } + } + return virtualFalse + } +} + +func makeFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_vararr []orderedValue) func(tags element.Tags, key Key, closed bool) bool { + if findValueInOrderedValue("__any__", v_vararr) { // check __any__ + if len(v_vararr) > 1 { + log.Warnf("Multiple filter value with '__any__' keywords is probably not valid! (tablename:" + tablename + ")") + } + return func(tags element.Tags, key Key, closed bool) bool { + if _, ok := tags[v_keyname]; ok { + return virtualTrue + } + return virtualFalse + } + } else if len(v_vararr) == 1 { // IF 1 parameter THEN we can generate optimal code + return func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags[v_keyname]; ok { + if Value(v) == v_vararr[0].value { + return virtualTrue + } + } + return virtualFalse + } + } else { + return func(tags element.Tags, key Key, closed bool) bool { + if v, ok := tags[v_keyname]; ok { + if findValueInOrderedValue(Value(v), v_vararr) { + return virtualTrue + } + } + return virtualFalse + } + } +} diff --git a/mapping/config_test.go b/mapping/config_test.go new file mode 100644 index 0000000..0d44526 --- /dev/null +++ b/mapping/config_test.go @@ -0,0 +1,364 @@ +package mapping + +import ( + "testing" + + "github.com/omniscale/imposm3/element" +) + +// go test ./mapping -run TestFilter_t0 -v +func TestFilter_t0(t *testing.T) { + + /* ./config_test_mapping.yml .. + + filters: + require: + boundary: ["administrative","maritime"] + mapping: + admin_level: ['2','4'] + type: linestring + + */ + filterTest( + // *testing.T + t, + // tablename + "config_test_t0", + // Accept + []element.Tags{ + element.Tags{"admin_level": "2", "boundary": "administrative"}, + element.Tags{"admin_level": "2", "boundary": "maritime"}, + element.Tags{"admin_level": "4", "boundary": "administrative", "name": "N4"}, + element.Tags{"admin_level": "4", "boundary": "maritime", "name": "N4"}, + }, + // Reject + []element.Tags{ + element.Tags{"admin_level": "0", "boundary": "administrative"}, + element.Tags{"admin_level": "1", "boundary": "administrative"}, + element.Tags{"admin_level": "2", "boundary": "postal_code"}, + element.Tags{"admin_level": "4", "boundary": "census"}, + element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"}, + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +// go test ./mapping -run TestFilter_t1 -v +func TestFilter_t1(t *testing.T) { + + /* ./config_test_mapping.yml .. + + filters: + require: + admin_level: ["2","4"] + mapping: + boundary: + - administrative + - maritime + type: linestring + + */ + + filterTest( + // *testing.T + t, + // tablename + "config_test_t1", + // Accept + []element.Tags{ + element.Tags{"admin_level": "2", "boundary": "administrative"}, + element.Tags{"admin_level": "2", "boundary": "maritime"}, + element.Tags{"admin_level": "4", "boundary": "administrative", "name": "N4"}, + element.Tags{"admin_level": "4", "boundary": "maritime", "name": "N4"}, + }, + // Reject + []element.Tags{ + element.Tags{"admin_level": "0", "boundary": "administrative"}, + element.Tags{"admin_level": "1", "boundary": "administrative"}, + element.Tags{"admin_level": "2", "boundary": "postal_code"}, + element.Tags{"admin_level": "4", "boundary": "census"}, + element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"}, + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +// go test ./mapping -run TestFilter_t2_building -v +func TestFilter_t2_building(t *testing.T) { + + /* ./config_test_mapping.yml .. + filters: + reject: + building: ["no","none"] + require_regexp: + 'addr:housenumber': '^\d+[a-zA-Z,]*$' + building: '^[a-z_]+$' + mapping: + building: + - __any__ + type: linestring + + */ + + filterTest( + // *testing.T + t, + // tablename + "config_test_t2_building", + // Accept + []element.Tags{ + element.Tags{"building": "yes", "addr:housenumber": "1a"}, + element.Tags{"building": "house", "addr:housenumber": "131"}, + element.Tags{"building": "residential", "addr:housenumber": "21"}, + element.Tags{"building": "garage", "addr:housenumber": "0"}, + element.Tags{"building": "hut", "addr:housenumber": "99999999"}, + element.Tags{"building": "_", "addr:housenumber": "333"}, + element.Tags{"building": "y", "addr:housenumber": "1abcdefg"}, + element.Tags{"building": "tower_block", "addr:housenumber": "1A"}, + element.Tags{"building": "shed", "name": "N4", "addr:housenumber": "1AAA"}, + element.Tags{"building": "office", "name": "N4", "addr:housenumber": "0XYAB,"}, + }, + // Reject + []element.Tags{ + element.Tags{"building": "yes", "addr:housenumber": "aaaaa-number"}, + element.Tags{"building": "house", "addr:housenumber": "1-3a"}, + element.Tags{"building": "house", "addr:housenumber": "❤"}, + element.Tags{"building": "house", "addr:housenumber": "two"}, + element.Tags{"building": "residential", "addr:housenumber": "x21"}, + + element.Tags{"building": "no"}, + element.Tags{"building": "no", "addr:housenumber": "1a"}, + element.Tags{"building": "No", "addr:housenumber": "1a"}, + element.Tags{"building": "NO", "addr:housenumber": "1a"}, + element.Tags{"building": "none"}, + element.Tags{"building": "none", "addr:housenumber": "0"}, + element.Tags{"building": "nONe", "addr:housenumber": "0"}, + element.Tags{"building": "No"}, + element.Tags{"building": "NO"}, + element.Tags{"building": "NONe"}, + element.Tags{"building": "Garage"}, + element.Tags{"building": "Hut"}, + element.Tags{"building": "Farm"}, + element.Tags{"building": "tower-block"}, + element.Tags{"building": "❤"}, + element.Tags{"building": "Ümlåütê"}, + element.Tags{"building": "中"}, + element.Tags{"building": "SheD", "name": "N4"}, + element.Tags{"building": "oFFice", "name": "N4"}, + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +// go test ./mapping -run TestFilter_t3_highway_with_name -v +func TestFilter_t3_highway_with_name(t *testing.T) { + + /* ./config_test_mapping.yml .. + filters: + require: + name: ["__any__"] + reject: + highway: ["no","none"] + mapping: + highway: + - __any__ + type: linestring + */ + + filterTest( + // *testing.T + t, + // tablename + "config_test_t3_highway_with_name", + // Accept + []element.Tags{ + element.Tags{"highway": "residential", "name": "N1"}, + element.Tags{"highway": "service", "name": "N2"}, + element.Tags{"highway": "track", "name": "N3"}, + element.Tags{"highway": "unclassified", "name": "N4"}, + element.Tags{"highway": "path", "name": "N5"}, + element.Tags{"highway": "_", "name": "N6"}, + element.Tags{"highway": "y", "name": "N7"}, + element.Tags{"highway": "tower_block", "name": "N8"}, + element.Tags{"highway": "shed", "name": "N9"}, + element.Tags{"highway": "office", "name": "N10"}, + element.Tags{"highway": "SheD", "name": "N11"}, + element.Tags{"highway": "oFFice", "name": "N12"}, + element.Tags{"highway": "❤", "name": "❤"}, + element.Tags{"highway": "Ümlåütê", "name": "Ümlåütê"}, + element.Tags{"highway": "中", "name": "中"}, + }, + // Reject + []element.Tags{ + element.Tags{"highway": "no", "name": "N1"}, + element.Tags{"highway": "none", "name": "N2"}, + element.Tags{"highway": "yes"}, + element.Tags{"highway": "no"}, + element.Tags{"highway": "none"}, + element.Tags{"highway": "No"}, + element.Tags{"highway": "NO"}, + element.Tags{"highway": "NONe"}, + element.Tags{"highway": "Garage"}, + element.Tags{"highway": "residential"}, + element.Tags{"highway": "path"}, + element.Tags{"highway": "tower-block"}, + element.Tags{"highway": "❤"}, + element.Tags{"highway": "Ümlåütê"}, + element.Tags{"highway": "中"}, + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +// go test ./mapping -run TestFilter_t4_waterway_with_name -v +func TestFilter_t4_waterway_with_name(t *testing.T) { + + /* ./config_test_mapping.yml .. + + filters: + require: + name: ["__any__"] + waterway: + - stream + - river + - canal + - drain + - ditch + reject: + fixme: ['__any__'] + amenity: ['__any__'] + shop: ['__any__'] + building: ['__any__'] + tunnel: ['yes'] + reject_regexp: + level: '^\D+.*$' + mapping: + waterway: + - __any__ + type: linestring + + */ + + filterTest( + // *testing.T + t, + // tablename + "config_test_t4_waterway_with_name", + // Accept + []element.Tags{ + element.Tags{"waterway": "stream", "name": "N1"}, + element.Tags{"waterway": "river", "name": "N2"}, + element.Tags{"waterway": "canal", "name": "N3"}, + element.Tags{"waterway": "drain", "name": "N4"}, + element.Tags{"waterway": "ditch", "name": "N5"}, + + element.Tags{"waterway": "stream", "name": "N1", "tunnel": "no"}, + element.Tags{"waterway": "river", "name": "N2", "boat": "no"}, + element.Tags{"waterway": "canal", "name": "N3"}, + element.Tags{"waterway": "ditch", "name": "N4", "level": "3"}, + }, + // Reject + []element.Tags{ + element.Tags{"waterway": "ditch", "name": "N1", "fixme": "incomplete"}, + element.Tags{"waterway": "stream", "name": "N1", "amenity": "parking"}, + element.Tags{"waterway": "river", "name": "N2", "shop": "hairdresser"}, + element.Tags{"waterway": "canal", "name": "N3", "building": "house"}, + element.Tags{"waterway": "drain", "name": "N1 tunnel", "tunnel": "yes"}, + + element.Tags{"waterway": "river", "name": "N4", "level": "unknown"}, + element.Tags{"waterway": "ditch", "name": "N4", "level": "primary"}, + + element.Tags{"waterway": "path", "name": "N5"}, + element.Tags{"waterway": "_", "name": "N6"}, + element.Tags{"waterway": "y", "name": "N7"}, + element.Tags{"waterway": "tower_block", "name": "N8"}, + element.Tags{"waterway": "shed", "name": "N9"}, + element.Tags{"waterway": "office", "name": "N10"}, + element.Tags{"waterway": "SheD", "name": "N11"}, + element.Tags{"waterway": "oFFice", "name": "N12"}, + element.Tags{"waterway": "❤", "name": "❤"}, + element.Tags{"waterway": "Ümlåütê", "name": "Ümlåütê"}, + element.Tags{"waterway": "中", "name": "中"}, + + element.Tags{"waterway": "no", "name": "N1"}, + element.Tags{"waterway": "none", "name": "N2"}, + element.Tags{"waterway": "yes"}, + element.Tags{"waterway": "no"}, + element.Tags{"waterway": "none"}, + element.Tags{"waterway": "tower-block"}, + element.Tags{"waterway": "❤"}, + element.Tags{"waterway": "Ümlåütê"}, + element.Tags{"waterway": "中"}, + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +func filterTest(t *testing.T, tablename string, accept []element.Tags, reject []element.Tags) { + + var configTestMapping *Mapping + var err error + + configTestMapping, err = NewMapping("./config_test_mapping.yml") + if err != nil { + panic(err) + } + + var actualMatch []Match + + elem := element.Way{} + ls := configTestMapping.LineStringMatcher() + + for _, et := range accept { + elem.Tags = et + actualMatch = ls.MatchWay(&elem) + + included := false + for _, mt := range actualMatch { + if tablename == mt.Table.Name { + included = true + break + } + } + if included == false { + t.Errorf("TestFilter - Not Accepted : (%s) (%+v) ", tablename, et) + } + } + + for _, et := range reject { + elem.Tags = et + actualMatch = ls.MatchWay(&elem) + + included := false + for _, mt := range actualMatch { + if tablename == mt.Table.Name { + included = true + break + } + } + if included == true { + t.Errorf("TestFilter - Not Rejected : (%s) (%+v) ", tablename, et) + } + } + +} diff --git a/mapping/config_test_mapping.yml b/mapping/config_test_mapping.yml new file mode 100644 index 0000000..0e03d61 --- /dev/null +++ b/mapping/config_test_mapping.yml @@ -0,0 +1,112 @@ + +# test mappings +# +# only `type: linestring` impemented in config_test.go +# +# Regexp Note: be careful because yaml interprets escape sequences inside double quoted strings! +# so use single quoted string ! + + +tables: + config_test_t0: + fields: + - name: id + type: id + - key: admin_level + name: admin_level + type: integer + filters: + require: + boundary: ["administrative","maritime"] + mapping: + admin_level: ['2','4'] + type: linestring + + config_test_t1: + fields: + - name: id + type: id + - key: admin_level + name: admin_level + type: integer + filters: + require: + admin_level: ["2","4"] + mapping: + boundary: + - administrative + - maritime + type: linestring + + + config_test_t2_building: + fields: + - name: id + type: id + - key: building + name: building + type: string + filters: + reject: + building: ["no","none"] + require_regexp: + 'addr:housenumber': '^\d+[a-zA-Z,]*$' + building: '^[a-z_]+$' + mapping: + building: + - __any__ + type: linestring + + + config_test_t3_highway_with_name: + fields: + - name: id + type: id + - key: highway + name: highway + type: string + - key: name + name: name + type: string + filters: + require: + name: ["__any__"] + reject: + highway: ["no","none"] + mapping: + highway: + - __any__ + type: linestring + + + config_test_t4_waterway_with_name: + fields: + - name: id + type: id + - key: waterway + name: waterway + type: string + - key: name + name: name + type: string + filters: + require: + name: ["__any__"] + waterway: + - stream + - river + - canal + - drain + - ditch + reject: + fixme: ['__any__'] + amenity: ['__any__'] + shop: ['__any__'] + building: ['__any__'] + tunnel: ['yes'] + reject_regexp: + level: '^\D+.*$' + mapping: + waterway: + - __any__ + type: linestring