filter - reject/require

filter:
        reject:
          key1: [ val1, val2, .. valn ]
          key2: [ val1, val2, .. valn ]
	require:
          key1: [ val1, val2, .. valn ]
          key2: [ val1, val2, .. valn ]
	reject_regexp:
          key1: 'regexp'
          key2: 'regexp'
	require_regexp:
          key1: 'regexp'
          key2: 'regexp'
master
ImreSamu 2016-12-14 00:25:34 +01:00
parent 5f1dd7cabe
commit 759b041606
3 changed files with 572 additions and 1 deletions

View File

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"io/ioutil"
"regexp"
"github.com/omniscale/imposm3/element"
@ -38,7 +39,11 @@ type GeneralizedTable struct {
}
type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
ExcludeTags *[][]string `yaml:"exclude_tags"`
Reject KeyValues `yaml:"reject"`
Require KeyValues `yaml:"require"`
RejectRegexp KeyRegexpValue `yaml:"reject_regexp"`
RequireRegexp KeyRegexpValue `yaml:"require_regexp"`
}
type Tables map[string]*Table
@ -72,6 +77,8 @@ type orderedValue struct {
}
type KeyValues map[Key][]orderedValue
type KeyRegexpValue map[Key]string
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
if *kv == nil {
*kv = make(map[Key][]orderedValue)
@ -330,7 +337,13 @@ func (m *Mapping) ElementFilters() map[string][]ElementFilter {
continue
}
if t.Filters.ExcludeTags != nil {
if len(*t.Filters.ExcludeTags) > 1 {
log.Warnf("Multiple exclude_tags not supported! (tablename:" + name + ") Please use the filter:'require'/'reject' ")
}
for _, filterKeyVal := range *t.Filters.ExcludeTags {
if filterKeyVal[1] == "__nil__" {
log.Warnf("exclude_tags __nil__ is not implemented! (tablename:" + name + ") Please use the filter: 'require'/'reject' ")
}
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[filterKeyVal[0]]; ok {
if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] {
@ -342,6 +355,88 @@ func (m *Mapping) ElementFilters() map[string][]ElementFilter {
result[name] = append(result[name], f)
}
}
if t.Filters.Require != nil {
for keyname, vararr := range t.Filters.Require {
result[name] = append(result[name], makeFiltersFunction(name, true, false, string(keyname), vararr))
}
}
if t.Filters.Reject != nil {
for keyname, vararr := range t.Filters.Reject {
result[name] = append(result[name], makeFiltersFunction(name, false, true, string(keyname), vararr))
}
}
if t.Filters.RequireRegexp != nil {
for keyname, regexp := range t.Filters.RequireRegexp {
result[name] = append(result[name], makeRegexpFiltersFunction(name, true, false, string(keyname), regexp))
}
}
if t.Filters.RejectRegexp != nil {
for keyname, regexp := range t.Filters.RejectRegexp {
result[name] = append(result[name], makeRegexpFiltersFunction(name, false, true, string(keyname), regexp))
}
}
}
return result
}
func findValueInOrderedValue(v Value, list []orderedValue) bool {
for _, item := range list {
if item.value == v {
return true
}
}
return false
}
func makeRegexpFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_regexp string) func(tags element.Tags, key Key, closed bool) bool {
// Compile regular expression
// if not valid regexp --> panic !
// log.Warnf("Regexp filter is experimental! (tablename:" + tablename + ")")
r := regexp.MustCompile(v_regexp)
return func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[v_keyname]; ok {
if r.MatchString(v) {
return virtualTrue
}
}
return virtualFalse
}
}
func makeFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_vararr []orderedValue) func(tags element.Tags, key Key, closed bool) bool {
if findValueInOrderedValue("__any__", v_vararr) { // check __any__
if len(v_vararr) > 1 {
log.Warnf("Multiple filter value with '__any__' keywords is probably not valid! (tablename:" + tablename + ")")
}
return func(tags element.Tags, key Key, closed bool) bool {
if _, ok := tags[v_keyname]; ok {
return virtualTrue
}
return virtualFalse
}
} else if len(v_vararr) == 1 { // IF 1 parameter THEN we can generate optimal code
return func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[v_keyname]; ok {
if Value(v) == v_vararr[0].value {
return virtualTrue
}
}
return virtualFalse
}
} else {
return func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags[v_keyname]; ok {
if findValueInOrderedValue(Value(v), v_vararr) {
return virtualTrue
}
}
return virtualFalse
}
}
}

364
mapping/config_test.go Normal file
View File

@ -0,0 +1,364 @@
package mapping
import (
"testing"
"github.com/omniscale/imposm3/element"
)
// go test ./mapping -run TestFilter_t0 -v
func TestFilter_t0(t *testing.T) {
/* ./config_test_mapping.yml ..
filters:
require:
boundary: ["administrative","maritime"]
mapping:
admin_level: ['2','4']
type: linestring
*/
filterTest(
// *testing.T
t,
// tablename
"config_test_t0",
// Accept
[]element.Tags{
element.Tags{"admin_level": "2", "boundary": "administrative"},
element.Tags{"admin_level": "2", "boundary": "maritime"},
element.Tags{"admin_level": "4", "boundary": "administrative", "name": "N4"},
element.Tags{"admin_level": "4", "boundary": "maritime", "name": "N4"},
},
// Reject
[]element.Tags{
element.Tags{"admin_level": "0", "boundary": "administrative"},
element.Tags{"admin_level": "1", "boundary": "administrative"},
element.Tags{"admin_level": "2", "boundary": "postal_code"},
element.Tags{"admin_level": "4", "boundary": "census"},
element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"},
element.Tags{"admin_level": "2"},
element.Tags{"admin_level": "4"},
element.Tags{"boundary": "administrative"},
element.Tags{"boundary": "maritime"},
element.Tags{"name": "maritime"},
},
)
}
// go test ./mapping -run TestFilter_t1 -v
func TestFilter_t1(t *testing.T) {
/* ./config_test_mapping.yml ..
filters:
require:
admin_level: ["2","4"]
mapping:
boundary:
- administrative
- maritime
type: linestring
*/
filterTest(
// *testing.T
t,
// tablename
"config_test_t1",
// Accept
[]element.Tags{
element.Tags{"admin_level": "2", "boundary": "administrative"},
element.Tags{"admin_level": "2", "boundary": "maritime"},
element.Tags{"admin_level": "4", "boundary": "administrative", "name": "N4"},
element.Tags{"admin_level": "4", "boundary": "maritime", "name": "N4"},
},
// Reject
[]element.Tags{
element.Tags{"admin_level": "0", "boundary": "administrative"},
element.Tags{"admin_level": "1", "boundary": "administrative"},
element.Tags{"admin_level": "2", "boundary": "postal_code"},
element.Tags{"admin_level": "4", "boundary": "census"},
element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"},
element.Tags{"admin_level": "2"},
element.Tags{"admin_level": "4"},
element.Tags{"boundary": "administrative"},
element.Tags{"boundary": "maritime"},
element.Tags{"name": "maritime"},
},
)
}
// go test ./mapping -run TestFilter_t2_building -v
func TestFilter_t2_building(t *testing.T) {
/* ./config_test_mapping.yml ..
filters:
reject:
building: ["no","none"]
require_regexp:
'addr:housenumber': '^\d+[a-zA-Z,]*$'
building: '^[a-z_]+$'
mapping:
building:
- __any__
type: linestring
*/
filterTest(
// *testing.T
t,
// tablename
"config_test_t2_building",
// Accept
[]element.Tags{
element.Tags{"building": "yes", "addr:housenumber": "1a"},
element.Tags{"building": "house", "addr:housenumber": "131"},
element.Tags{"building": "residential", "addr:housenumber": "21"},
element.Tags{"building": "garage", "addr:housenumber": "0"},
element.Tags{"building": "hut", "addr:housenumber": "99999999"},
element.Tags{"building": "_", "addr:housenumber": "333"},
element.Tags{"building": "y", "addr:housenumber": "1abcdefg"},
element.Tags{"building": "tower_block", "addr:housenumber": "1A"},
element.Tags{"building": "shed", "name": "N4", "addr:housenumber": "1AAA"},
element.Tags{"building": "office", "name": "N4", "addr:housenumber": "0XYAB,"},
},
// Reject
[]element.Tags{
element.Tags{"building": "yes", "addr:housenumber": "aaaaa-number"},
element.Tags{"building": "house", "addr:housenumber": "1-3a"},
element.Tags{"building": "house", "addr:housenumber": "❤"},
element.Tags{"building": "house", "addr:housenumber": "two"},
element.Tags{"building": "residential", "addr:housenumber": "x21"},
element.Tags{"building": "no"},
element.Tags{"building": "no", "addr:housenumber": "1a"},
element.Tags{"building": "No", "addr:housenumber": "1a"},
element.Tags{"building": "NO", "addr:housenumber": "1a"},
element.Tags{"building": "none"},
element.Tags{"building": "none", "addr:housenumber": "0"},
element.Tags{"building": "nONe", "addr:housenumber": "0"},
element.Tags{"building": "No"},
element.Tags{"building": "NO"},
element.Tags{"building": "NONe"},
element.Tags{"building": "Garage"},
element.Tags{"building": "Hut"},
element.Tags{"building": "Farm"},
element.Tags{"building": "tower-block"},
element.Tags{"building": "❤"},
element.Tags{"building": "Ümlåütê"},
element.Tags{"building": "中"},
element.Tags{"building": "SheD", "name": "N4"},
element.Tags{"building": "oFFice", "name": "N4"},
element.Tags{"admin_level": "2"},
element.Tags{"admin_level": "4"},
element.Tags{"boundary": "administrative"},
element.Tags{"boundary": "maritime"},
element.Tags{"name": "maritime"},
},
)
}
// go test ./mapping -run TestFilter_t3_highway_with_name -v
func TestFilter_t3_highway_with_name(t *testing.T) {
/* ./config_test_mapping.yml ..
filters:
require:
name: ["__any__"]
reject:
highway: ["no","none"]
mapping:
highway:
- __any__
type: linestring
*/
filterTest(
// *testing.T
t,
// tablename
"config_test_t3_highway_with_name",
// Accept
[]element.Tags{
element.Tags{"highway": "residential", "name": "N1"},
element.Tags{"highway": "service", "name": "N2"},
element.Tags{"highway": "track", "name": "N3"},
element.Tags{"highway": "unclassified", "name": "N4"},
element.Tags{"highway": "path", "name": "N5"},
element.Tags{"highway": "_", "name": "N6"},
element.Tags{"highway": "y", "name": "N7"},
element.Tags{"highway": "tower_block", "name": "N8"},
element.Tags{"highway": "shed", "name": "N9"},
element.Tags{"highway": "office", "name": "N10"},
element.Tags{"highway": "SheD", "name": "N11"},
element.Tags{"highway": "oFFice", "name": "N12"},
element.Tags{"highway": "❤", "name": "❤"},
element.Tags{"highway": "Ümlåütê", "name": "Ümlåütê"},
element.Tags{"highway": "中", "name": "中"},
},
// Reject
[]element.Tags{
element.Tags{"highway": "no", "name": "N1"},
element.Tags{"highway": "none", "name": "N2"},
element.Tags{"highway": "yes"},
element.Tags{"highway": "no"},
element.Tags{"highway": "none"},
element.Tags{"highway": "No"},
element.Tags{"highway": "NO"},
element.Tags{"highway": "NONe"},
element.Tags{"highway": "Garage"},
element.Tags{"highway": "residential"},
element.Tags{"highway": "path"},
element.Tags{"highway": "tower-block"},
element.Tags{"highway": "❤"},
element.Tags{"highway": "Ümlåütê"},
element.Tags{"highway": "中"},
element.Tags{"admin_level": "2"},
element.Tags{"admin_level": "4"},
element.Tags{"boundary": "administrative"},
element.Tags{"boundary": "maritime"},
element.Tags{"name": "maritime"},
},
)
}
// go test ./mapping -run TestFilter_t4_waterway_with_name -v
func TestFilter_t4_waterway_with_name(t *testing.T) {
/* ./config_test_mapping.yml ..
filters:
require:
name: ["__any__"]
waterway:
- stream
- river
- canal
- drain
- ditch
reject:
fixme: ['__any__']
amenity: ['__any__']
shop: ['__any__']
building: ['__any__']
tunnel: ['yes']
reject_regexp:
level: '^\D+.*$'
mapping:
waterway:
- __any__
type: linestring
*/
filterTest(
// *testing.T
t,
// tablename
"config_test_t4_waterway_with_name",
// Accept
[]element.Tags{
element.Tags{"waterway": "stream", "name": "N1"},
element.Tags{"waterway": "river", "name": "N2"},
element.Tags{"waterway": "canal", "name": "N3"},
element.Tags{"waterway": "drain", "name": "N4"},
element.Tags{"waterway": "ditch", "name": "N5"},
element.Tags{"waterway": "stream", "name": "N1", "tunnel": "no"},
element.Tags{"waterway": "river", "name": "N2", "boat": "no"},
element.Tags{"waterway": "canal", "name": "N3"},
element.Tags{"waterway": "ditch", "name": "N4", "level": "3"},
},
// Reject
[]element.Tags{
element.Tags{"waterway": "ditch", "name": "N1", "fixme": "incomplete"},
element.Tags{"waterway": "stream", "name": "N1", "amenity": "parking"},
element.Tags{"waterway": "river", "name": "N2", "shop": "hairdresser"},
element.Tags{"waterway": "canal", "name": "N3", "building": "house"},
element.Tags{"waterway": "drain", "name": "N1 tunnel", "tunnel": "yes"},
element.Tags{"waterway": "river", "name": "N4", "level": "unknown"},
element.Tags{"waterway": "ditch", "name": "N4", "level": "primary"},
element.Tags{"waterway": "path", "name": "N5"},
element.Tags{"waterway": "_", "name": "N6"},
element.Tags{"waterway": "y", "name": "N7"},
element.Tags{"waterway": "tower_block", "name": "N8"},
element.Tags{"waterway": "shed", "name": "N9"},
element.Tags{"waterway": "office", "name": "N10"},
element.Tags{"waterway": "SheD", "name": "N11"},
element.Tags{"waterway": "oFFice", "name": "N12"},
element.Tags{"waterway": "❤", "name": "❤"},
element.Tags{"waterway": "Ümlåütê", "name": "Ümlåütê"},
element.Tags{"waterway": "中", "name": "中"},
element.Tags{"waterway": "no", "name": "N1"},
element.Tags{"waterway": "none", "name": "N2"},
element.Tags{"waterway": "yes"},
element.Tags{"waterway": "no"},
element.Tags{"waterway": "none"},
element.Tags{"waterway": "tower-block"},
element.Tags{"waterway": "❤"},
element.Tags{"waterway": "Ümlåütê"},
element.Tags{"waterway": "中"},
element.Tags{"admin_level": "2"},
element.Tags{"admin_level": "4"},
element.Tags{"boundary": "administrative"},
element.Tags{"boundary": "maritime"},
element.Tags{"name": "maritime"},
},
)
}
func filterTest(t *testing.T, tablename string, accept []element.Tags, reject []element.Tags) {
var configTestMapping *Mapping
var err error
configTestMapping, err = NewMapping("./config_test_mapping.yml")
if err != nil {
panic(err)
}
var actualMatch []Match
elem := element.Way{}
ls := configTestMapping.LineStringMatcher()
for _, et := range accept {
elem.Tags = et
actualMatch = ls.MatchWay(&elem)
included := false
for _, mt := range actualMatch {
if tablename == mt.Table.Name {
included = true
break
}
}
if included == false {
t.Errorf("TestFilter - Not Accepted : (%s) (%+v) ", tablename, et)
}
}
for _, et := range reject {
elem.Tags = et
actualMatch = ls.MatchWay(&elem)
included := false
for _, mt := range actualMatch {
if tablename == mt.Table.Name {
included = true
break
}
}
if included == true {
t.Errorf("TestFilter - Not Rejected : (%s) (%+v) ", tablename, et)
}
}
}

View File

@ -0,0 +1,112 @@
# test mappings
#
# only `type: linestring` impemented in config_test.go
#
# Regexp Note: be careful because yaml interprets escape sequences inside double quoted strings!
# so use single quoted string !
tables:
config_test_t0:
fields:
- name: id
type: id
- key: admin_level
name: admin_level
type: integer
filters:
require:
boundary: ["administrative","maritime"]
mapping:
admin_level: ['2','4']
type: linestring
config_test_t1:
fields:
- name: id
type: id
- key: admin_level
name: admin_level
type: integer
filters:
require:
admin_level: ["2","4"]
mapping:
boundary:
- administrative
- maritime
type: linestring
config_test_t2_building:
fields:
- name: id
type: id
- key: building
name: building
type: string
filters:
reject:
building: ["no","none"]
require_regexp:
'addr:housenumber': '^\d+[a-zA-Z,]*$'
building: '^[a-z_]+$'
mapping:
building:
- __any__
type: linestring
config_test_t3_highway_with_name:
fields:
- name: id
type: id
- key: highway
name: highway
type: string
- key: name
name: name
type: string
filters:
require:
name: ["__any__"]
reject:
highway: ["no","none"]
mapping:
highway:
- __any__
type: linestring
config_test_t4_waterway_with_name:
fields:
- name: id
type: id
- key: waterway
name: waterway
type: string
- key: name
name: name
type: string
filters:
require:
name: ["__any__"]
waterway:
- stream
- river
- canal
- drain
- ditch
reject:
fixme: ['__any__']
amenity: ['__any__']
shop: ['__any__']
building: ['__any__']
tunnel: ['yes']
reject_regexp:
level: '^\D+.*$'
mapping:
waterway:
- __any__
type: linestring