diff --git a/mapping/config.go b/mapping/config.go index 5a9d62d..290df02 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -337,22 +337,17 @@ func (m *Mapping) ElementFilters() map[string][]ElementFilter { continue } if t.Filters.ExcludeTags != nil { - if len(*t.Filters.ExcludeTags) > 1 { - log.Warnf("Multiple exclude_tags not supported! (tablename:" + name + ") Please use the filter:'require'/'reject' ") - } + log.Print("warn: exclude_tags filter is deprecated and will be removed. See require and reject filter.") for _, filterKeyVal := range *t.Filters.ExcludeTags { - if filterKeyVal[1] == "__nil__" { - log.Warnf("exclude_tags __nil__ is not implemented! (tablename:" + name + ") Please use the filter: 'require'/'reject' ") + // Convert `exclude_tags`` filter to `reject` filter ! + keyname := string(filterKeyVal[0]) + vararr := []orderedValue{ + { + value: Value(filterKeyVal[1]), + order: 1, + }, } - f := func(tags element.Tags, key Key, closed bool) bool { - if v, ok := tags[filterKeyVal[0]]; ok { - if filterKeyVal[1] == "__any__" || v == filterKeyVal[1] { - return false - } - } - return true - } - result[name] = append(result[name], f) + result[name] = append(result[name], makeFiltersFunction(name, false, true, string(keyname), vararr)) } } @@ -394,10 +389,7 @@ func findValueInOrderedValue(v Value, list []orderedValue) bool { } func makeRegexpFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_regexp string) func(tags element.Tags, key Key, closed bool) bool { - // Compile regular expression - // if not valid regexp --> panic ! - - // log.Warnf("Regexp filter is experimental! (tablename:" + tablename + ")") + // Compile regular expression, if not valid regexp --> panic ! r := regexp.MustCompile(v_regexp) return func(tags element.Tags, key Key, closed bool) bool { if v, ok := tags[v_keyname]; ok { @@ -410,9 +402,14 @@ func makeRegexpFiltersFunction(tablename string, virtualTrue bool, virtualFalse } func makeFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, v_keyname string, v_vararr []orderedValue) func(tags element.Tags, key Key, closed bool) bool { + + if findValueInOrderedValue("__nil__", v_vararr) { // check __nil__ + log.Print("warn: Filter value '__nil__' is not supported ! (tablename:" + tablename + ")") + } + if findValueInOrderedValue("__any__", v_vararr) { // check __any__ if len(v_vararr) > 1 { - log.Warnf("Multiple filter value with '__any__' keywords is probably not valid! (tablename:" + tablename + ")") + log.Print("warn: Multiple filter value with '__any__' keywords is not valid! (tablename:" + tablename + ")") } return func(tags element.Tags, key Key, closed bool) bool { if _, ok := tags[v_keyname]; ok { @@ -429,7 +426,7 @@ func makeFiltersFunction(tablename string, virtualTrue bool, virtualFalse bool, } return virtualFalse } - } else { + } else { // > 1 parameter - less optimal code return func(tags element.Tags, key Key, closed bool) bool { if v, ok := tags[v_keyname]; ok { if findValueInOrderedValue(Value(v), v_vararr) { diff --git a/mapping/config_test.go b/mapping/config_test.go index 0d44526..cb59ed0 100644 --- a/mapping/config_test.go +++ b/mapping/config_test.go @@ -11,12 +11,12 @@ func TestFilter_t0(t *testing.T) { /* ./config_test_mapping.yml .. - filters: - require: - boundary: ["administrative","maritime"] - mapping: - admin_level: ['2','4'] - type: linestring + filters: + require: + boundary: ["administrative","maritime"] + mapping: + admin_level: ['2','4'] + type: linestring */ filterTest( @@ -36,10 +36,14 @@ func TestFilter_t0(t *testing.T) { element.Tags{"admin_level": "0", "boundary": "administrative"}, element.Tags{"admin_level": "1", "boundary": "administrative"}, element.Tags{"admin_level": "2", "boundary": "postal_code"}, + element.Tags{"admin_level": "2", "boundary": ""}, + element.Tags{"admin_level": "2", "boundary": "__nil__"}, element.Tags{"admin_level": "4", "boundary": "census"}, element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"}, element.Tags{"admin_level": "2"}, element.Tags{"admin_level": "4"}, + element.Tags{"admin_level": "❤"}, + element.Tags{"admin_level": "__any__", "boundary": "__any__"}, element.Tags{"boundary": "administrative"}, element.Tags{"boundary": "maritime"}, element.Tags{"name": "maritime"}, @@ -80,10 +84,14 @@ func TestFilter_t1(t *testing.T) { element.Tags{"admin_level": "0", "boundary": "administrative"}, element.Tags{"admin_level": "1", "boundary": "administrative"}, element.Tags{"admin_level": "2", "boundary": "postal_code"}, + element.Tags{"admin_level": "2", "boundary": ""}, + element.Tags{"admin_level": "2", "boundary": "__nil__"}, element.Tags{"admin_level": "4", "boundary": "census"}, element.Tags{"admin_level": "3", "boundary": "administrative", "name": "NX"}, element.Tags{"admin_level": "2"}, element.Tags{"admin_level": "4"}, + element.Tags{"admin_level": "❤"}, + element.Tags{"admin_level": "__any__", "boundary": "__any__"}, element.Tags{"boundary": "administrative"}, element.Tags{"boundary": "maritime"}, element.Tags{"name": "maritime"}, @@ -121,6 +129,9 @@ func TestFilter_t2_building(t *testing.T) { element.Tags{"building": "garage", "addr:housenumber": "0"}, element.Tags{"building": "hut", "addr:housenumber": "99999999"}, element.Tags{"building": "_", "addr:housenumber": "333"}, + + element.Tags{"building": "__any__", "addr:housenumber": "333"}, + element.Tags{"building": "__nil__", "addr:housenumber": "333"}, element.Tags{"building": "y", "addr:housenumber": "1abcdefg"}, element.Tags{"building": "tower_block", "addr:housenumber": "1A"}, element.Tags{"building": "shed", "name": "N4", "addr:housenumber": "1AAA"}, @@ -134,6 +145,8 @@ func TestFilter_t2_building(t *testing.T) { element.Tags{"building": "house", "addr:housenumber": "two"}, element.Tags{"building": "residential", "addr:housenumber": "x21"}, + element.Tags{"building": "", "addr:housenumber": "111"}, + element.Tags{"building": "no"}, element.Tags{"building": "no", "addr:housenumber": "1a"}, element.Tags{"building": "No", "addr:housenumber": "1a"}, @@ -150,7 +163,7 @@ func TestFilter_t2_building(t *testing.T) { element.Tags{"building": "tower-block"}, element.Tags{"building": "❤"}, element.Tags{"building": "Ümlåütê"}, - element.Tags{"building": "中"}, + element.Tags{"building": "木"}, element.Tags{"building": "SheD", "name": "N4"}, element.Tags{"building": "oFFice", "name": "N4"}, element.Tags{"admin_level": "2"}, @@ -189,6 +202,7 @@ func TestFilter_t3_highway_with_name(t *testing.T) { element.Tags{"highway": "track", "name": "N3"}, element.Tags{"highway": "unclassified", "name": "N4"}, element.Tags{"highway": "path", "name": "N5"}, + element.Tags{"highway": "", "name": "🌍🌎🌏"}, element.Tags{"highway": "_", "name": "N6"}, element.Tags{"highway": "y", "name": "N7"}, element.Tags{"highway": "tower_block", "name": "N8"}, @@ -198,7 +212,7 @@ func TestFilter_t3_highway_with_name(t *testing.T) { element.Tags{"highway": "oFFice", "name": "N12"}, element.Tags{"highway": "❤", "name": "❤"}, element.Tags{"highway": "Ümlåütê", "name": "Ümlåütê"}, - element.Tags{"highway": "中", "name": "中"}, + element.Tags{"highway": "木", "name": "木"}, }, // Reject []element.Tags{ @@ -216,7 +230,7 @@ func TestFilter_t3_highway_with_name(t *testing.T) { element.Tags{"highway": "tower-block"}, element.Tags{"highway": "❤"}, element.Tags{"highway": "Ümlåütê"}, - element.Tags{"highway": "中"}, + element.Tags{"highway": "木"}, element.Tags{"admin_level": "2"}, element.Tags{"admin_level": "4"}, element.Tags{"boundary": "administrative"}, @@ -272,6 +286,13 @@ func TestFilter_t4_waterway_with_name(t *testing.T) { element.Tags{"waterway": "river", "name": "N2", "boat": "no"}, element.Tags{"waterway": "canal", "name": "N3"}, element.Tags{"waterway": "ditch", "name": "N4", "level": "3"}, + + element.Tags{"waterway": "stream", "name": "__any__"}, + element.Tags{"waterway": "stream", "name": "__nil__"}, + + element.Tags{"waterway": "stream", "name": "❤"}, + element.Tags{"waterway": "stream", "name": "木"}, + element.Tags{"waterway": "stream", "name": "Ümlåütê"}, }, // Reject []element.Tags{ @@ -294,7 +315,106 @@ func TestFilter_t4_waterway_with_name(t *testing.T) { element.Tags{"waterway": "oFFice", "name": "N12"}, element.Tags{"waterway": "❤", "name": "❤"}, element.Tags{"waterway": "Ümlåütê", "name": "Ümlåütê"}, - element.Tags{"waterway": "中", "name": "中"}, + element.Tags{"waterway": "木", "name": "木"}, + element.Tags{"waterway": "no", "name": "N1"}, + element.Tags{"waterway": "none", "name": "N2"}, + + element.Tags{"waterway": "yes"}, + element.Tags{"waterway": "no"}, + element.Tags{"waterway": "none"}, + element.Tags{"waterway": "tower-block"}, + element.Tags{"waterway": "❤"}, + element.Tags{"waterway": "Ümlåütê"}, + element.Tags{"waterway": "木"}, + + element.Tags{"waterway": "__nil__", "name": "__nil__"}, + element.Tags{"waterway": "__any__", "name": "__nil__"}, + + element.Tags{"waterway": "stream", "name": "__any__", "shop": "__any__"}, + element.Tags{"waterway": "stream", "name": "__nil__", "shop": "__any__"}, + element.Tags{"waterway": "stream", "name": "__any__", "shop": "__nil__"}, + element.Tags{"waterway": "stream", "name": "__nil__", "shop": "__nil__"}, + element.Tags{"waterway": "stream", "name": "__any__", "shop": ""}, + element.Tags{"waterway": "stream", "name": "__nil__", "shop": ""}, + + element.Tags{"admin_level": "2"}, + element.Tags{"admin_level": "4"}, + element.Tags{"boundary": "administrative"}, + element.Tags{"boundary": "maritime"}, + element.Tags{"name": "maritime"}, + }, + ) +} + +// go test ./mapping -run TestFilter_t5_depricated_exclude_tags -v +func TestFilter_t5_depricated_exclude_tags(t *testing.T) { + + /* ./config_test_mapping.yml .. + + filters: + require: + waterway: + - stream + - river + - canal + - drain + - ditch + exclude_tags: + - ['waterway', 'stream'] + - ['waterway', 'river'] + - ['waterway', 'canal'] + - ['waterway', 'drain'] + - ['waterway', 'ditch'] + mapping: + waterway: + - __any__ + type: linestring + + comment: + the `exclude_tags` will be converted to `reject` filter` + + */ + + filterTest( + // *testing.T + t, + // tablename + "config_test_t5_depricated_exclude_tags", + // Accept - in this case Must be EMPTY ! + []element.Tags{}, + // Reject + []element.Tags{ + element.Tags{"waterway": "stream", "name": "N1"}, + element.Tags{"waterway": "river", "name": "N2"}, + element.Tags{"waterway": "canal", "name": "N3"}, + element.Tags{"waterway": "drain", "name": "N4"}, + element.Tags{"waterway": "ditch", "name": "N5"}, + + element.Tags{"waterway": "stream", "name": "N1", "tunnel": "no"}, + element.Tags{"waterway": "river", "name": "N2", "boat": "no"}, + element.Tags{"waterway": "canal", "name": "N3"}, + element.Tags{"waterway": "ditch", "name": "N4", "level": "3"}, + + element.Tags{"waterway": "ditch", "name": "N1", "fixme": "incomplete"}, + element.Tags{"waterway": "stream", "name": "N1", "amenity": "parking"}, + element.Tags{"waterway": "river", "name": "N2", "shop": "hairdresser"}, + element.Tags{"waterway": "canal", "name": "N3", "building": "house"}, + element.Tags{"waterway": "drain", "name": "N1 tunnel", "tunnel": "yes"}, + + element.Tags{"waterway": "river", "name": "N4", "level": "unknown"}, + element.Tags{"waterway": "ditch", "name": "N4", "level": "primary"}, + + element.Tags{"waterway": "path", "name": "N5"}, + element.Tags{"waterway": "_", "name": "N6"}, + element.Tags{"waterway": "y", "name": "N7"}, + element.Tags{"waterway": "tower_block", "name": "N8"}, + element.Tags{"waterway": "shed", "name": "N9"}, + element.Tags{"waterway": "office", "name": "N10"}, + element.Tags{"waterway": "SheD", "name": "N11"}, + element.Tags{"waterway": "oFFice", "name": "N12"}, + element.Tags{"waterway": "❤", "name": "❤"}, + element.Tags{"waterway": "Ümlåütê", "name": "Ümlåütê"}, + element.Tags{"waterway": "木", "name": "木"}, element.Tags{"waterway": "no", "name": "N1"}, element.Tags{"waterway": "none", "name": "N2"}, @@ -304,7 +424,7 @@ func TestFilter_t4_waterway_with_name(t *testing.T) { element.Tags{"waterway": "tower-block"}, element.Tags{"waterway": "❤"}, element.Tags{"waterway": "Ümlåütê"}, - element.Tags{"waterway": "中"}, + element.Tags{"waterway": "木"}, element.Tags{"admin_level": "2"}, element.Tags{"admin_level": "4"}, element.Tags{"boundary": "administrative"}, diff --git a/mapping/config_test_mapping.yml b/mapping/config_test_mapping.yml index 0e03d61..c35f1f7 100644 --- a/mapping/config_test_mapping.yml +++ b/mapping/config_test_mapping.yml @@ -1,7 +1,6 @@ - -# test mappings +# test mappings for config_test.go # -# only `type: linestring` impemented in config_test.go +# only `type: linestring` impemented! # # Regexp Note: be careful because yaml interprets escape sequences inside double quoted strings! # so use single quoted string ! @@ -110,3 +109,36 @@ tables: waterway: - __any__ type: linestring + + + config_test_t5_depricated_exclude_tags: + _comment: Allways Empty ! + fields: + - name: id + type: id + - key: waterway + name: waterway + type: string + - key: name + name: name + type: string + filters: + require: + waterway: + - stream + - river + - canal + - drain + - ditch + # the `exclude_tags` will be converted to `reject` filter` + # and `require` and the `reject` filter will be same! + exclude_tags: + - ['waterway', 'stream'] + - ['waterway', 'river'] + - ['waterway', 'canal'] + - ['waterway', 'drain'] + - ['waterway', 'ditch'] + mapping: + waterway: + - __any__ + type: linestring