From eea414dfbb3038908681df0b0be08c0e61f18642 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Wed, 16 Nov 2016 16:27:00 +0100 Subject: [PATCH] add hstore.include option --- docs/index.rst | 4 +-- docs/mapping.rst | 4 ++- mapping/fields.go | 29 ++++++++++++++---- mapping/fields_test.go | 56 ++++++++++++++++++++++++++++------- test/complete_db.osm | 22 ++++++++++++++ test/complete_db_mapping.json | 18 ++++++++--- test/completedb_test.go | 12 ++++++++ 7 files changed, 121 insertions(+), 24 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7b950b9..2983a28 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,8 +31,8 @@ Generalized tables Limit to polygons It can limit imported geometries to polygons from GeoJSON. -HStore support - Don't know which tags you will be needing? Store all tags in an `HStore column `_. +hstore support + Don't know which tags you will be needing? Store all tags in an `hstore column `_. Support diff --git a/docs/mapping.rst b/docs/mapping.rst index 6093351..32161b1 100644 --- a/docs/mapping.rst +++ b/docs/mapping.rst @@ -266,7 +266,9 @@ Area of polygon geometries in m². This field only works for the webmercator pro ``hstore_tags`` ^^^^^^^^^^^^^^^ -Stores all tags in a HStore column. Requires the PostGIS HStore extension. This will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to import all available tags. +Stores tags in an `hstore` column. Requires the `PostgreSQL hstore extension `_. You can select tags with the ``include`` option, otherwise all tags will be inserted. + +In any case, ``hstore_tags`` will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to make additional tags available for import. .. TODO diff --git a/mapping/fields.go b/mapping/fields.go index 8804f63..3a51f35 100644 --- a/mapping/fields.go +++ b/mapping/fields.go @@ -33,7 +33,7 @@ func init() { "member_index": {"member_index", "int32", nil, nil, RelationMemberIndex, true}, "geometry": {"geometry", "geometry", Geometry, nil, nil, false}, "validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil, nil, false}, - "hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil, nil, false}, + "hstore_tags": {"hstore_tags", "hstore_string", nil, MakeHStoreString, nil, false}, "wayzorder": {"wayzorder", "int32", nil, MakeWayZOrder, nil, false}, "pseudoarea": {"pseudoarea", "float32", nil, MakePseudoArea, nil, false}, "area": {"area", "float32", Area, nil, nil, false}, @@ -248,12 +248,29 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") -func HstoreString(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { - tags := make([]string, 0, len(elem.Tags)) - for k, v := range elem.Tags { - tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`) +func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { + var includeAll bool + var err error + var include map[string]int + if _, ok := field.Args["include"]; !ok { + includeAll = true + } else { + include, err = decodeEnumArg(field, "include") + if err != nil { + return nil, err + } + } - return strings.Join(tags, ", ") + hstoreString := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} { + tags := make([]string, 0, len(elem.Tags)) + for k, v := range elem.Tags { + if includeAll || include[k] != 0 { + tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`) + } + } + return strings.Join(tags, ", ") + } + return hstoreString, nil } func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) { diff --git a/mapping/fields_test.go b/mapping/fields_test.go index ba261ed..b71fb99 100644 --- a/mapping/fields_test.go +++ b/mapping/fields_test.go @@ -297,16 +297,50 @@ func TestMakeSuffixReplace(t *testing.T) { } } -func assertEq(t *testing.T, a, b string) { - if a != b { - t.Errorf("'%v' != '%v'", a, b) - } -} - func TestHstoreString(t *testing.T) { - match := Match{} - assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"key": "value"}}, nil, match).(string), `"key"=>"value"`) - assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`"key"`: `'"value"'`}}, nil, match).(string), `"\"key\""=>"'\"value\"'"`) - assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`\`: `\\\\`}}, nil, match).(string), `"\\"=>"\\\\\\\\"`) - assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"Ümlåütê=>": ""}}, nil, match).(string), `"Ümlåütê=>"=>""`) + field := Field{ + Name: "tags", + Type: "hstore_tags", + } + hstoreAll, err := MakeHStoreString("tags", FieldType{}, field) + if err != nil { + t.Fatal(err) + } + + field = Field{ + Name: "tags", + Type: "hstore_tags", + Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}}, + } + hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + field MakeValue + tags element.Tags + expected interface{} + }{ + {hstoreAll, element.Tags{}, ``}, + {hstoreAll, element.Tags{"key": "value"}, `"key"=>"value"`}, + {hstoreAll, element.Tags{`"key"`: `'"value"'`}, `"\"key\""=>"'\"value\"'"`}, + {hstoreAll, element.Tags{`\`: `\\\\`}, `"\\"=>"\\\\\\\\"`}, + {hstoreAll, element.Tags{"Ümlåütê=>": ""}, `"Ümlåütê=>"=>""`}, + {hstoreInclude, element.Tags{"key": "value"}, ``}, + {hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`}, + {hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`}, + } { + actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{}) + if actual.(string) != test.expected { + t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags) + } + } + + actual := hstoreAll("", &element.OSMElem{Tags: element.Tags{"key1": "value", "key2": "value"}}, nil, Match{}) + // check mutliple tags, can be in any order + if actual.(string) != `"key1"=>"value", "key2"=>"value"` && actual.(string) != `"key2"=>"value", "key1"=>"value"` { + t.Error("unexpected value", actual) + } + } diff --git a/test/complete_db.osm b/test/complete_db.osm index a2f4882..ed5ceb4 100644 --- a/test/complete_db.osm +++ b/test/complete_db.osm @@ -1200,4 +1200,26 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/complete_db_mapping.json b/test/complete_db_mapping.json index 4555cbe..874926a 100644 --- a/test/complete_db_mapping.json +++ b/test/complete_db_mapping.json @@ -1,4 +1,11 @@ { + "tags": { + "include": [ + "shop", + "amenity", + "opening_hours" + ] + }, "areas": { "area_tags": ["leisure"], "linear_tags": ["highway"], @@ -204,10 +211,16 @@ "type": "mapping_value", "name": "type", "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "args": { + "include": ["amenity", "shop", "opening_hours"] + } } ], "type": "polygon", - "filters": {"areas": {}}, "mapping": { "building": [ "__any__" @@ -716,9 +729,6 @@ } ], "type": "linestring", - "filters": { - "areas": {}, - }, "mappings": { "railway": { "mapping": { diff --git a/test/completedb_test.go b/test/completedb_test.go index 4adb8a0..eb82d4a 100644 --- a/test/completedb_test.go +++ b/test/completedb_test.go @@ -386,10 +386,22 @@ func TestComplete_AreaMapping(t *testing.T) { }) } +func TestComplete_HstoreTags(t *testing.T) { + // Mapping type dependent area-defaults. + + assertHstore(t, []checkElem{ + {"osm_buildings", 401151, "*", map[string]string{"amenity": "fuel", "opening_hours": "24/7"}}, + }) +} + +// ####################################################################### + func TestComplete_Update(t *testing.T) { ts.updateOsm(t, "./build/complete_db.osc.gz") } +// ####################################################################### + func TestComplete_NoDuplicates(t *testing.T) { // Relations/ways are only inserted once Checks #66