add hstore.include option

docs/index.rst

@@ -31,8 +31,8 @@ Generalized tables
 Limit to polygons
   It can limit imported geometries to polygons from GeoJSON.
 
-HStore support
+hstore support
-  Don't know which tags you will be needing? Store all tags in an `HStore column <http://www.postgresql.org/docs/9.3/static/hstore.html>`_.
+  Don't know which tags you will be needing? Store all tags in an `hstore column <http://www.postgresql.org/docs/9.6/static/hstore.html>`_.
 
 
 Support

docs/mapping.rst

@@ -266,7 +266,9 @@ Area of polygon geometries in m². This field only works for the webmercator pro
 ``hstore_tags``
 ^^^^^^^^^^^^^^^
 
-Stores all tags in a HStore column. Requires the PostGIS HStore extension. This will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to import all available tags.
+Stores tags in an `hstore` column. Requires the `PostgreSQL hstore extension <http://www.postgresql.org/docs/9.6/static/hstore.html>`_. You can select tags with the ``include`` option, otherwise all tags will be inserted.
+
+In any case, ``hstore_tags`` will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to make additional tags available for import.
 
 
 .. TODO

mapping/fields.go

@@ -33,7 +33,7 @@ func init() {
 		"member_index":         {"member_index", "int32", nil, nil, RelationMemberIndex, true},
 		"geometry":             {"geometry", "geometry", Geometry, nil, nil, false},
 		"validated_geometry":   {"validated_geometry", "validated_geometry", Geometry, nil, nil, false},
-		"hstore_tags":          {"hstore_tags", "hstore_string", HstoreString, nil, nil, false},
+		"hstore_tags":          {"hstore_tags", "hstore_string", nil, MakeHStoreString, nil, false},
 		"wayzorder":            {"wayzorder", "int32", nil, MakeWayZOrder, nil, false},
 		"pseudoarea":           {"pseudoarea", "float32", nil, MakePseudoArea, nil, false},
 		"area":                 {"area", "float32", Area, nil, nil, false},
@@ -248,12 +248,29 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M
 
 var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-func HstoreString(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
+func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
-	tags := make([]string, 0, len(elem.Tags))
+	var includeAll bool
-	for k, v := range elem.Tags {
+	var err error
-		tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`)
+	var include map[string]int
+	if _, ok := field.Args["include"]; !ok {
+		includeAll = true
+	} else {
+		include, err = decodeEnumArg(field, "include")
+		if err != nil {
+			return nil, err
+		}
+
 	}
-	return strings.Join(tags, ", ")
+	hstoreString := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
+		tags := make([]string, 0, len(elem.Tags))
+		for k, v := range elem.Tags {
+			if includeAll || include[k] != 0 {
+				tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`)
+			}
+		}
+		return strings.Join(tags, ", ")
+	}
+	return hstoreString, nil
 }
 
 func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {

mapping/fields_test.go

@@ -297,16 +297,50 @@ func TestMakeSuffixReplace(t *testing.T) {
 	}
 }
 
-func assertEq(t *testing.T, a, b string) {
-	if a != b {
-		t.Errorf("'%v' != '%v'", a, b)
-	}
-}
-
 func TestHstoreString(t *testing.T) {
-	match := Match{}
+	field := Field{
-	assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"key": "value"}}, nil, match).(string), `"key"=>"value"`)
+		Name: "tags",
-	assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`"key"`: `'"value"'`}}, nil, match).(string), `"\"key\""=>"'\"value\"'"`)
+		Type: "hstore_tags",
-	assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`\`: `\\\\`}}, nil, match).(string), `"\\"=>"\\\\\\\\"`)
+	}
-	assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"Ümlåütê=>": ""}}, nil, match).(string), `"Ümlåütê=>"=>""`)
+	hstoreAll, err := MakeHStoreString("tags", FieldType{}, field)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	field = Field{
+		Name: "tags",
+		Type: "hstore_tags",
+		Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}},
+	}
+	hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range []struct {
+		field    MakeValue
+		tags     element.Tags
+		expected interface{}
+	}{
+		{hstoreAll, element.Tags{}, ``},
+		{hstoreAll, element.Tags{"key": "value"}, `"key"=>"value"`},
+		{hstoreAll, element.Tags{`"key"`: `'"value"'`}, `"\"key\""=>"'\"value\"'"`},
+		{hstoreAll, element.Tags{`\`: `\\\\`}, `"\\"=>"\\\\\\\\"`},
+		{hstoreAll, element.Tags{"Ümlåütê=>": ""}, `"Ümlåütê=>"=>""`},
+		{hstoreInclude, element.Tags{"key": "value"}, ``},
+		{hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`},
+		{hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`},
+	} {
+		actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{})
+		if actual.(string) != test.expected {
+			t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags)
+		}
+	}
+
+	actual := hstoreAll("", &element.OSMElem{Tags: element.Tags{"key1": "value", "key2": "value"}}, nil, Match{})
+	// check mutliple tags, can be in any order
+	if actual.(string) != `"key1"=>"value", "key2"=>"value"` && actual.(string) != `"key2"=>"value", "key1"=>"value"` {
+		t.Error("unexpected value", actual)
+	}
+
 }

test/complete_db.osm

@@ -1200,4 +1200,26 @@
     <tag k="area" v="no"/>
   </way>
 
+
+  <!-- test hstore: check only included tags are inserted -->
+  <node id="401101" version="1" timestamp="2011-11-11T00:11:11Z" lat="47.5" lon="13"/>
+  <node id="401102" version="1" timestamp="2011-11-11T00:11:11Z" lat="50" lon="14.5"/>
+  <node id="401103" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="16.5"/>
+  <node id="401104" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="17"/>
+  <node id="401105" version="1" timestamp="2011-11-11T00:11:11Z" lat="45.5" lon="14.5"/>
+
+  <way id="401151" version="1" timestamp="2011-11-11T00:11:11Z">
+    <nd ref="401101"/>
+    <nd ref="401102"/>
+    <nd ref="401103"/>
+    <nd ref="401104"/>
+    <nd ref="401105"/>
+    <nd ref="401101"/>
+    <tag k="name" v="hstore test"/>
+    <tag k="building" v="yes"/>
+    <tag k="opening_hours" v="24/7"/>
+    <tag k="amenity" v="fuel"/>
+    <tag k="leisure" v="not added" />
+  </way>
+
 </osm>

test/complete_db_mapping.json

@@ -1,4 +1,11 @@
 {
+    "tags": {
+        "include": [
+            "shop",
+            "amenity",
+            "opening_hours"
+        ]
+    },
     "areas": {
         "area_tags": ["leisure"],
         "linear_tags": ["highway"],
@@ -204,10 +211,16 @@
                     "type": "mapping_value",
                     "name": "type",
                     "key": null
+                },
+                {
+                    "type": "hstore_tags",
+                    "name": "tags",
+                    "args": {
+                        "include": ["amenity", "shop", "opening_hours"]
+                    }
                 }
             ],
             "type": "polygon",
-            "filters": {"areas": {}},
             "mapping": {
                 "building": [
                     "__any__"
@@ -716,9 +729,6 @@
                 }
             ],
             "type": "linestring",
-            "filters": {
-                "areas": {},
-            },
             "mappings": {
                 "railway": {
                     "mapping": {

test/completedb_test.go

@@ -386,10 +386,22 @@ func TestComplete_AreaMapping(t *testing.T) {
 	})
 }
 
+func TestComplete_HstoreTags(t *testing.T) {
+	// Mapping type dependent area-defaults.
+
+	assertHstore(t, []checkElem{
+		{"osm_buildings", 401151, "*", map[string]string{"amenity": "fuel", "opening_hours": "24/7"}},
+	})
+}
+
+// #######################################################################
+
 func TestComplete_Update(t *testing.T) {
 	ts.updateOsm(t, "./build/complete_db.osc.gz")
 }
 
+// #######################################################################
+
 func TestComplete_NoDuplicates(t *testing.T) {
 	// Relations/ways are only inserted once Checks #66