vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add hstore.include option

master
Oliver Tonnhofer 2016-11-16 16:27:00 +01:00
parent 2eb219be09
commit eea414dfbb
7 changed files with 121 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/index.rst
View File

@ -31,8 +31,8 @@ Generalized tables
Limit to polygons
It can limit imported geometries to polygons from GeoJSON.
HStore support
Don't know which tags you will be needing? Store all tags in an `HStore column <http://www.postgresql.org/docs/9.3/static/hstore.html>`_.
hstore support
Don't know which tags you will be needing? Store all tags in an `hstore column <http://www.postgresql.org/docs/9.6/static/hstore.html>`_.
Support

4
docs/mapping.rst
View File

@ -266,7 +266,9 @@ Area of polygon geometries in m². This field only works for the webmercator pro
``hstore_tags``
^^^^^^^^^^^^^^^
Stores all tags in a HStore column. Requires the PostGIS HStore extension. This will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to import all available tags.
Stores tags in an `hstore` column. Requires the `PostgreSQL hstore extension <http://www.postgresql.org/docs/9.6/static/hstore.html>`_. You can select tags with the ``include`` option, otherwise all tags will be inserted.
In any case, ``hstore_tags`` will only insert tags that are referenced in the ``mapping`` or ``columns`` of any table. See :ref:`tags` on how to make additional tags available for import.
.. TODO

29
mapping/fields.go
View File

@ -33,7 +33,7 @@ func init() {
"member_index": {"member_index", "int32", nil, nil, RelationMemberIndex, true},
"geometry": {"geometry", "geometry", Geometry, nil, nil, false},
"validated_geometry": {"validated_geometry", "validated_geometry", Geometry, nil, nil, false},
"hstore_tags": {"hstore_tags", "hstore_string", HstoreString, nil, nil, false},
"hstore_tags": {"hstore_tags", "hstore_string", nil, MakeHStoreString, nil, false},
"wayzorder": {"wayzorder", "int32", nil, MakeWayZOrder, nil, false},
"pseudoarea": {"pseudoarea", "float32", nil, MakePseudoArea, nil, false},
"area": {"area", "float32", Area, nil, nil, false},
@ -248,12 +248,29 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M
var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
func HstoreString(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
tags := make([]string, 0, len(elem.Tags))
for k, v := range elem.Tags {
tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`)
func MakeHStoreString(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {
var includeAll bool
var err error
var include map[string]int
if _, ok := field.Args["include"]; !ok {
includeAll = true
} else {
include, err = decodeEnumArg(field, "include")
if err != nil {
return nil, err
}
}
return strings.Join(tags, ", ")
hstoreString := func(val string, elem *element.OSMElem, geom *geom.Geometry, match Match) interface{} {
tags := make([]string, 0, len(elem.Tags))
for k, v := range elem.Tags {
if includeAll || include[k] != 0 {
tags = append(tags, `"`+hstoreReplacer.Replace(k)+`"=>"`+hstoreReplacer.Replace(v)+`"`)
}
}
return strings.Join(tags, ", ")
}
return hstoreString, nil
}
func MakeWayZOrder(fieldName string, fieldType FieldType, field Field) (MakeValue, error) {

56
mapping/fields_test.go
View File

@ -297,16 +297,50 @@ func TestMakeSuffixReplace(t *testing.T) {
}
}
func assertEq(t *testing.T, a, b string) {
if a != b {
t.Errorf("'%v' != '%v'", a, b)
}
}
func TestHstoreString(t *testing.T) {
match := Match{}
assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"key": "value"}}, nil, match).(string), `"key"=>"value"`)
assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`"key"`: `'"value"'`}}, nil, match).(string), `"\"key\""=>"'\"value\"'"`)
assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{`\`: `\\\\`}}, nil, match).(string), `"\\"=>"\\\\\\\\"`)
assertEq(t, HstoreString("", &element.OSMElem{Tags: element.Tags{"Ümlåütê=>": ""}}, nil, match).(string), `"Ümlåütê=>"=>""`)
field := Field{
Name: "tags",
Type: "hstore_tags",
}
hstoreAll, err := MakeHStoreString("tags", FieldType{}, field)
if err != nil {
t.Fatal(err)
}
field = Field{
Name: "tags",
Type: "hstore_tags",
Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}},
}
hstoreInclude, err := MakeHStoreString("tags", FieldType{}, field)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
field MakeValue
tags element.Tags
expected interface{}
}{
{hstoreAll, element.Tags{}, ``},
{hstoreAll, element.Tags{"key": "value"}, `"key"=>"value"`},
{hstoreAll, element.Tags{`"key"`: `'"value"'`}, `"\"key\""=>"'\"value\"'"`},
{hstoreAll, element.Tags{`\`: `\\\\`}, `"\\"=>"\\\\\\\\"`},
{hstoreAll, element.Tags{"Ümlåütê=>": ""}, `"Ümlåütê=>"=>""`},
{hstoreInclude, element.Tags{"key": "value"}, ``},
{hstoreInclude, element.Tags{"key1": "value"}, `"key1"=>"value"`},
{hstoreInclude, element.Tags{"key": "value", "key2": "value"}, `"key2"=>"value"`},
} {
actual := test.field("", &element.OSMElem{Tags: test.tags}, nil, Match{})
if actual.(string) != test.expected {
t.Errorf("%#v != %#v for %#v", actual, test.expected, test.tags)
}
}
actual := hstoreAll("", &element.OSMElem{Tags: element.Tags{"key1": "value", "key2": "value"}}, nil, Match{})
// check mutliple tags, can be in any order
if actual.(string) != `"key1"=>"value", "key2"=>"value"` && actual.(string) != `"key2"=>"value", "key1"=>"value"` {
t.Error("unexpected value", actual)
}
}

22
test/complete_db.osm
View File

@ -1200,4 +1200,26 @@
<tag k="area" v="no"/>
</way>
<!-- test hstore: check only included tags are inserted -->
<node id="401101" version="1" timestamp="2011-11-11T00:11:11Z" lat="47.5" lon="13"/>
<node id="401102" version="1" timestamp="2011-11-11T00:11:11Z" lat="50" lon="14.5"/>
<node id="401103" version="1" timestamp="2011-11-11T00:11:11Z" lat="49" lon="16.5"/>
<node id="401104" version="1" timestamp="2011-11-11T00:11:11Z" lat="47" lon="17"/>
<node id="401105" version="1" timestamp="2011-11-11T00:11:11Z" lat="45.5" lon="14.5"/>
<way id="401151" version="1" timestamp="2011-11-11T00:11:11Z">
<nd ref="401101"/>
<nd ref="401102"/>
<nd ref="401103"/>
<nd ref="401104"/>
<nd ref="401105"/>
<nd ref="401101"/>
<tag k="name" v="hstore test"/>
<tag k="building" v="yes"/>
<tag k="opening_hours" v="24/7"/>
<tag k="amenity" v="fuel"/>
<tag k="leisure" v="not added" />
</way>
</osm>

18
test/complete_db_mapping.json
View File

@ -1,4 +1,11 @@
{
"tags": {
"include": [
"shop",
"amenity",
"opening_hours"
]
},
"areas": {
"area_tags": ["leisure"],
"linear_tags": ["highway"],
@ -204,10 +211,16 @@
"type": "mapping_value",
"name": "type",
"key": null
},
{
"type": "hstore_tags",
"name": "tags",
"args": {
"include": ["amenity", "shop", "opening_hours"]
}
}
],
"type": "polygon",
"filters": {"areas": {}},
"mapping": {
"building": [
"__any__"
@ -716,9 +729,6 @@
}
],
"type": "linestring",
"filters": {
"areas": {},
},
"mappings": {
"railway": {
"mapping": {

12
test/completedb_test.go
View File

@ -386,10 +386,22 @@ func TestComplete_AreaMapping(t *testing.T) {
})
}
func TestComplete_HstoreTags(t *testing.T) {
// Mapping type dependent area-defaults.
assertHstore(t, []checkElem{
{"osm_buildings", 401151, "*", map[string]string{"amenity": "fuel", "opening_hours": "24/7"}},
})
}
// #######################################################################
func TestComplete_Update(t *testing.T) {
ts.updateOsm(t, "./build/complete_db.osc.gz")
}
// #######################################################################
func TestComplete_NoDuplicates(t *testing.T) {
// Relations/ways are only inserted once Checks #66