Also parse YAML as mapping file

master
Yohan Boniface 2015-08-11 19:22:01 +02:00 committed by Oliver Tonnhofer
parent 063730b8e8
commit 690668f5d5
5 changed files with 1260 additions and 38 deletions

616
example-mapping.yml Normal file
View File

@ -0,0 +1,616 @@
generalized_tables:
landusages_gen0:
source: landusages_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
landusages_gen1:
source: landusages
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
roads_gen0:
source: roads_gen1
sql_filter: null
tolerance: 200.0
roads_gen1:
source: roads
sql_filter: type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary',
'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link')
OR class IN('railway')
tolerance: 50.0
waterareas_gen0:
source: waterareas_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
waterareas_gen1:
source: waterareas
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
waterways_gen0:
source: waterways_gen1
sql_filter: null
tolerance: 200
waterways_gen1:
source: waterways
sql_filter: null
tolerance: 50.0
tables:
admin:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: admin_level
name: admin_level
type: integer
mapping:
boundary:
- administrative
type: polygon
aeroways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
aeroway:
- runway
- taxiway
type: linestring
amenities:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
amenity:
- university
- school
- library
- fuel
- hospital
- fire_station
- police
- townhall
type: point
barrierpoints:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- block
- bollard
- cattle_grid
- chain
- cycle_barrier
- entrance
- horse_stile
- gate
- spikes
- lift_gate
- kissing_gate
- fence
- 'yes'
- wire_fence
- toll_booth
- stile
type: point
barrierways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- city_wall
- fence
- hedge
- retaining_wall
- wall
- bollard
- gate
- spikes
- lift_gate
- kissing_gate
- embankment
- 'yes'
- wire_fence
type: linestring
buildings:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
building:
- __any__
type: polygon
housenumbers:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
mapping:
addr:housenumber:
- __any__
type: point
housenumbers_interpolated:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
- key: addr:inclusion
name: addr:inclusion
type: string
mapping:
addr:interpolation:
- __any__
type: linestring
landusages:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: null
name: area
type: pseudoarea
- args:
ranks:
- pedestrian
- footway
- playground
- park
- forest
- cemetery
- farmyard
- farm
- farmland
- wood
- meadow
- grass
- wetland
- village_green
- recreation_ground
- garden
- sports_centre
- pitch
- common
- allotments
- golf_course
- university
- school
- college
- library
- baracks
- fuel
- parking
- nature_reserve
- cinema
- theatre
- place_of_worship
- hospital
- scrub
- orchard
- vineyard
- zoo
- quarry
- residential
- retail
- commercial
- industrial
- railway
- heath
- island
- land
key: z_order
name: z_order
type: zorder
mapping:
aeroway:
- runway
- taxiway
amenity:
- university
- school
- college
- library
- fuel
- parking
- cinema
- theatre
- place_of_worship
- hospital
barrier:
- hedge
highway:
- pedestrian
- footway
landuse:
- park
- forest
- residential
- retail
- commercial
- industrial
- railway
- cemetery
- grass
- farmyard
- farm
- farmland
- orchard
- vineyard
- wood
- meadow
- village_green
- recreation_ground
- allotments
- quarry
leisure:
- park
- garden
- playground
- golf_course
- sports_centre
- pitch
- stadium
- common
- nature_reserve
man_made:
- pier
military:
- barracks
natural:
- wood
- land
- scrub
- wetland
- heath
place:
- island
tourism:
- zoo
type: polygon
places:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- args:
ranks:
- country
- state
- region
- county
- city
- town
- village
- hamlet
- suburb
- locality
key: z_order
name: z_order
type: zorder
- key: population
name: population
type: integer
mapping:
place:
- country
- state
- region
- county
- city
- town
- village
- hamlet
- suburb
- locality
type: point
roads:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: null
name: type
type: mapping_value
- key: name
name: name
type: string
- key: tunnel
name: tunnel
type: boolint
- key: bridge
name: bridge
type: boolint
- key: oneway
name: oneway
type: direction
- key: ref
name: ref
type: string
- key: layer
name: z_order
type: wayzorder
- key: access
name: access
type: string
- key: service
name: service
type: string
- key: null
name: class
type: mapping_key
filters:
exclude_tags:
- - area
- 'yes'
mappings:
railway:
mapping:
railway:
- rail
- tram
- light_rail
- subway
- narrow_gauge
- preserved
- funicular
- monorail
- disused
roads:
mapping:
highway:
- motorway
- motorway_link
- trunk
- trunk_link
- primary
- primary_link
- secondary
- secondary_link
- tertiary
- tertiary_link
- road
- path
- track
- service
- footway
- bridleway
- cycleway
- steps
- pedestrian
- living_street
- unclassified
- residential
- raceway
man_made:
- pier
- groyne
type: linestring
transport_areas:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- apron
railway:
- station
- platform
type: polygon
transport_points:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: ref
name: ref
type: string
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- gate
highway:
- motorway_junction
- turning_circle
- bus_stop
railway:
- station
- halt
- tram_stop
- crossing
- level_crossing
- subway_entrance
type: point
waterareas:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: null
name: area
type: pseudoarea
mapping:
amenity:
- swimming_pool
landuse:
- basin
- reservoir
leisure:
- swimming_pool
natural:
- water
waterway:
- riverbank
type: polygon
waterways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- ditch
waterway:
- stream
- river
- canal
- drain
- ditch
type: linestring

View File

@ -1,41 +1,42 @@
package mapping
import (
"encoding/json"
"errors"
"os"
"io/ioutil"
"github.com/omniscale/imposm3/element"
"gopkg.in/yaml.v2"
)
type Field struct {
Name string `json:"name"`
Key Key `json:"key"`
Keys []Key `json:"keys"`
Type string `json:"type"`
Args map[string]interface{} `json:"args"`
Name string `yaml:"name"`
Key Key `yaml:"key"`
Keys []Key `yaml:"keys"`
Type string `yaml:"type"`
Args map[string]interface{} `yaml:"args"`
}
type Table struct {
Name string
Type TableType `json:"type"`
Mapping map[Key][]Value `json:"mapping"`
Mappings map[string]SubMapping `json:"mappings"`
TypeMappings TypeMappings `json:"type_mappings"`
Fields []*Field `json:"columns"` // TODO rename Fields internaly to Columns
OldFields []*Field `json:"fields"`
Filters *Filters `json:"filters"`
Type TableType `yaml:"type"`
Mapping map[Key][]Value `yaml:"mapping"`
Mappings map[string]SubMapping `yaml:"mappings"`
TypeMappings TypeMappings `yaml:"type_mappings"`
Fields []*Field `yaml:"columns"` // TODO rename Fields internaly to Columns
OldFields []*Field `yaml:"fields"`
Filters *Filters `yaml:"filters"`
}
type GeneralizedTable struct {
Name string
SourceTableName string `json:"source"`
Tolerance float64 `json:"tolerance"`
SqlFilter string `json:"sql_filter"`
SourceTableName string `yaml:"source"`
Tolerance float64 `yaml:"tolerance"`
SqlFilter string `yaml:"sql_filter"`
}
type Filters struct {
ExcludeTags *[][2]string `json:"exclude_tags"`
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Tables map[string]*Table
@ -43,17 +44,17 @@ type Tables map[string]*Table
type GeneralizedTables map[string]*GeneralizedTable
type Mapping struct {
Tables Tables `json:"tables"`
GeneralizedTables GeneralizedTables `json:"generalized_tables"`
Tags Tags `json:"tags"`
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `json:"use_single_id_space"`
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Tags struct {
LoadAll bool `json:"load_all"`
Exclude []Key `json:"exclude"`
LoadAll bool `yaml:"load_all"`
Exclude []Key `yaml:"exclude"`
}
type SubMapping struct {
@ -61,9 +62,9 @@ type SubMapping struct {
}
type TypeMappings struct {
Points map[Key][]Value `json:"points"`
LineStrings map[Key][]Value `json:"linestrings"`
Polygons map[Key][]Value `json:"polygons"`
Points map[Key][]Value `yaml:"points"`
LineStrings map[Key][]Value `yaml:"linestrings"`
Polygons map[Key][]Value `yaml:"polygons"`
}
type ElementFilter func(tags *element.Tags) bool
@ -103,15 +104,13 @@ const (
)
func NewMapping(filename string) (*Mapping, error) {
f, err := os.Open(filename)
f, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
defer f.Close()
decoder := json.NewDecoder(f)
mapping := Mapping{}
err = decoder.Decode(&mapping)
err = yaml.Unmarshal(f, &mapping)
if err != nil {
return nil, err
}

View File

@ -11,7 +11,7 @@ var mapping *Mapping
func init() {
var err error
mapping, err = NewMapping("./test_mapping.json")
mapping, err = NewMapping("./test_mapping.yml")
if err != nil {
panic(err)
}

View File

@ -7,7 +7,7 @@ import (
)
func BenchmarkTagMatch(b *testing.B) {
m, err := NewMapping("matcher_test_mapping.json")
m, err := NewMapping("matcher_test_mapping.yml")
if err != nil {
b.Fatal(err)
}
@ -38,7 +38,7 @@ func makeMemberRole(id int64, tags element.Tags, role string) element.Member {
}
func TestSelectRelationPolygonsSimple(t *testing.T) {
mapping, err := NewMapping("test_mapping.json")
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
@ -63,7 +63,7 @@ func TestSelectRelationPolygonsSimple(t *testing.T) {
}
func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
mapping, err := NewMapping("test_mapping.json")
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
@ -86,7 +86,7 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) {
}
func TestSelectRelationPolygonsMultiple(t *testing.T) {
mapping, err := NewMapping("test_mapping.json")
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
@ -112,7 +112,7 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) {
}
func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
mapping, err := NewMapping("test_mapping.json")
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}
@ -134,7 +134,7 @@ func TestSelectRelationPolygonsMultipleTags(t *testing.T) {
}
func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) {
mapping, err := NewMapping("test_mapping.json")
mapping, err := NewMapping("test_mapping.yml")
if err != nil {
t.Fatal(err)
}

607
mapping/test_mapping.yml Normal file
View File

@ -0,0 +1,607 @@
generalized_tables:
landusages_gen0:
source: landusages_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
landusages_gen1:
source: landusages
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
roads_gen0:
source: roads_gen1
sql_filter: null
tolerance: 200.0
roads_gen1:
source: roads
sql_filter: type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary',
'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link')
OR class IN('railway')
tolerance: 50.0
waterareas_gen0:
source: waterareas_gen1
sql_filter: ST_Area(geometry)>500000.000000
tolerance: 200.0
waterareas_gen1:
source: waterareas
sql_filter: ST_Area(geometry)>50000.000000
tolerance: 50.0
waterways_gen0:
source: waterways_gen1
sql_filter: null
tolerance: 200
waterways_gen1:
source: waterways
sql_filter: null
tolerance: 50.0
tables:
admin:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: admin_level
name: admin_level
type: integer
mapping:
boundary:
- administrative
type: polygon
aeroways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
aeroway:
- runway
- taxiway
type: linestring
amenities:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
amenity:
- university
- school
- library
- fuel
- hospital
- fire_station
- police
- townhall
type: point
amenity_areas:
_comment: for testing duplicate inserts with __any__ and exact match
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
building:
- shop
type: polygon
barrierpoints:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- block
- bollard
- cattle_grid
- chain
- cycle_barrier
- entrance
- horse_stile
- gate
- spikes
- lift_gate
- kissing_gate
- fence
- 'yes'
- wire_fence
- toll_booth
- stile
type: point
barrierways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- city_wall
- fence
- hedge
- retaining_wall
- wall
- bollard
- gate
- spikes
- lift_gate
- kissing_gate
- embankment
- 'yes'
- wire_fence
type: linestring
buildings:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
building:
- __any__
type: polygon
housenumbers:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
mapping:
addr:housenumber:
- __any__
type: point
housenumbers_interpolated:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: addr:street
name: addr:street
type: string
- key: addr:postcode
name: addr:postcode
type: string
- key: addr:city
name: addr:city
type: string
- key: addr:inclusion
name: addr:inclusion
type: string
mapping:
addr:interpolation:
- __any__
type: linestring
landusages:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: null
name: area
type: pseudoarea
- args:
ranks:
- pedestrian
- footway
- playground
- park
- forest
- cemetery
- farmyard
- farm
- farmland
- wood
- meadow
- grass
- wetland
- village_green
- recreation_ground
- garden
- sports_centre
- pitch
- common
- allotments
- golf_course
- university
- school
- college
- library
- baracks
- fuel
- parking
- nature_reserve
- cinema
- theatre
- place_of_worship
- hospital
- scrub
- orchard
- vineyard
- zoo
- quarry
- residential
- retail
- commercial
- industrial
- railway
- heath
- island
- land
key: z_order
name: z_order
type: zorder
mapping:
aeroway:
- runway
- taxiway
amenity:
- university
- school
- college
- library
- fuel
- parking
- cinema
- theatre
- place_of_worship
- hospital
barrier:
- hedge
highway:
- pedestrian
- footway
landuse:
- park
- forest
- residential
- retail
- commercial
- industrial
- railway
- cemetery
- grass
- farmyard
- farm
- farmland
- orchard
- vineyard
- wood
- meadow
- village_green
- recreation_ground
- allotments
- quarry
leisure:
- park
- garden
- playground
- golf_course
- sports_centre
- pitch
- stadium
- common
- nature_reserve
man_made:
- pier
military:
- barracks
natural:
- wood
- water
- land
- scrub
- wetland
- heath
place:
- island
tourism:
- zoo
waterway:
- riverbank
type: polygon
places:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- args:
ranks:
- country
- state
- region
- county
- city
- town
- village
- hamlet
- suburb
- locality
key: z_order
name: z_order
type: zorder
- key: population
name: population
type: integer
mapping:
place:
- country
- state
- region
- county
- city
- town
- village
- hamlet
- suburb
- locality
type: point
roads:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: null
name: type
type: mapping_value
- key: name
name: name
type: string
- key: tunnel
name: tunnel
type: boolint
- key: bridge
name: bridge
type: boolint
- key: oneway
name: oneway
type: direction
- key: ref
name: ref
type: string
- key: layer
name: z_order
type: wayzorder
- key: access
name: access
type: string
- key: service
name: service
type: string
- key: null
name: class
type: mapping_key
filters:
exclude_tags:
- ["area", "yes"]
mappings:
railway:
mapping:
railway:
- rail
- tram
- light_rail
- subway
- narrow_gauge
- preserved
- funicular
- monorail
- disused
roads:
mapping:
highway:
- motorway
- motorway_link
- trunk
- trunk_link
- primary
- primary_link
- secondary
- secondary_link
- tertiary
- tertiary_link
- road
- path
- track
- service
- footway
- bridleway
- cycleway
- steps
- pedestrian
- living_street
- unclassified
- residential
- raceway
man_made:
- pier
- groyne
type: linestring
transport_areas:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- apron
railway:
- station
- platform
type: polygon
transport_points:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
- key: ref
name: ref
type: string
mapping:
aeroway:
- aerodrome
- terminal
- helipad
- gate
highway:
- motorway_junction
- turning_circle
- bus_stop
railway:
- station
- halt
- tram_stop
- crossing
- level_crossing
- subway_entrance
type: point
waterways:
fields:
- key: null
name: osm_id
type: id
- key: null
name: geometry
type: geometry
- key: name
name: name
type: string
- key: null
name: type
type: mapping_value
mapping:
barrier:
- ditch
waterway:
- stream
- river
- canal
- drain
- ditch
type: linestring