diff --git a/example-mapping.yml b/example-mapping.yml new file mode 100644 index 0000000..d199309 --- /dev/null +++ b/example-mapping.yml @@ -0,0 +1,616 @@ +generalized_tables: + landusages_gen0: + source: landusages_gen1 + sql_filter: ST_Area(geometry)>500000.000000 + tolerance: 200.0 + landusages_gen1: + source: landusages + sql_filter: ST_Area(geometry)>50000.000000 + tolerance: 50.0 + roads_gen0: + source: roads_gen1 + sql_filter: null + tolerance: 200.0 + roads_gen1: + source: roads + sql_filter: type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', + 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') + OR class IN('railway') + tolerance: 50.0 + waterareas_gen0: + source: waterareas_gen1 + sql_filter: ST_Area(geometry)>500000.000000 + tolerance: 200.0 + waterareas_gen1: + source: waterareas + sql_filter: ST_Area(geometry)>50000.000000 + tolerance: 50.0 + waterways_gen0: + source: waterways_gen1 + sql_filter: null + tolerance: 200 + waterways_gen1: + source: waterways + sql_filter: null + tolerance: 50.0 +tables: + admin: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: admin_level + name: admin_level + type: integer + mapping: + boundary: + - administrative + type: polygon + aeroways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + aeroway: + - runway + - taxiway + type: linestring + amenities: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + amenity: + - university + - school + - library + - fuel + - hospital + - fire_station + - police + - townhall + type: point + barrierpoints: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - block + - bollard + - cattle_grid + - chain + - cycle_barrier + - entrance + - horse_stile + - gate + - spikes + - lift_gate + - kissing_gate + - fence + - 'yes' + - wire_fence + - toll_booth + - stile + type: point + barrierways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - city_wall + - fence + - hedge + - retaining_wall + - wall + - bollard + - gate + - spikes + - lift_gate + - kissing_gate + - embankment + - 'yes' + - wire_fence + type: linestring + buildings: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + building: + - __any__ + type: polygon + housenumbers: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: addr:street + name: addr:street + type: string + - key: addr:postcode + name: addr:postcode + type: string + - key: addr:city + name: addr:city + type: string + mapping: + addr:housenumber: + - __any__ + type: point + housenumbers_interpolated: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: addr:street + name: addr:street + type: string + - key: addr:postcode + name: addr:postcode + type: string + - key: addr:city + name: addr:city + type: string + - key: addr:inclusion + name: addr:inclusion + type: string + mapping: + addr:interpolation: + - __any__ + type: linestring + landusages: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: null + name: area + type: pseudoarea + - args: + ranks: + - pedestrian + - footway + - playground + - park + - forest + - cemetery + - farmyard + - farm + - farmland + - wood + - meadow + - grass + - wetland + - village_green + - recreation_ground + - garden + - sports_centre + - pitch + - common + - allotments + - golf_course + - university + - school + - college + - library + - baracks + - fuel + - parking + - nature_reserve + - cinema + - theatre + - place_of_worship + - hospital + - scrub + - orchard + - vineyard + - zoo + - quarry + - residential + - retail + - commercial + - industrial + - railway + - heath + - island + - land + key: z_order + name: z_order + type: zorder + mapping: + aeroway: + - runway + - taxiway + amenity: + - university + - school + - college + - library + - fuel + - parking + - cinema + - theatre + - place_of_worship + - hospital + barrier: + - hedge + highway: + - pedestrian + - footway + landuse: + - park + - forest + - residential + - retail + - commercial + - industrial + - railway + - cemetery + - grass + - farmyard + - farm + - farmland + - orchard + - vineyard + - wood + - meadow + - village_green + - recreation_ground + - allotments + - quarry + leisure: + - park + - garden + - playground + - golf_course + - sports_centre + - pitch + - stadium + - common + - nature_reserve + man_made: + - pier + military: + - barracks + natural: + - wood + - land + - scrub + - wetland + - heath + place: + - island + tourism: + - zoo + type: polygon + places: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - args: + ranks: + - country + - state + - region + - county + - city + - town + - village + - hamlet + - suburb + - locality + key: z_order + name: z_order + type: zorder + - key: population + name: population + type: integer + mapping: + place: + - country + - state + - region + - county + - city + - town + - village + - hamlet + - suburb + - locality + type: point + roads: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: null + name: type + type: mapping_value + - key: name + name: name + type: string + - key: tunnel + name: tunnel + type: boolint + - key: bridge + name: bridge + type: boolint + - key: oneway + name: oneway + type: direction + - key: ref + name: ref + type: string + - key: layer + name: z_order + type: wayzorder + - key: access + name: access + type: string + - key: service + name: service + type: string + - key: null + name: class + type: mapping_key + filters: + exclude_tags: + - - area + - 'yes' + mappings: + railway: + mapping: + railway: + - rail + - tram + - light_rail + - subway + - narrow_gauge + - preserved + - funicular + - monorail + - disused + roads: + mapping: + highway: + - motorway + - motorway_link + - trunk + - trunk_link + - primary + - primary_link + - secondary + - secondary_link + - tertiary + - tertiary_link + - road + - path + - track + - service + - footway + - bridleway + - cycleway + - steps + - pedestrian + - living_street + - unclassified + - residential + - raceway + man_made: + - pier + - groyne + type: linestring + transport_areas: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + aeroway: + - aerodrome + - terminal + - helipad + - apron + railway: + - station + - platform + type: polygon + transport_points: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: ref + name: ref + type: string + mapping: + aeroway: + - aerodrome + - terminal + - helipad + - gate + highway: + - motorway_junction + - turning_circle + - bus_stop + railway: + - station + - halt + - tram_stop + - crossing + - level_crossing + - subway_entrance + type: point + waterareas: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: null + name: area + type: pseudoarea + mapping: + amenity: + - swimming_pool + landuse: + - basin + - reservoir + leisure: + - swimming_pool + natural: + - water + waterway: + - riverbank + type: polygon + waterways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - ditch + waterway: + - stream + - river + - canal + - drain + - ditch + type: linestring diff --git a/mapping/config.go b/mapping/config.go index 0900482..873c607 100644 --- a/mapping/config.go +++ b/mapping/config.go @@ -1,41 +1,42 @@ package mapping import ( - "encoding/json" "errors" - "os" + "io/ioutil" "github.com/omniscale/imposm3/element" + + "gopkg.in/yaml.v2" ) type Field struct { - Name string `json:"name"` - Key Key `json:"key"` - Keys []Key `json:"keys"` - Type string `json:"type"` - Args map[string]interface{} `json:"args"` + Name string `yaml:"name"` + Key Key `yaml:"key"` + Keys []Key `yaml:"keys"` + Type string `yaml:"type"` + Args map[string]interface{} `yaml:"args"` } type Table struct { Name string - Type TableType `json:"type"` - Mapping map[Key][]Value `json:"mapping"` - Mappings map[string]SubMapping `json:"mappings"` - TypeMappings TypeMappings `json:"type_mappings"` - Fields []*Field `json:"columns"` // TODO rename Fields internaly to Columns - OldFields []*Field `json:"fields"` - Filters *Filters `json:"filters"` + Type TableType `yaml:"type"` + Mapping map[Key][]Value `yaml:"mapping"` + Mappings map[string]SubMapping `yaml:"mappings"` + TypeMappings TypeMappings `yaml:"type_mappings"` + Fields []*Field `yaml:"columns"` // TODO rename Fields internaly to Columns + OldFields []*Field `yaml:"fields"` + Filters *Filters `yaml:"filters"` } type GeneralizedTable struct { Name string - SourceTableName string `json:"source"` - Tolerance float64 `json:"tolerance"` - SqlFilter string `json:"sql_filter"` + SourceTableName string `yaml:"source"` + Tolerance float64 `yaml:"tolerance"` + SqlFilter string `yaml:"sql_filter"` } type Filters struct { - ExcludeTags *[][2]string `json:"exclude_tags"` + ExcludeTags *[][]string `yaml:"exclude_tags"` } type Tables map[string]*Table @@ -43,17 +44,17 @@ type Tables map[string]*Table type GeneralizedTables map[string]*GeneralizedTable type Mapping struct { - Tables Tables `json:"tables"` - GeneralizedTables GeneralizedTables `json:"generalized_tables"` - Tags Tags `json:"tags"` + Tables Tables `yaml:"tables"` + GeneralizedTables GeneralizedTables `yaml:"generalized_tables"` + Tags Tags `yaml:"tags"` // SingleIdSpace mangles the overlapping node/way/relation IDs // to be unique (nodes positive, ways negative, relations negative -1e17) - SingleIdSpace bool `json:"use_single_id_space"` + SingleIdSpace bool `yaml:"use_single_id_space"` } type Tags struct { - LoadAll bool `json:"load_all"` - Exclude []Key `json:"exclude"` + LoadAll bool `yaml:"load_all"` + Exclude []Key `yaml:"exclude"` } type SubMapping struct { @@ -61,9 +62,9 @@ type SubMapping struct { } type TypeMappings struct { - Points map[Key][]Value `json:"points"` - LineStrings map[Key][]Value `json:"linestrings"` - Polygons map[Key][]Value `json:"polygons"` + Points map[Key][]Value `yaml:"points"` + LineStrings map[Key][]Value `yaml:"linestrings"` + Polygons map[Key][]Value `yaml:"polygons"` } type ElementFilter func(tags *element.Tags) bool @@ -103,15 +104,13 @@ const ( ) func NewMapping(filename string) (*Mapping, error) { - f, err := os.Open(filename) + f, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - defer f.Close() - decoder := json.NewDecoder(f) mapping := Mapping{} - err = decoder.Decode(&mapping) + err = yaml.Unmarshal(f, &mapping) if err != nil { return nil, err } diff --git a/mapping/filter_test.go b/mapping/filter_test.go index 1568a95..62b19d6 100644 --- a/mapping/filter_test.go +++ b/mapping/filter_test.go @@ -11,7 +11,7 @@ var mapping *Mapping func init() { var err error - mapping, err = NewMapping("./test_mapping.json") + mapping, err = NewMapping("./test_mapping.yml") if err != nil { panic(err) } diff --git a/mapping/matcher_test.go b/mapping/matcher_test.go index b7d101d..57d8bd5 100644 --- a/mapping/matcher_test.go +++ b/mapping/matcher_test.go @@ -7,7 +7,7 @@ import ( ) func BenchmarkTagMatch(b *testing.B) { - m, err := NewMapping("matcher_test_mapping.json") + m, err := NewMapping("matcher_test_mapping.yml") if err != nil { b.Fatal(err) } @@ -38,7 +38,7 @@ func makeMemberRole(id int64, tags element.Tags, role string) element.Member { } func TestSelectRelationPolygonsSimple(t *testing.T) { - mapping, err := NewMapping("test_mapping.json") + mapping, err := NewMapping("test_mapping.yml") if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestSelectRelationPolygonsSimple(t *testing.T) { } func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) { - mapping, err := NewMapping("test_mapping.json") + mapping, err := NewMapping("test_mapping.yml") if err != nil { t.Fatal(err) } @@ -86,7 +86,7 @@ func TestSelectRelationPolygonsUnrelatedTags(t *testing.T) { } func TestSelectRelationPolygonsMultiple(t *testing.T) { - mapping, err := NewMapping("test_mapping.json") + mapping, err := NewMapping("test_mapping.yml") if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestSelectRelationPolygonsMultiple(t *testing.T) { } func TestSelectRelationPolygonsMultipleTags(t *testing.T) { - mapping, err := NewMapping("test_mapping.json") + mapping, err := NewMapping("test_mapping.yml") if err != nil { t.Fatal(err) } @@ -134,7 +134,7 @@ func TestSelectRelationPolygonsMultipleTags(t *testing.T) { } func TestSelectRelationPolygonsMultipleTagsOnWay(t *testing.T) { - mapping, err := NewMapping("test_mapping.json") + mapping, err := NewMapping("test_mapping.yml") if err != nil { t.Fatal(err) } diff --git a/mapping/test_mapping.yml b/mapping/test_mapping.yml new file mode 100644 index 0000000..a3b1d79 --- /dev/null +++ b/mapping/test_mapping.yml @@ -0,0 +1,607 @@ +generalized_tables: + landusages_gen0: + source: landusages_gen1 + sql_filter: ST_Area(geometry)>500000.000000 + tolerance: 200.0 + landusages_gen1: + source: landusages + sql_filter: ST_Area(geometry)>50000.000000 + tolerance: 50.0 + roads_gen0: + source: roads_gen1 + sql_filter: null + tolerance: 200.0 + roads_gen1: + source: roads + sql_filter: type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', + 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') + OR class IN('railway') + tolerance: 50.0 + waterareas_gen0: + source: waterareas_gen1 + sql_filter: ST_Area(geometry)>500000.000000 + tolerance: 200.0 + waterareas_gen1: + source: waterareas + sql_filter: ST_Area(geometry)>50000.000000 + tolerance: 50.0 + waterways_gen0: + source: waterways_gen1 + sql_filter: null + tolerance: 200 + waterways_gen1: + source: waterways + sql_filter: null + tolerance: 50.0 +tables: + admin: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: admin_level + name: admin_level + type: integer + mapping: + boundary: + - administrative + type: polygon + aeroways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + aeroway: + - runway + - taxiway + type: linestring + amenities: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + amenity: + - university + - school + - library + - fuel + - hospital + - fire_station + - police + - townhall + type: point + amenity_areas: + _comment: for testing duplicate inserts with __any__ and exact match + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + building: + - shop + type: polygon + barrierpoints: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - block + - bollard + - cattle_grid + - chain + - cycle_barrier + - entrance + - horse_stile + - gate + - spikes + - lift_gate + - kissing_gate + - fence + - 'yes' + - wire_fence + - toll_booth + - stile + type: point + barrierways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - city_wall + - fence + - hedge + - retaining_wall + - wall + - bollard + - gate + - spikes + - lift_gate + - kissing_gate + - embankment + - 'yes' + - wire_fence + type: linestring + buildings: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + building: + - __any__ + type: polygon + housenumbers: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: addr:street + name: addr:street + type: string + - key: addr:postcode + name: addr:postcode + type: string + - key: addr:city + name: addr:city + type: string + mapping: + addr:housenumber: + - __any__ + type: point + housenumbers_interpolated: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: addr:street + name: addr:street + type: string + - key: addr:postcode + name: addr:postcode + type: string + - key: addr:city + name: addr:city + type: string + - key: addr:inclusion + name: addr:inclusion + type: string + mapping: + addr:interpolation: + - __any__ + type: linestring + landusages: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: null + name: area + type: pseudoarea + - args: + ranks: + - pedestrian + - footway + - playground + - park + - forest + - cemetery + - farmyard + - farm + - farmland + - wood + - meadow + - grass + - wetland + - village_green + - recreation_ground + - garden + - sports_centre + - pitch + - common + - allotments + - golf_course + - university + - school + - college + - library + - baracks + - fuel + - parking + - nature_reserve + - cinema + - theatre + - place_of_worship + - hospital + - scrub + - orchard + - vineyard + - zoo + - quarry + - residential + - retail + - commercial + - industrial + - railway + - heath + - island + - land + key: z_order + name: z_order + type: zorder + mapping: + aeroway: + - runway + - taxiway + amenity: + - university + - school + - college + - library + - fuel + - parking + - cinema + - theatre + - place_of_worship + - hospital + barrier: + - hedge + highway: + - pedestrian + - footway + landuse: + - park + - forest + - residential + - retail + - commercial + - industrial + - railway + - cemetery + - grass + - farmyard + - farm + - farmland + - orchard + - vineyard + - wood + - meadow + - village_green + - recreation_ground + - allotments + - quarry + leisure: + - park + - garden + - playground + - golf_course + - sports_centre + - pitch + - stadium + - common + - nature_reserve + man_made: + - pier + military: + - barracks + natural: + - wood + - water + - land + - scrub + - wetland + - heath + place: + - island + tourism: + - zoo + waterway: + - riverbank + type: polygon + places: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - args: + ranks: + - country + - state + - region + - county + - city + - town + - village + - hamlet + - suburb + - locality + key: z_order + name: z_order + type: zorder + - key: population + name: population + type: integer + mapping: + place: + - country + - state + - region + - county + - city + - town + - village + - hamlet + - suburb + - locality + type: point + roads: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: null + name: type + type: mapping_value + - key: name + name: name + type: string + - key: tunnel + name: tunnel + type: boolint + - key: bridge + name: bridge + type: boolint + - key: oneway + name: oneway + type: direction + - key: ref + name: ref + type: string + - key: layer + name: z_order + type: wayzorder + - key: access + name: access + type: string + - key: service + name: service + type: string + - key: null + name: class + type: mapping_key + filters: + exclude_tags: + - ["area", "yes"] + mappings: + railway: + mapping: + railway: + - rail + - tram + - light_rail + - subway + - narrow_gauge + - preserved + - funicular + - monorail + - disused + roads: + mapping: + highway: + - motorway + - motorway_link + - trunk + - trunk_link + - primary + - primary_link + - secondary + - secondary_link + - tertiary + - tertiary_link + - road + - path + - track + - service + - footway + - bridleway + - cycleway + - steps + - pedestrian + - living_street + - unclassified + - residential + - raceway + man_made: + - pier + - groyne + type: linestring + transport_areas: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + aeroway: + - aerodrome + - terminal + - helipad + - apron + railway: + - station + - platform + type: polygon + transport_points: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + - key: ref + name: ref + type: string + mapping: + aeroway: + - aerodrome + - terminal + - helipad + - gate + highway: + - motorway_junction + - turning_circle + - bus_stop + railway: + - station + - halt + - tram_stop + - crossing + - level_crossing + - subway_entrance + type: point + waterways: + fields: + - key: null + name: osm_id + type: id + - key: null + name: geometry + type: geometry + - key: name + name: name + type: string + - key: null + name: type + type: mapping_value + mapping: + barrier: + - ditch + waterway: + - stream + - river + - canal + - drain + - ditch + type: linestring