From 77b999274de590f799e909db302d0807c6666383 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 13 May 2013 13:58:44 +0200 Subject: [PATCH] add first mapping from json config --- config/config.go | 250 +++++++++++++++++++++++ goposm.go | 23 ++- mapping.json | 518 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 785 insertions(+), 6 deletions(-) create mode 100644 config/config.go create mode 100644 mapping.json diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..e7c875c --- /dev/null +++ b/config/config.go @@ -0,0 +1,250 @@ +package config + +import ( + "encoding/json" + "flag" + "log" + "os" +) + +type Field struct { + Key string `json:"key"` + Type string `json:"type"` +} + +type Table struct { + Type string `json:"type"` + Mapping map[string][]string `json:"mapping"` + Fields map[string]*Field `json:"fields"` +} + +type Tables map[string]Table + +type Mapping struct { + Tables Tables `json:"tables"` +} + +func (t *Table) FillFieldKeys() { + for key, field := range t.Fields { + if field.Key == "" { + field.Key = key + } + } +} + +func (t *Table) Mappings() map[string][]string { + return t.Mapping +} + +func (t *Table) ExtraTags() map[string]bool { + tags := make(map[string]bool) + for _, field := range t.Fields { + tags[field.Key] = true + } + return tags +} + +func (m *Mapping) FillFieldKeys() { + for _, t := range m.Tables { + t.FillFieldKeys() + } +} + +func (m *Mapping) mappings(tableType string, mappings map[string]map[string][]string) { + for name, t := range m.Tables { + if t.Type != tableType { + continue + } + for key, vals := range t.Mappings() { + for _, v := range vals { + vals, ok := mappings[key] + if ok { + vals[v] = append(vals[v], name) + } else { + mappings[key] = make(map[string][]string) + mappings[key][v] = append(mappings[key][v], name) + } + } + } + } +} + +func (m *Mapping) extraTags(tableType string, tags map[string]bool) { + for _, t := range m.Tables { + if t.Type != tableType { + continue + } + for key, _ := range t.ExtraTags() { + tags[key] = true + } + } +} + +func (m *Mapping) NodeTagFilter() *TagFilter { + mappings := make(map[string]map[string][]string) + m.mappings("point", mappings) + tags := make(map[string]bool) + m.extraTags("point", tags) + return &TagFilter{mappings, tags} +} + +func (m *Mapping) WayTagFilter() *TagFilter { + mappings := make(map[string]map[string][]string) + m.mappings("linestring", mappings) + m.mappings("polygon", mappings) + tags := make(map[string]bool) + m.extraTags("linestring", tags) + m.extraTags("polygon", tags) + return &TagFilter{mappings, tags} +} + +func (m *Mapping) RelationTagFilter() *TagFilter { + mappings := make(map[string]map[string][]string) + m.mappings("linestring", mappings) + m.mappings("polygon", mappings) + tags := make(map[string]bool) + m.extraTags("linestring", tags) + m.extraTags("polygon", tags) + return &TagFilter{mappings, tags} +} + +type TagFilter struct { + mappings map[string]map[string][]string + extraTags map[string]bool +} + +type RelationTagFilter struct { + TagFilter +} + +func (f *TagFilter) Filter(tags map[string]string) bool { + foundMapping := false + for k, v := range tags { + values, ok := f.mappings[k] + if ok { + if _, ok := values["__any__"]; ok { + foundMapping = true + continue + } else if _, ok := values[v]; ok { + foundMapping = true + continue + } else { + delete(tags, k) + } + } else if _, ok := f.extraTags[k]; !ok { + delete(tags, k) + } + } + if foundMapping { + return true + } else { + return false + } +} + +func (f *RelationTagFilter) Filter(tags map[string]string) bool { + if t, ok := tags["type"]; ok { + if t != "multipolygon" || t != "boundary" || t != "land_area" { + return false + } + } else { + return false + } + return f.TagFilter.Filter(tags) +} + +func (f *TagFilter) Tables(tags map[string]string) []string { + tables := make(map[string]bool) + + for k, v := range tags { + values, ok := f.mappings[k] + if ok { + if tbls, ok := values["__any__"]; ok { + for _, t := range tbls { + tables[t] = true + } + continue + } else if tbls, ok := values[v]; ok { + for _, t := range tbls { + tables[t] = true + } + continue + } + } + } + var tableNames []string + for name, _ := range tables { + tableNames = append(tableNames, name) + } + return tableNames +} + +func NewMapping(filename string) (*Mapping, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(f) + + mapping := Mapping{} + err = decoder.Decode(&mapping) + if err != nil { + return nil, err + } + + mapping.FillFieldKeys() + return &mapping, nil +} + +func main() { + // data := ` + // { + // "tables": { + // "roads": { + // "mapping": { + // "highway": [ + // "motorway", + // "motorway_link", + // "trunk", + // "trunk_link" + // ] + // }, + // "fields": { + // "tunnel": {"type": "bool", "key": "tunnel"}, + // "bridge": {"type": "bool"}, + // "oneway": {"type": "direction"}, + // "ref": {"type": "string"}, + // "z_order": {"type": "wayzorder", "key": "NONE"} + // } + // } + // } + // } + // ` + + // t := Table{map[string][]string{"highway": {"motorway", "trunk"}}} + // b, err := json.Marshal(t) + // if err != nil { + // log.Fatal(err) + // } + // log.Println(string(b)) + + flag.Parse() + + mapping, err := NewMapping(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + // log.Println(mapping.Mappings("point")) + // log.Println(mapping.ExtraTags("point")) + log.Println(mapping.NodeTagFilter()) + log.Println(mapping.WayTagFilter()) + log.Println(mapping.RelationTagFilter()) + + // log.Println(mapping) + + // b, err := json.MarshalIndent(mapping, "", " ") + // if err != nil { + // log.Fatal(err) + // } + // log.Println(string(b)) +} diff --git a/goposm.go b/goposm.go index 0c6cd2c..358b9e3 100644 --- a/goposm.go +++ b/goposm.go @@ -3,11 +3,11 @@ package main import ( "flag" "goposm/cache" + "goposm/config" "goposm/db" "goposm/element" "goposm/geom" "goposm/geom/geos" - "goposm/mapping" "goposm/parser" "goposm/proj" "goposm/stats" @@ -47,7 +47,7 @@ type ErrorLevel interface { Level() int } -func parse(cache *cache.OSMCache, progress *stats.Statistics, filename string) { +func parse(cache *cache.OSMCache, progress *stats.Statistics, mapping *config.Mapping, filename string) { nodes := make(chan []element.Node) coords := make(chan []element.Node) ways := make(chan []element.Way) @@ -78,12 +78,13 @@ func parse(cache *cache.OSMCache, progress *stats.Statistics, filename string) { for i := 0; i < runtime.NumCPU(); i++ { waitCounter.Add(1) go func() { + m := mapping.WayTagFilter() for ws := range ways { if skipWays { continue } for _, w := range ws { - mapping.WayTags.Filter(w.Tags) + m.Filter(w.Tags) } cache.Ways.PutWays(ws) progress.AddWays(len(ws)) @@ -94,9 +95,10 @@ func parse(cache *cache.OSMCache, progress *stats.Statistics, filename string) { for i := 0; i < runtime.NumCPU(); i++ { waitCounter.Add(1) go func() { + m := mapping.RelationTagFilter() for rels := range relations { for _, r := range rels { - mapping.RelationTags.Filter(r.Tags) + m.Filter(r.Tags) } cache.Relations.PutRelations(rels) progress.AddRelations(len(rels)) @@ -120,12 +122,13 @@ func parse(cache *cache.OSMCache, progress *stats.Statistics, filename string) { for i := 0; i < 2; i++ { waitCounter.Add(1) go func() { + m := mapping.NodeTagFilter() for nds := range nodes { if skipNodes { continue } for _, nd := range nds { - ok := mapping.PointTags.Filter(nd.Tags) + ok := m.Filter(nd.Tags) if !ok { nd.Tags = nil } @@ -155,6 +158,7 @@ var ( write = flag.Bool("write", false, "write") connection = flag.String("connection", "", "connection parameters") diff = flag.Bool("diff", false, "enable diff support") + mappingFile = flag.String("mapping", "", "mapping file") ) func main() { @@ -210,9 +214,14 @@ func main() { progress := stats.StatsReporter() + mapping, err := config.NewMapping(*mappingFile) + if err != nil { + log.Fatal(err) + } + if *read != "" { osmCache.Coords.SetLinearImport(true) - parse(osmCache, progress, *read) + parse(osmCache, progress, mapping, *read) osmCache.Coords.SetLinearImport(false) progress.Reset() } @@ -280,6 +289,7 @@ func main() { for i := 0; i < runtime.NumCPU(); i++ { waitFill.Add(1) go func() { + m := mapping.WayTagFilter() var err error geos := geos.NewGEOS() defer geos.Finish() @@ -302,6 +312,7 @@ func main() { log.Println(err) continue } + // log.Println(w.Id, w.Tags, m.Tables(w.Tags)) batch = append(batch, *w) if len(batch) >= int(dbImportBatchSize) { diff --git a/mapping.json b/mapping.json new file mode 100644 index 0000000..eae8f1b --- /dev/null +++ b/mapping.json @@ -0,0 +1,518 @@ +{ + "tables": { + "landusages": { + "fields": { + "z_order": { + "type": "zorder", + "order": { + "industrial": 2, + "meadow": 28, + "cinema": 11, + "pitch": 22, + "allotments": 20, + "library": 15, + "village_green": 26, + "college": 16, + "parking": 13, + "recreation_ground": 25, + "nature_reserve": 12, + "quarry": 6, + "residential": 5, + "hospital": 8, + "cemetery": 33, + "wood": 29, + "forest": 34, + "fuel": 14, + "playground": 36, + "scrub": 7, + "garden": 24, + "theatre": 10, + "farm": 31, + "park": 35, + "golf_course": 19, + "commercial": 3, + "sports_centre": 23, + "farmland": 30, + "farmyard": 32, + "place_of_worship": 9, + "school": 17, + "land": 0, + "footway": 37, + "pedestrian": 38, + "university": 18, + "grass": 27, + "common": 21, + "railway": 1, + "retail": 4 + } + }, + "area": { + "type": "pseudoarea" + }, + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "polygon", + "mapping": { + "amenity": [ + "university", + "school", + "college", + "library", + "fuel", + "parking", + "cinema", + "theatre", + "place_of_worship", + "hospital" + ], + "landuse": [ + "park", + "forest", + "residential", + "retail", + "commercial", + "industrial", + "railway", + "cemetery", + "grass", + "farmyard", + "farm", + "farmland", + "wood", + "meadow", + "village_green", + "recreation_ground", + "allotments", + "quarry" + ], + "natural": [ + "wood", + "land", + "scrub" + ], + "highway": [ + "pedestrian", + "footway" + ], + "leisure": [ + "park", + "garden", + "playground", + "golf_course", + "sports_centre", + "pitch", + "stadium", + "common", + "nature_reserve" + ] + } + }, + "buildings": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "polygon", + "mapping": { + "building": [ + "__any__" + ] + } + }, + "places": { + "fields": { + "z_order": { + "type": "zorder", + "order": { + "town": 4, + "city": 5, + "locality": 0, + "country": 9, + "region": 7, + "hamlet": 2, + "county": 6, + "suburb": 1, + "state": 8, + "village": 3 + } + }, + "population": { + "type": "integer" + }, + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "point", + "mapping": { + "place": [ + "country", + "state", + "region", + "county", + "city", + "town", + "village", + "hamlet", + "suburb", + "locality" + ] + } + }, + "aeroways": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "linestring", + "mapping": { + "aeroway": [ + "runway", + "taxiway" + ] + } + }, + "admin": { + "fields": { + "admin_level": { + "type": "integer" + }, + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "polygon", + "mapping": { + "boundary": [ + "administrative" + ] + } + }, + "mainroads": { + "fields": { + "z_order": { + "type": "wayzorder" + }, + "bridge": { + "type": "bool" + }, + "name": { + "type": "string" + }, + "tunnel": { + "type": "bool" + }, + "ref": { + "type": "string" + }, + "osm_id": { + "type": "id" + }, + "oneway": { + "type": "direction" + }, + "type": { + "type": "mapping_value" + } + }, + "type": "linestring", + "mapping": { + "highway": [ + "primary", + "primary_link", + "secondary", + "secondary_link", + "tertiary" + ] + } + }, + "transport_areas": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "polygon", + "mapping": { + "railway": [ + "station" + ], + "aeroway": [ + "aerodrome", + "terminal", + "helipad", + "apron" + ] + } + }, + "waterways": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "linestring", + "mapping": { + "waterway": [ + "stream", + "river", + "canal", + "drain" + ] + } + }, + "minorroads": { + "fields": { + "z_order": { + "type": "wayzorder" + }, + "bridge": { + "type": "bool" + }, + "name": { + "type": "string" + }, + "tunnel": { + "type": "bool" + }, + "ref": { + "type": "string" + }, + "osm_id": { + "type": "id" + }, + "oneway": { + "type": "direction" + }, + "type": { + "type": "mapping_value" + } + }, + "type": "linestring", + "mapping": { + "highway": [ + "road", + "path", + "track", + "service", + "footway", + "bridleway", + "cycleway", + "steps", + "pedestrian", + "living_street", + "unclassified", + "residential" + ] + } + }, + "railways": { + "fields": { + "z_order": { + "type": "wayzorder" + }, + "bridge": { + "type": "bool" + }, + "name": { + "type": "string" + }, + "tunnel": { + "type": "bool" + }, + "osm_id": { + "type": "id" + }, + "type": { + "type": "mapping_value" + } + }, + "type": "linestring", + "mapping": { + "railway": [ + "rail", + "tram", + "light_rail", + "subway", + "narrow_gauge", + "preserved", + "funicular", + "monorail" + ] + } + }, + "motorways": { + "fields": { + "z_order": { + "type": "wayzorder" + }, + "bridge": { + "type": "bool" + }, + "name": { + "type": "string" + }, + "tunnel": { + "type": "bool" + }, + "ref": { + "type": "string" + }, + "osm_id": { + "type": "id" + }, + "oneway": { + "type": "direction" + }, + "type": { + "type": "mapping_value" + } + }, + "type": "linestring", + "mapping": { + "highway": [ + "motorway", + "motorway_link", + "trunk", + "trunk_link" + ] + } + }, + "transport_points": { + "fields": { + "ref": { + "type": "string" + }, + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "point", + "mapping": { + "railway": [ + "station", + "halt", + "tram_stop", + "crossing", + "level_crossing", + "subway_entrance" + ], + "aeroway": [ + "aerodome", + "terminal", + "helipad", + "gate" + ], + "highway": [ + "motorway_junction", + "turning_circle", + "bus_stop" + ] + } + }, + "amenities": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "point", + "mapping": { + "amenity": [ + "university", + "school", + "library", + "fuel", + "hospital", + "fire_station", + "police", + "townhall" + ] + } + }, + "waterareas": { + "fields": { + "type": { + "type": "mapping_value" + }, + "name": { + "type": "string" + }, + "osm_id": { + "type": "id" + } + }, + "type": "polygon", + "mapping": { + "waterway": [ + "riverbank" + ], + "landuse": [ + "basin", + "reservoir" + ], + "natural": [ + "water" + ] + } + } + } +}