vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

separate mapping config from mapping logic 

create new imposm3/mapping/config package that includes
all "dumb" config structs
master
Oliver Tonnhofer 2017-05-11 11:08:06 +02:00
parent a30442d254
commit e211940af3
11 changed files with 236 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
database/database.go
View File

@ -7,6 +7,7 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
type Config struct {
@ -67,17 +68,17 @@ type Optimizer interface {
Optimize() error
}
var databases map[string]func(Config, *mapping.Mapping) (DB, error)
var databases map[string]func(Config, *config.Mapping) (DB, error)
func init() {
databases = make(map[string]func(Config, *mapping.Mapping) (DB, error))
databases = make(map[string]func(Config, *config.Mapping) (DB, error))
}
func Register(name string, f func(Config, *mapping.Mapping) (DB, error)) {
func Register(name string, f func(Config, *config.Mapping) (DB, error)) {
databases[name] = f
}
func Open(conf Config, m *mapping.Mapping) (DB, error) {
func Open(conf Config, m *config.Mapping) (DB, error) {
parts := strings.SplitN(conf.ConnectionParams, ":", 2)
connectionType := parts[0]
@ -108,7 +109,7 @@ func (n *nullDb) InsertRelationMember(element.Relation, element.Member, geom.Geo
return nil
}
func newNullDb(conf Config, m *mapping.Mapping) (DB, error) {
func newNullDb(conf Config, m *config.Mapping) (DB, error) {
return &nullDb{}, nil
}

3
database/postgis/postgis.go
View File

@ -14,6 +14,7 @@ import (
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/logging"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
var log = logging.NewLogger("PostGIS")
@ -599,7 +600,7 @@ func (pg *PostGIS) Close() error {
return pg.Db.Close()
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
func New(conf database.Config, m *config.Mapping) (database.DB, error) {
db := &PostGIS{}
db.Tables = make(map[string]*TableSpec)

12
database/postgis/spec.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/omniscale/imposm3/mapping"
"github.com/omniscale/imposm3/mapping/config"
)
type ColumnSpec struct {
@ -124,12 +125,11 @@ func (spec *TableSpec) DeleteSQL() string {
)
}
func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
func NewTableSpec(pg *PostGIS, t *config.Table) *TableSpec {
var geomType string
switch t.Type {
case mapping.RelationMemberTable:
if mapping.TableType(t.Type) == mapping.RelationMemberTable {
geomType = "geometry"
default:
} else {
geomType = string(t.Type)
}
@ -141,7 +141,7 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
Srid: pg.Config.Srid,
}
for _, column := range t.Columns {
columnType := column.ColumnType()
columnType := mapping.MakeColumnType(column)
if columnType == nil {
continue
}
@ -156,7 +156,7 @@ func NewTableSpec(pg *PostGIS, t *mapping.Table) *TableSpec {
return &spec
}
func NewGeneralizedTableSpec(pg *PostGIS, t *mapping.GeneralizedTable) *GeneralizedTableSpec {
func NewGeneralizedTableSpec(pg *PostGIS, t *config.GeneralizedTable) *GeneralizedTableSpec {
spec := GeneralizedTableSpec{
Name: t.Name,
FullName: pg.Prefix + t.Name,

6
import_/import.go
View File

@ -67,7 +67,7 @@ func Import() {
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err = database.Open(conf, tagmapping)
db, err = database.Open(conf, &tagmapping.Conf)
if err != nil {
log.Fatal(err)
}
@ -181,7 +181,7 @@ func Import() {
relations := osmCache.Relations.Iter()
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
@ -196,7 +196,7 @@ func Import() {
ways := osmCache.Ways.Iter()
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(), tagmapping.LineStringMatcher(),

35
mapping/columns.go
View File

@ -11,6 +11,7 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/logging"
"github.com/omniscale/imposm3/mapping/config"
)
var log = logging.NewLogger("mapping")
@ -47,7 +48,7 @@ func init() {
type MakeValue func(string, *element.OSMElem, *geom.Geometry, Match) interface{}
type MakeMemberValue func(*element.Relation, *element.Member, Match) interface{}
type MakeMakeValue func(string, ColumnType, Column) (MakeValue, error)
type MakeMakeValue func(string, ColumnType, config.Column) (MakeValue, error)
type Key string
type Value string
@ -100,24 +101,6 @@ func (t *TableSpec) MakeMemberRow(rel *element.Relation, member *element.Member,
return row
}
func (t *Table) TableSpec() *TableSpec {
result := TableSpec{}
for _, mappingColumn := range t.Columns {
column := ColumnSpec{}
column.Key = mappingColumn.Key
columnType := mappingColumn.ColumnType()
if columnType != nil {
column.Type = *columnType
} else {
log.Warn("unhandled type: ", mappingColumn.Type)
}
result.columns = append(result.columns, column)
}
return &result
}
type ColumnType struct {
Name string
GoType string
@ -200,7 +183,7 @@ func Geometry(val string, elem *element.OSMElem, geom *geom.Geometry, match Matc
return string(geom.Wkb)
}
func MakePseudoArea(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakePseudoArea(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
log.Print("warn: pseudoarea type is deprecated and will be removed. See area and webmercarea type.")
return Area, nil
}
@ -232,7 +215,7 @@ func WebmercArea(val string, elem *element.OSMElem, geom *geom.Geometry, match M
var hstoreReplacer = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
func MakeHStoreString(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakeHStoreString(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
var includeAll bool
var err error
var include map[string]int
@ -257,7 +240,7 @@ func MakeHStoreString(columnName string, columnType ColumnType, column Column) (
return hstoreString, nil
}
func MakeWayZOrder(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakeWayZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
if _, ok := column.Args["ranks"]; !ok {
return DefaultWayZOrder, nil
}
@ -345,7 +328,7 @@ func DefaultWayZOrder(val string, elem *element.OSMElem, geom *geom.Geometry, ma
return z
}
func MakeZOrder(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakeZOrder(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
log.Print("warn: zorder type is deprecated and will be removed. See enumerate type.")
_rankList, ok := column.Args["ranks"]
if !ok {
@ -392,7 +375,7 @@ func MakeZOrder(columnName string, columnType ColumnType, column Column) (MakeVa
return zOrder, nil
}
func MakeEnumerate(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakeEnumerate(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
values, err := decodeEnumArg(column, "values")
if err != nil {
return nil, err
@ -413,7 +396,7 @@ func MakeEnumerate(columnName string, columnType ColumnType, column Column) (Mak
return enumerate, nil
}
func decodeEnumArg(column Column, key string) (map[string]int, error) {
func decodeEnumArg(column config.Column, key string) (map[string]int, error) {
_valuesList, ok := column.Args[key]
if !ok {
return nil, fmt.Errorf("missing '%v' in args for %s", key, column.Type)
@ -436,7 +419,7 @@ func decodeEnumArg(column Column, key string) (map[string]int, error) {
return values, nil
}
func MakeSuffixReplace(columnName string, columnType ColumnType, column Column) (MakeValue, error) {
func MakeSuffixReplace(columnName string, columnType ColumnType, column config.Column) (MakeValue, error) {
_changes, ok := column.Args["suffixes"]
if !ok {
return nil, errors.New("missing suffixes in args for string_suffixreplace")

15
mapping/columns_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/geom"
"github.com/omniscale/imposm3/geom/geos"
"github.com/omniscale/imposm3/mapping/config"
)
func TestBool(t *testing.T) {
@ -75,7 +76,7 @@ func TestZOrder(t *testing.T) {
zOrder, err := MakeZOrder("z_order",
AvailableColumnTypes["z_order"],
Column{
config.Column{
Name: "z_order",
Key: "",
Type: "z_order",
@ -114,7 +115,7 @@ func TestEnumerate_Match(t *testing.T) {
zOrder, err := MakeEnumerate("enumerate",
AvailableColumnTypes["enumerate"],
Column{
config.Column{
Name: "enumerate",
Key: "",
Type: "enumerate",
@ -149,7 +150,7 @@ func TestEnumerate_Key(t *testing.T) {
zOrder, err := MakeEnumerate("enumerate",
AvailableColumnTypes["enumerate"],
Column{
config.Column{
Name: "enumerate",
Key: "fips",
Type: "enumerate",
@ -183,7 +184,7 @@ func TestEnumerate_Key(t *testing.T) {
func TestWayZOrder(t *testing.T) {
zOrder, err := MakeWayZOrder("z_order",
AvailableColumnTypes["wayzorder"],
Column{
config.Column{
Name: "zorder",
Type: "wayzorder",
Args: map[string]interface{}{
@ -277,7 +278,7 @@ func TestAreaColumn(t *testing.T) {
}
func TestMakeSuffixReplace(t *testing.T) {
column := Column{
column := config.Column{
Name: "name", Key: "name", Type: "string_suffixreplace",
Args: map[string]interface{}{"suffixes": map[interface{}]interface{}{"Straße": "Str.", "straße": "str."}}}
suffixReplace, err := MakeSuffixReplace("name", ColumnType{}, column)
@ -298,7 +299,7 @@ func TestMakeSuffixReplace(t *testing.T) {
}
func TestHstoreString(t *testing.T) {
column := Column{
column := config.Column{
Name: "tags",
Type: "hstore_tags",
}
@ -307,7 +308,7 @@ func TestHstoreString(t *testing.T) {
t.Fatal(err)
}
column = Column{
column = config.Column{
Name: "tags",
Type: "hstore_tags",
Args: map[string]interface{}{"include": []interface{}{"key1", "key2"}},

225
mapping/config.go
View File

@ -2,133 +2,14 @@ package mapping
import (
"errors"
"fmt"
"io/ioutil"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
"gopkg.in/yaml.v2"
)
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Column struct {
Name string `yaml:"name"`
Key Key `yaml:"key"`
Keys []Key `yaml:"keys"`
Type string `yaml:"type"`
Args map[string]interface{} `yaml:"args"`
FromMember bool `yaml:"from_member"`
}
func (c *Column) ColumnType() *ColumnType {
if columnType, ok := AvailableColumnTypes[c.Type]; ok {
if columnType.MakeFunc != nil {
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
if err != nil {
log.Print(err)
return nil
}
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
}
columnType.FromMember = c.FromMember
return &columnType
}
return nil
}
type Tables map[string]*Table
type Table struct {
Name string
Type TableType `yaml:"type"`
Mapping KeyValues `yaml:"mapping"`
Mappings map[string]SubMapping `yaml:"mappings"`
TypeMappings TypeMappings `yaml:"type_mappings"`
Columns []*Column `yaml:"columns"`
OldFields []*Column `yaml:"fields"`
Filters *Filters `yaml:"filters"`
RelationTypes []string `yaml:"relation_types"`
}
type GeneralizedTables map[string]*GeneralizedTable
type GeneralizedTable struct {
Name string
SourceTableName string `yaml:"source"`
Tolerance float64 `yaml:"tolerance"`
SqlFilter string `yaml:"sql_filter"`
}
type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Areas struct {
AreaTags []Key `yaml:"area_tags"`
LinearTags []Key `yaml:"linear_tags"`
}
type Tags struct {
LoadAll bool `yaml:"load_all"`
Exclude []Key `yaml:"exclude"`
Include []Key `yaml:"include"`
}
type orderedValue struct {
value Value
order int
}
type KeyValues map[Key][]orderedValue
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
if *kv == nil {
*kv = make(map[Key][]orderedValue)
}
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
order := 0
for _, item := range slice {
k, ok := item.Key.(string)
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
values, ok := item.Value.([]interface{})
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
for _, v := range values {
if v, ok := v.(string); ok {
(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{value: Value(v), order: order})
} else {
return fmt.Errorf("mapping value '%s' not a string", v)
}
order += 1
}
}
return nil
}
type SubMapping struct {
Mapping KeyValues
}
type TypeMappings struct {
Points KeyValues `yaml:"points"`
LineStrings KeyValues `yaml:"linestrings"`
Polygons KeyValues `yaml:"polygons"`
}
type ElementFilter func(tags element.Tags, key Key, closed bool) bool
type orderedDestTable struct {
@ -138,16 +19,16 @@ type orderedDestTable struct {
type TagTableMapping map[Key]map[Value][]orderedDestTable
func (tt TagTableMapping) addFromMapping(mapping KeyValues, table DestTable) {
func (tt TagTableMapping) addFromMapping(mapping config.KeyValues, table DestTable) {
for key, vals := range mapping {
for _, v := range vals {
vals, ok := tt[key]
tbl := orderedDestTable{DestTable: table, order: v.order}
vals, ok := tt[Key(key)]
tbl := orderedDestTable{DestTable: table, order: v.Order}
if ok {
vals[v.value] = append(vals[v.value], tbl)
vals[Value(v.Value)] = append(vals[Value(v.Value)], tbl)
} else {
tt[key] = make(map[Value][]orderedDestTable)
tt[key][v.value] = append(tt[key][v.value], tbl)
tt[Key(key)] = make(map[Value][]orderedDestTable)
tt[Key(key)][Value(v.Value)] = append(tt[Key(key)][Value(v.Value)], tbl)
}
}
}
@ -202,6 +83,10 @@ const (
RelationMemberTable TableType = "relation_member"
)
type Mapping struct {
Conf config.Mapping
}
func NewMapping(filename string) (*Mapping, error) {
f, err := ioutil.ReadFile(filename)
if err != nil {
@ -209,7 +94,7 @@ func NewMapping(filename string) (*Mapping, error) {
}
mapping := Mapping{}
err = yaml.Unmarshal(f, &mapping)
err = yaml.Unmarshal(f, &mapping.Conf)
if err != nil {
return nil, err
}
@ -222,7 +107,7 @@ func NewMapping(filename string) (*Mapping, error) {
}
func (m *Mapping) prepare() error {
for name, t := range m.Tables {
for name, t := range m.Conf.Tables {
t.Name = name
if t.OldFields != nil {
// todo deprecate 'fields'
@ -230,15 +115,15 @@ func (m *Mapping) prepare() error {
}
}
for name, t := range m.GeneralizedTables {
for name, t := range m.Conf.GeneralizedTables {
t.Name = name
}
return nil
}
func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
for name, t := range m.Tables {
if t.Type != GeometryTable && t.Type != tableType {
for name, t := range m.Conf.Tables {
if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
continue
}
mappings.addFromMapping(t.Mapping, DestTable{Name: name})
@ -260,26 +145,60 @@ func (m *Mapping) mappings(tableType TableType, mappings TagTableMapping) {
func (m *Mapping) tables(tableType TableType) map[string]*TableSpec {
result := make(map[string]*TableSpec)
for name, t := range m.Tables {
if t.Type == tableType || t.Type == GeometryTable {
result[name] = t.TableSpec()
for name, t := range m.Conf.Tables {
if TableType(t.Type) == tableType || TableType(t.Type) == GeometryTable {
result[name] = makeTableSpec(t)
}
}
return result
}
func makeTableSpec(tbl *config.Table) *TableSpec {
result := TableSpec{}
for _, mappingColumn := range tbl.Columns {
column := ColumnSpec{}
column.Key = Key(mappingColumn.Key)
columnType := MakeColumnType(mappingColumn)
if columnType != nil {
column.Type = *columnType
} else {
log.Warn("unhandled type: ", mappingColumn.Type)
}
result.columns = append(result.columns, column)
}
return &result
}
func MakeColumnType(c *config.Column) *ColumnType {
if columnType, ok := AvailableColumnTypes[c.Type]; ok {
if columnType.MakeFunc != nil {
makeValue, err := columnType.MakeFunc(c.Name, columnType, *c)
if err != nil {
log.Print(err)
return nil
}
columnType = ColumnType{columnType.Name, columnType.GoType, makeValue, nil, nil, columnType.FromMember}
}
columnType.FromMember = c.FromMember
return &columnType
}
return nil
}
func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
for _, t := range m.Tables {
if t.Type != tableType && t.Type != GeometryTable {
for _, t := range m.Conf.Tables {
if TableType(t.Type) != tableType && TableType(t.Type) != GeometryTable {
continue
}
for _, col := range t.Columns {
if col.Key != "" {
tags[col.Key] = true
tags[Key(col.Key)] = true
}
for _, k := range col.Keys {
tags[k] = true
tags[Key(k)] = true
}
}
@ -295,8 +214,8 @@ func (m *Mapping) extraTags(tableType TableType, tags map[Key]bool) {
}
}
}
for _, k := range m.Tags.Include {
tags[k] = true
for _, k := range m.Conf.Tags.Include {
tags[Key(k)] = true
}
// always include area tag for closed-way handling
@ -308,24 +227,24 @@ type tableElementFilters map[string][]ElementFilter
func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilters) {
var areaTags map[Key]struct{}
var linearTags map[Key]struct{}
if m.Areas.AreaTags != nil {
if m.Conf.Areas.AreaTags != nil {
areaTags = make(map[Key]struct{})
for _, tag := range m.Areas.AreaTags {
areaTags[tag] = struct{}{}
for _, tag := range m.Conf.Areas.AreaTags {
areaTags[Key(tag)] = struct{}{}
}
}
if m.Areas.LinearTags != nil {
if m.Conf.Areas.LinearTags != nil {
linearTags = make(map[Key]struct{})
for _, tag := range m.Areas.LinearTags {
linearTags[tag] = struct{}{}
for _, tag := range m.Conf.Areas.LinearTags {
linearTags[Key(tag)] = struct{}{}
}
}
for name, t := range m.Tables {
if t.Type != GeometryTable && t.Type != tableType {
for name, t := range m.Conf.Tables {
if TableType(t.Type) != GeometryTable && TableType(t.Type) != tableType {
continue
}
if t.Type == LineStringTable && areaTags != nil {
if TableType(t.Type) == LineStringTable && areaTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed {
if tags["area"] == "yes" {
@ -341,7 +260,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilte
}
filters[name] = append(filters[name], f)
}
if t.Type == PolygonTable && linearTags != nil {
if TableType(t.Type) == PolygonTable && linearTags != nil {
f := func(tags element.Tags, key Key, closed bool) bool {
if closed && tags["area"] == "no" {
return false
@ -359,7 +278,7 @@ func (m *Mapping) addTypedFilters(tableType TableType, filters tableElementFilte
}
func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFilters) {
for name, t := range m.Tables {
for name, t := range m.Conf.Tables {
if t.RelationTypes != nil {
relTypes := t.RelationTypes // copy loop var for closure
f := func(tags element.Tags, key Key, closed bool) bool {
@ -374,7 +293,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFi
}
filters[name] = append(filters[name], f)
} else {
if t.Type == PolygonTable {
if TableType(t.Type) == PolygonTable {
// standard mulipolygon handling (boundary and land_area are for backwards compatibility)
f := func(tags element.Tags, key Key, closed bool) bool {
if v, ok := tags["type"]; ok {
@ -391,7 +310,7 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFi
}
func (m *Mapping) addFilters(filters tableElementFilters) {
for name, t := range m.Tables {
for name, t := range m.Conf.Tables {
if t.Filters == nil {
continue
}

113
mapping/config/config.go Normal file
View File

@ -0,0 +1,113 @@
package config
import (
"fmt"
"gopkg.in/yaml.v2"
)
type Mapping struct {
Tables Tables `yaml:"tables"`
GeneralizedTables GeneralizedTables `yaml:"generalized_tables"`
Tags Tags `yaml:"tags"`
Areas Areas `yaml:"areas"`
// SingleIdSpace mangles the overlapping node/way/relation IDs
// to be unique (nodes positive, ways negative, relations negative -1e17)
SingleIdSpace bool `yaml:"use_single_id_space"`
}
type Column struct {
Name string `yaml:"name"`
Key Key `yaml:"key"`
Keys []Key `yaml:"keys"`
Type string `yaml:"type"`
Args map[string]interface{} `yaml:"args"`
FromMember bool `yaml:"from_member"`
}
type Tables map[string]*Table
type Table struct {
Name string
Type string `yaml:"type"`
Mapping KeyValues `yaml:"mapping"`
Mappings map[string]SubMapping `yaml:"mappings"`
TypeMappings TypeMappings `yaml:"type_mappings"`
Columns []*Column `yaml:"columns"`
OldFields []*Column `yaml:"fields"`
Filters *Filters `yaml:"filters"`
RelationTypes []string `yaml:"relation_types"`
}
type GeneralizedTables map[string]*GeneralizedTable
type GeneralizedTable struct {
Name string
SourceTableName string `yaml:"source"`
Tolerance float64 `yaml:"tolerance"`
SqlFilter string `yaml:"sql_filter"`
}
type Filters struct {
ExcludeTags *[][]string `yaml:"exclude_tags"`
}
type Areas struct {
AreaTags []Key `yaml:"area_tags"`
LinearTags []Key `yaml:"linear_tags"`
}
type Tags struct {
LoadAll bool `yaml:"load_all"`
Exclude []Key `yaml:"exclude"`
Include []Key `yaml:"include"`
}
type Key string
type Value string
type orderedValue struct {
Value Value
Order int
}
type KeyValues map[Key][]orderedValue
func (kv *KeyValues) UnmarshalYAML(unmarshal func(interface{}) error) error {
if *kv == nil {
*kv = make(map[Key][]orderedValue)
}
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
order := 0
for _, item := range slice {
k, ok := item.Key.(string)
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
values, ok := item.Value.([]interface{})
if !ok {
return fmt.Errorf("mapping key '%s' not a string", k)
}
for _, v := range values {
if v, ok := v.(string); ok {
(*kv)[Key(k)] = append((*kv)[Key(k)], orderedValue{Value: Value(v), Order: order})
} else {
return fmt.Errorf("mapping value '%s' not a string", v)
}
order += 1
}
}
return nil
}
type SubMapping struct {
Mapping KeyValues
}
type TypeMappings struct {
Points KeyValues `yaml:"points"`
LineStrings KeyValues `yaml:"linestrings"`
Polygons KeyValues `yaml:"polygons"`
}

17
mapping/filter.go
View File

@ -5,6 +5,7 @@ import (
"strings"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
)
type TagFilterer interface {
@ -12,8 +13,8 @@ type TagFilterer interface {
}
func (m *Mapping) NodeTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(TagTableMapping)
m.mappings(PointTable, mappings)
@ -24,8 +25,8 @@ func (m *Mapping) NodeTagFilter() TagFilterer {
}
func (m *Mapping) WayTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(TagTableMapping)
m.mappings(LineStringTable, mappings)
@ -38,8 +39,8 @@ func (m *Mapping) WayTagFilter() TagFilterer {
}
func (m *Mapping) RelationTagFilter() TagFilterer {
if m.Tags.LoadAll {
return newExcludeFilter(m.Tags.Exclude)
if m.Conf.Tags.LoadAll {
return newExcludeFilter(m.Conf.Tags.Exclude)
}
mappings := make(TagTableMapping)
// do not filter out type tag for common relations
@ -92,7 +93,7 @@ type excludeFilter struct {
matches []string
}
func newExcludeFilter(tags []Key) *excludeFilter {
func newExcludeFilter(tags []config.Key) *excludeFilter {
f := excludeFilter{
keys: make(map[Key]struct{}),
matches: make([]string, 0),
@ -101,7 +102,7 @@ func newExcludeFilter(tags []Key) *excludeFilter {
if strings.ContainsAny(string(t), "?*[") {
f.matches = append(f.matches, string(t))
} else {
f.keys[t] = struct{}{}
f.keys[Key(t)] = struct{}{}
}
}
return &f

7
mapping/filter_test.go
View File

@ -5,6 +5,7 @@ import (
"testing"
"github.com/omniscale/imposm3/element"
"github.com/omniscale/imposm3/mapping/config"
)
var mapping *Mapping
@ -349,7 +350,7 @@ func TestExcludeFilter(t *testing.T) {
var tags element.Tags
// no matches
f = newExcludeFilter([]Key{})
f = newExcludeFilter([]config.Key{})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}) {
@ -357,7 +358,7 @@ func TestExcludeFilter(t *testing.T) {
}
// match all
f = newExcludeFilter([]Key{"*"})
f = newExcludeFilter([]config.Key{"*"})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{}) {
@ -365,7 +366,7 @@ func TestExcludeFilter(t *testing.T) {
}
// fixed string and wildcard match
f = newExcludeFilter([]Key{"source", "tiger:*"})
f = newExcludeFilter([]config.Key{"source", "tiger:*"})
tags = element.Tags{"source": "1", "tiger:foo": "1", "source:foo": "1"}
f.Filter(&tags)
if !reflect.DeepEqual(tags, element.Tags{"source:foo": "1"}) {

8
update/process.go
View File

@ -119,7 +119,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
ProductionSchema: config.BaseOptions.Schemas.Production,
BackupSchema: config.BaseOptions.Schemas.Backup,
}
db, err := database.Open(dbConf, tagmapping)
db, err := database.Open(dbConf, &tagmapping.Conf)
if err != nil {
return errors.New("database open: " + err.Error())
}
@ -144,7 +144,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
delDb,
osmCache,
diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
tagmapping.PointMatcher(),
tagmapping.LineStringMatcher(),
tagmapping.PolygonMatcher(),
@ -162,7 +162,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
nodes := make(chan *element.Node)
relWriter := writer.NewRelationWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
relations,
db, progress,
tagmapping.PolygonMatcher(),
@ -174,7 +174,7 @@ func Update(oscFile string, geometryLimiter *limit.Limiter, expireor expire.Expi
relWriter.Start()
wayWriter := writer.NewWayWriter(osmCache, diffCache,
tagmapping.SingleIdSpace,
tagmapping.Conf.SingleIdSpace,
ways, db,
progress,
tagmapping.PolygonMatcher(),