From 2d2dad7b7eb7f97c1c6118784c4771d1ffcde8b5 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Mon, 4 Nov 2013 08:51:21 +0100 Subject: [PATCH] limit cache to -limitto --- cache/delta.go | 3 +++ cache/nodes.go | 6 ++++++ cache/osm.go | 2 ++ config/config.go | 33 +++++++++++++++++---------------- import_/import.go | 15 ++++++++++++--- imposm3.go | 2 +- reader/reader.go | 30 ++++++++++++++++++++++++++++-- 7 files changed, 69 insertions(+), 22 deletions(-) diff --git a/cache/delta.go b/cache/delta.go index 800f0af..2cda314 100644 --- a/cache/delta.go +++ b/cache/delta.go @@ -211,6 +211,9 @@ func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error { start = 0 totalNodes := len(nodes) for i, node := range nodes { + if node.Id == SKIP { + continue + } bunchId := self.getBunchId(node.Id) if bunchId != currentBunchId { if self.linearImport && int64(i) > self.bunchSize && int64(i) < int64(totalNodes)-self.bunchSize { diff --git a/cache/nodes.go b/cache/nodes.go index b2540f5..756ca3c 100644 --- a/cache/nodes.go +++ b/cache/nodes.go @@ -21,6 +21,9 @@ func newNodesCache(path string) (*NodesCache, error) { } func (p *NodesCache) PutNode(node *element.Node) error { + if node.Id == SKIP { + return nil + } if node.Tags == nil { return nil } @@ -38,6 +41,9 @@ func (p *NodesCache) PutNodes(nodes []element.Node) (int, error) { var n int for _, node := range nodes { + if node.Id == SKIP { + continue + } if len(node.Tags) == 0 { continue } diff --git a/cache/osm.go b/cache/osm.go index a21c82e..1cfefd0 100644 --- a/cache/osm.go +++ b/cache/osm.go @@ -12,6 +12,8 @@ var ( NotFound = errors.New("not found") ) +const SKIP int64 = -1 + type OSMCache struct { dir string Coords *DeltaCoordsCache diff --git a/config/config.go b/config/config.go index 992c11d..eb43ad7 100644 --- a/config/config.go +++ b/config/config.go @@ -10,12 +10,12 @@ import ( ) type Config struct { - CacheDir string `json:"cachedir"` - Connection string `json:"connection"` - MappingFile string `json:"mapping"` - LimitTo string `json:"limitto"` - LimitToCacheBuffer float64 `json:"limitto_cache_buffer"` - Srid int `json:"srid"` + CacheDir string `json:"cachedir"` + Connection string `json:"connection"` + MappingFile string `json:"mapping"` + LimitTo string `json:"limitto"` + LimitToDiffBuffer float64 `json:"limitto_cache_buffer"` + Srid int `json:"srid"` } const defaultSrid = 3857 @@ -25,14 +25,14 @@ var ImportFlags = flag.NewFlagSet("import", flag.ExitOnError) var DiffFlags = flag.NewFlagSet("diff", flag.ExitOnError) type _BaseOptions struct { - Connection string - CacheDir string - MappingFile string - Srid int - LimitTo string - LimitToCacheBuffer float64 - ConfigFile string - Httpprofile string + Connection string + CacheDir string + MappingFile string + Srid int + LimitTo string + LimitToDiffBuffer float64 + ConfigFile string + Httpprofile string } func (o *_BaseOptions) updateFromConfig() error { @@ -68,8 +68,8 @@ func (o *_BaseOptions) updateFromConfig() error { if o.LimitTo == "" { o.LimitTo = conf.LimitTo } - if o.LimitToCacheBuffer == 0.0 { - o.LimitToCacheBuffer = conf.LimitToCacheBuffer + if o.LimitToDiffBuffer == 0.0 { + o.LimitToDiffBuffer = conf.LimitToDiffBuffer } if o.CacheDir == defaultCacheDir { o.CacheDir = conf.CacheDir @@ -110,6 +110,7 @@ func addBaseFlags(flags *flag.FlagSet) { flags.StringVar(&BaseOptions.MappingFile, "mapping", "", "mapping file") flags.IntVar(&BaseOptions.Srid, "srid", defaultSrid, "srs id") flags.StringVar(&BaseOptions.LimitTo, "limitto", "", "limit to geometries") + flags.Float64Var(&BaseOptions.LimitToDiffBuffer, "limittodiffbuffer", 0.0, "limit to buffer for cache") flags.StringVar(&BaseOptions.ConfigFile, "config", "", "config (json)") flags.StringVar(&BaseOptions.Httpprofile, "httpprofile", "", "bind address for profile server") } diff --git a/import_/import.go b/import_/import.go index 55415de..7b5c584 100644 --- a/import_/import.go +++ b/import_/import.go @@ -36,10 +36,13 @@ func Import() { } var geometryLimiter *limit.Limiter - if config.ImportOptions.Write && config.BaseOptions.LimitTo != "" { + if (config.ImportOptions.Write || config.ImportOptions.Read != "") && config.BaseOptions.LimitTo != "" { var err error step := log.StartStep("Reading limitto geometries") - geometryLimiter, err = limit.NewFromGeoJson(config.BaseOptions.LimitTo) + geometryLimiter, err = limit.NewFromGeoJsonWithBuffered( + config.BaseOptions.LimitTo, + config.BaseOptions.LimitToDiffBuffer, + ) if err != nil { log.Fatal(err) } @@ -99,7 +102,13 @@ func Import() { } osmCache.Coords.SetLinearImport(true) - reader.ReadPbf(osmCache, progress, tagmapping, pbfFile) + readLimiter := geometryLimiter + if config.BaseOptions.LimitToDiffBuffer == 0.0 { + readLimiter = nil + } + reader.ReadPbf(osmCache, progress, tagmapping, + pbfFile, readLimiter) + osmCache.Coords.SetLinearImport(false) elementCounts = progress.Stop() osmCache.Close() diff --git a/imposm3.go b/imposm3.go index 1f9abe3..804d32a 100644 --- a/imposm3.go +++ b/imposm3.go @@ -57,7 +57,7 @@ func main() { step := log.StartStep("Reading limitto geometries") geometryLimiter, err = limit.NewFromGeoJsonWithBuffered( config.BaseOptions.LimitTo, - config.BaseOptions.LimitToCacheBuffer, + config.BaseOptions.LimitToDiffBuffer, ) if err != nil { log.Fatal(err) diff --git a/reader/reader.go b/reader/reader.go index f8bd109..619f01c 100644 --- a/reader/reader.go +++ b/reader/reader.go @@ -1,8 +1,10 @@ package reader import ( - "imposm3/cache" + osmcache "imposm3/cache" "imposm3/element" + "imposm3/geom/geos" + "imposm3/geom/limit" "imposm3/logging" "imposm3/mapping" "imposm3/parser/pbf" @@ -45,12 +47,20 @@ func init() { } -func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapping.Mapping, pbfFile *pbf.Pbf) { +func ReadPbf(cache *osmcache.OSMCache, progress *stats.Statistics, + tagmapping *mapping.Mapping, pbfFile *pbf.Pbf, + limiter *limit.Limiter, +) { nodes := make(chan []element.Node, 4) coords := make(chan []element.Node, 4) ways := make(chan []element.Way, 4) relations := make(chan []element.Relation, 4) + withLimiter := false + if limiter != nil { + withLimiter = true + } + if pbfFile.Header.Time.Unix() != 0 { log.Printf("reading %s with data till %v", pbfFile.Filename, pbfFile.Header.Time.Local()) } @@ -86,6 +96,7 @@ func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapp for i, _ := range ws { m.Filter(&ws[i].Tags) } + // TODO check withLimiter cache.Ways.PutWays(ws) progress.AddWays(len(ws)) } @@ -105,6 +116,7 @@ func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapp numWithTags += 1 } } + // TODO check withLimiter cache.Relations.PutRelations(rels) progress.AddRelations(numWithTags) } @@ -115,10 +127,19 @@ func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapp for i := 0; int64(i) < nCoords; i++ { waitWriter.Add(1) go func() { + g := geos.NewGeos() + defer g.Finish() for nds := range coords { if skipCoords { continue } + if withLimiter { + for i, _ := range nds { + if !limiter.IntersectsBuffer(g, nds[i].Long, nds[i].Lat) { + nds[i].Id = osmcache.SKIP + } + } + } cache.Coords.PutCoords(nds) progress.AddCoords(len(nds)) } @@ -129,6 +150,8 @@ func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapp for i := 0; int64(i) < nNodes; i++ { waitWriter.Add(1) go func() { + g := geos.NewGeos() + defer g.Finish() m := tagmapping.NodeTagFilter() for nds := range nodes { numWithTags := 0 @@ -137,6 +160,9 @@ func ReadPbf(cache *cache.OSMCache, progress *stats.Statistics, tagmapping *mapp if len(nds[i].Tags) > 0 { numWithTags += 1 } + if withLimiter && !limiter.IntersectsBuffer(g, nds[i].Long, nds[i].Lat) { + nds[i].Id = osmcache.SKIP + } } cache.Nodes.PutNodes(nds) progress.AddNodes(numWithTags)