close caches earlier to release memory; optimize read only access for coords
parent
1962abec78
commit
ecd5541c1e
|
@ -92,6 +92,7 @@ type DeltaCoordsCache struct {
|
||||||
linearImport bool
|
linearImport bool
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
bunchSize int64
|
bunchSize int64
|
||||||
|
readOnly bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
|
func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
|
||||||
|
@ -125,13 +126,21 @@ func (self *DeltaCoordsCache) Close() {
|
||||||
self.Cache.Close()
|
self.Cache.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (self *DeltaCoordsCache) SetReadOnly(val bool) {
|
||||||
|
self.readOnly = val
|
||||||
|
}
|
||||||
|
|
||||||
func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
|
func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
|
||||||
bunchId := self.getBunchId(id)
|
bunchId := self.getBunchId(id)
|
||||||
bunch, err := self.getBunch(bunchId)
|
bunch, err := self.getBunch(bunchId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer bunch.Unlock()
|
if self.readOnly {
|
||||||
|
bunch.Unlock()
|
||||||
|
} else {
|
||||||
|
defer bunch.Unlock()
|
||||||
|
}
|
||||||
return bunch.GetCoord(id)
|
return bunch.GetCoord(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
24
goposm.go
24
goposm.go
|
@ -124,12 +124,6 @@ func main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err := osmCache.Open()
|
|
||||||
if err != nil {
|
|
||||||
die(err)
|
|
||||||
}
|
|
||||||
defer osmCache.Close()
|
|
||||||
|
|
||||||
progress := stats.StatsReporter()
|
progress := stats.StatsReporter()
|
||||||
|
|
||||||
tagmapping, err := mapping.NewMapping(*mappingFile)
|
tagmapping, err := mapping.NewMapping(*mappingFile)
|
||||||
|
@ -158,12 +152,16 @@ func main() {
|
||||||
|
|
||||||
if *read != "" {
|
if *read != "" {
|
||||||
step := log.StartStep("Reading OSM data")
|
step := log.StartStep("Reading OSM data")
|
||||||
|
err = osmCache.Open()
|
||||||
|
if err != nil {
|
||||||
|
die(err)
|
||||||
|
}
|
||||||
progress.Start()
|
progress.Start()
|
||||||
osmCache.Coords.SetLinearImport(true)
|
osmCache.Coords.SetLinearImport(true)
|
||||||
reader.ReadPbf(osmCache, progress, tagmapping, *read)
|
reader.ReadPbf(osmCache, progress, tagmapping, *read)
|
||||||
osmCache.Coords.SetLinearImport(false)
|
osmCache.Coords.SetLinearImport(false)
|
||||||
progress.Stop()
|
progress.Stop()
|
||||||
osmCache.Coords.Flush()
|
osmCache.Close()
|
||||||
log.StopStep(step)
|
log.StopStep(step)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -192,6 +190,11 @@ func main() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = osmCache.Open()
|
||||||
|
if err != nil {
|
||||||
|
die(err)
|
||||||
|
}
|
||||||
|
osmCache.Coords.SetReadOnly(true)
|
||||||
pointsTagMatcher := tagmapping.PointMatcher()
|
pointsTagMatcher := tagmapping.PointMatcher()
|
||||||
lineStringsTagMatcher := tagmapping.LineStringMatcher()
|
lineStringsTagMatcher := tagmapping.LineStringMatcher()
|
||||||
polygonsTagMatcher := tagmapping.PolygonMatcher()
|
polygonsTagMatcher := tagmapping.PolygonMatcher()
|
||||||
|
@ -204,6 +207,7 @@ func main() {
|
||||||
|
|
||||||
// blocks till the Relations.Iter() finishes
|
// blocks till the Relations.Iter() finishes
|
||||||
relWriter.Close()
|
relWriter.Close()
|
||||||
|
osmCache.Relations.Close()
|
||||||
|
|
||||||
ways := osmCache.Ways.Iter()
|
ways := osmCache.Ways.Iter()
|
||||||
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
|
wayWriter := writer.NewWayWriter(osmCache, diffCache, ways, db,
|
||||||
|
@ -213,6 +217,7 @@ func main() {
|
||||||
|
|
||||||
// blocks till the Ways.Iter() finishes
|
// blocks till the Ways.Iter() finishes
|
||||||
wayWriter.Close()
|
wayWriter.Close()
|
||||||
|
osmCache.Ways.Close()
|
||||||
|
|
||||||
nodes := osmCache.Nodes.Iter()
|
nodes := osmCache.Nodes.Iter()
|
||||||
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
|
nodeWriter := writer.NewNodeWriter(osmCache, nodes, db,
|
||||||
|
@ -222,14 +227,15 @@ func main() {
|
||||||
|
|
||||||
// blocks till the Nodes.Iter() finishes
|
// blocks till the Nodes.Iter() finishes
|
||||||
nodeWriter.Close()
|
nodeWriter.Close()
|
||||||
|
osmCache.Nodes.Close()
|
||||||
|
osmCache.Coords.Close()
|
||||||
|
osmCache.Coords.SetReadOnly(false)
|
||||||
|
|
||||||
err = db.End()
|
err = db.End()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
die(err)
|
die(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// insertBuffer.Close()
|
|
||||||
// dbWriter.Close()
|
|
||||||
progress.Stop()
|
progress.Stop()
|
||||||
|
|
||||||
if *diff {
|
if *diff {
|
||||||
|
|
Loading…
Reference in New Issue