read back relations/ways; fill ways with nodes from cache

cache/db.go

@@ -242,6 +242,26 @@ func (p *WaysCache) GetWay(id int64) (*element.Way, error) {
 	return way, nil
 }
 
+func (p *WaysCache) Iter() chan *element.Way {
+	way := make(chan *element.Way)
+	go func() {
+		ro := levigo.NewReadOptions()
+		ro.SetFillCache(false)
+		it := p.db.NewIterator(ro)
+		defer it.Close()
+		it.SeekToFirst()
+		for it = it; it.Valid(); it.Next() {
+			ways, err := binary.UnmarshalWay(it.Value())
+			if err != nil {
+				panic(err)
+			}
+			way <- ways
+		}
+		close(way)
+	}()
+	return way
+}
+
 func (p *RelationsCache) PutRelation(relation *element.Relation) error {
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(relation.Id))
@@ -268,6 +288,26 @@ func (p *RelationsCache) PutRelations(rels []element.Relation) error {
 	return p.db.Write(p.wo, batch)
 }
 
+func (p *RelationsCache) Iter() chan *element.Relation {
+	rel := make(chan *element.Relation)
+	go func() {
+		ro := levigo.NewReadOptions()
+		ro.SetFillCache(false)
+		it := p.db.NewIterator(ro)
+		defer it.Close()
+		it.SeekToFirst()
+		for it = it; it.Valid(); it.Next() {
+			relation, err := binary.UnmarshalRelation(it.Value())
+			if err != nil {
+				panic(err)
+			}
+			rel <- relation
+		}
+		close(rel)
+	}()
+	return rel
+}
+
 func (p *RelationsCache) GetRelation(id int64) (*element.Relation, error) {
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(id))

cache/delta.go

@@ -81,7 +81,7 @@ func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
 	}
 	coordsCache.lruList = list.New()
 	coordsCache.table = make(map[int64]*CoordsBunch)
-	coordsCache.capacity = 100
+	coordsCache.capacity = 1024
 	return &coordsCache, nil
 }
 
@@ -105,6 +105,13 @@ func (self *DeltaCoordsCache) GetCoord(id int64) (element.Node, bool) {
 	return node, true
 }
 
+func (self *DeltaCoordsCache) FillWay(way *element.Way) {
+	way.Nodes = make([]element.Node, len(way.Refs))
+	for i, id := range way.Refs {
+		way.Nodes[i], _ = self.GetCoord(id)
+	}
+}
+
 func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) {
 	var start, currentgetBunchId int64
 	currentgetBunchId = getBunchId(nodes[0].Id)

parser.go

@@ -7,7 +7,9 @@ import (
 	"goposm/element"
 	"goposm/parser"
 	"log"
+	"os"
 	"runtime"
+	"runtime/pprof"
 	"sync"
 )
 
@@ -109,16 +111,48 @@ func parse(filename string) {
 }
 
 func main() {
-	//f, err := os.Create("/tmp/goposm.pprof")
-	//if err != nil {
-	//log.Fatal(err)
-	//}
-	//pprof.StartCPUProfile(f)
-	//defer pprof.StopCPUProfile()
+	f, err := os.Create("/tmp/goposm.pprof")
+	if err != nil {
+		log.Fatal(err)
+	}
+	pprof.StartCPUProfile(f)
+	defer pprof.StopCPUProfile()
 	log.SetFlags(log.LstdFlags | log.Llongfile)
 	runtime.GOMAXPROCS(runtime.NumCPU())
 	flag.Parse()
-	parse(flag.Arg(0))
+	//parse(flag.Arg(0))
+
+	relCache, err := cache.NewRelationsCache("/tmp/goposm/relation.cache")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer relCache.Close()
+
+	rel := relCache.Iter()
+	for r := range rel {
+		fmt.Println(r)
+	}
+
+	wayCache, err := cache.NewWaysCache("/tmp/goposm/way.cache")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer wayCache.Close()
+
+	coordCache, err := cache.NewDeltaCoordsCache("/tmp/goposm/coords.cache")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer coordCache.Close()
+
+	way := wayCache.Iter()
+	i := 0
+	for w := range way {
+		i += 1
+		coordCache.FillWay(w)
+		//fmt.Println(i)
+	}
+	fmt.Println(i)
 	//parser.PBFStats(os.Args[1])
 	fmt.Println("done")
 }