add separate cache for nodes and coords

cache/db.go

@@ -127,6 +127,9 @@ func (p *CoordsCache) GetCoord(id int64) (*element.Node, error) {
 }
 
 func (p *NodesCache) PutNode(node *element.Node) error {
+	if node.Tags == nil {
+		return nil
+	}
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(node.Id))
 	data, err := binary.MarshalNode(node)
@@ -136,6 +139,27 @@ func (p *NodesCache) PutNode(node *element.Node) error {
 	return p.db.Put(p.wo, keyBuf, data)
 }
 
+func (p *NodesCache) PutNodes(nodes []element.Node) (int, error) {
+	batch := levigo.NewWriteBatch()
+	defer batch.Close()
+
+	keyBuf := make([]byte, 8)
+	var n int
+	for _, node := range nodes {
+		if node.Tags == nil {
+			continue
+		}
+		bin.PutVarint(keyBuf, int64(node.Id))
+		data, err := binary.MarshalNode(&node)
+		if err != nil {
+			panic(err)
+		}
+		batch.Put(keyBuf, data)
+		n += 1
+	}
+	return n, p.db.Write(p.wo, batch)
+}
+
 func (p *NodesCache) GetNode(id int64) (*element.Node, error) {
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(id))

parser.go

@@ -1,18 +1,19 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"goposm/cache"
 	"goposm/element"
 	"goposm/parser"
 	"log"
-	"os"
 	"runtime"
 	"sync"
 )
 
 func parse(filename string) {
 	nodes := make(chan []element.Node)
+	coords := make(chan []element.Node)
 	ways := make(chan []element.Way)
 	relations := make(chan []element.Relation)
 
@@ -23,7 +24,7 @@ func parse(filename string) {
 		waitParser.Add(1)
 		go func() {
 			for pos := range positions {
-				parser.ParseBlock(pos, nodes, ways, relations)
+				parser.ParseBlock(pos, coords, nodes, ways, relations)
 			}
 			waitParser.Done()
 		}()
@@ -64,43 +65,43 @@ func parse(filename string) {
 			waitCounter.Done()
 		}()
 	}
-	nodeCache, err := cache.NewDeltaCoordsCache("/tmp/goposm/node.cache")
+	coordCache, err := cache.NewDeltaCoordsCache("/tmp/goposm/coords.cache")
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer nodeCache.Close()
+	defer coordCache.Close()
 	for i := 0; i < runtime.NumCPU(); i++ {
 		waitCounter.Add(1)
 		go func() {
 			nodeCounter := 0
-			for nds := range nodes {
-				nodeCache.PutCoords(nds)
+			for nds := range coords {
+				coordCache.PutCoords(nds)
 				nodeCounter += len(nds)
 			}
+			fmt.Println("coords", nodeCounter)
+			waitCounter.Done()
+		}()
+	}
+	nodeCache, err := cache.NewNodesCache("/tmp/goposm/node.cache")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer nodeCache.Close()
+	for i := 0; i < 2; i++ {
+		waitCounter.Add(1)
+		go func() {
+			nodeCounter := 0
+			for nds := range nodes {
+				n, _ := nodeCache.PutNodes(nds)
+				nodeCounter += n
+			}
 			fmt.Println("nodes", nodeCounter)
 			waitCounter.Done()
 		}()
 	}
-	/*
-		nodeCache := cache.NewCache("/tmp/goposm/node.cache")
-		defer nodeCache.Close()
-		for i := 0; i < 2; i++ {
-			waitCounter.Add(1)
-			go func() {
-				nodeCounter := 0
-				for nds := range nodes {
-					if len(nds) == 0 {
-						continue
-					}
-					nodeCache.PutCoordsPacked(nds[0].Id/8196, nds)
-					nodeCounter += 1
-				}
-				fmt.Println("nodes", nodeCounter)
-				waitCounter.Done()
-			}()
-		}
-	*/
+
 	waitParser.Wait()
+	close(coords)
 	close(nodes)
 	close(ways)
 	close(relations)
@@ -114,8 +115,10 @@ func main() {
 	//}
 	//pprof.StartCPUProfile(f)
 	//defer pprof.StopCPUProfile()
+	log.SetFlags(log.LstdFlags | log.Llongfile)
 	runtime.GOMAXPROCS(runtime.NumCPU())
-	parse(os.Args[1])
+	flag.Parse()
+	parse(flag.Arg(0))
 	//parser.PBFStats(os.Args[1])
 	fmt.Println("done")
 }

parser/pbf.go

@@ -78,10 +78,12 @@ func ReadDenseNodes(
 		nodes[i].Id = lastId
 		nodes[i].Long = (coordScale * float64(lonOffset+(granularity*lastLon)))
 		nodes[i].Lat = (coordScale * float64(latOffset+(granularity*lastLat)))
-		if dense.KeysVals[lastKeyValPos] != 0 {
-			nodes[i].Tags = ParseDenseNodeTags(stringtable, &dense.KeysVals, &lastKeyValPos)
-		} else {
-			lastKeyValPos += 1
+		if stringtable != nil {
+			if dense.KeysVals[lastKeyValPos] != 0 {
+				nodes[i].Tags = ParseDenseNodeTags(stringtable, &dense.KeysVals, &lastKeyValPos)
+			} else {
+				lastKeyValPos += 1
+			}
 		}
 	}
 	return nodes
@@ -132,7 +134,9 @@ func ReadNodes(
 		result[i].Id = id
 		result[i].Long = (coordScale * float64(lonOffset+(granularity*lon)))
 		result[i].Lat = (coordScale * float64(latOffset+(granularity*lat)))
-		result[i].Tags = ParseTags(stringtable, nodes[i].Keys, nodes[i].Vals)
+		if stringtable != nil {
+			result[i].Tags = ParseTags(stringtable, nodes[i].Keys, nodes[i].Vals)
+		}
 	}
 	return result
 }
@@ -212,7 +216,7 @@ func PBFBlockPositions(filename string) chan BlockPosition {
 	return pbf.BlockPositions()
 }
 
-func ParseBlock(pos BlockPosition, nodes chan []element.Node, ways chan []element.Way, relations chan []element.Relation) {
+func ParseBlock(pos BlockPosition, coords chan []element.Node, nodes chan []element.Node, ways chan []element.Way, relations chan []element.Relation) {
 	block := ReadPrimitiveBlock(pos)
 	stringtable := NewStringTable(block.GetStringtable())
 
@@ -222,11 +226,13 @@ func ParseBlock(pos BlockPosition, nodes chan []element.Node, ways chan []elemen
 			parsedNodes := ReadDenseNodes(dense, block, stringtable)
 			if len(parsedNodes) > 0 {
 				nodes <- parsedNodes
+				coords <- parsedNodes
 			}
 		}
 		parsedNodes := ReadNodes(group.Nodes, block, stringtable)
 		if len(parsedNodes) > 0 {
 			nodes <- parsedNodes
+			coords <- parsedNodes
 		}
 		parsedWays := ReadWays(group.Ways, block, stringtable)
 		if len(parsedWays) > 0 {