full parsing and caching; first delta encoding of coords

binary/serialize.go

@@ -10,11 +10,11 @@ import (
 
 const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
 
-func coordToInt(coord float64) uint32 {
+func CoordToInt(coord float64) uint32 {
 	return uint32((coord + 180.0) * COORD_FACTOR)
 }
 
-func intToCoord(coord uint32) float64 {
+func IntToCoord(coord uint32) float64 {
 	return float64((float64(coord) / COORD_FACTOR) - 180.0)
 }
 
@@ -33,11 +33,11 @@ func MarshalCoord(node *element.Node) ([]byte, error) {
 	data := make([]byte, 8)
 
 	buf := bytes.NewBuffer(data)
-	err := bin.Write(buf, bin.LittleEndian, coordToInt(node.Long))
+	err := bin.Write(buf, bin.LittleEndian, CoordToInt(node.Long))
 	if err != nil {
 		return nil, err
 	}
-	err = bin.Write(buf, bin.LittleEndian, coordToInt(node.Lat))
+	err = bin.Write(buf, bin.LittleEndian, CoordToInt(node.Lat))
 	if err != nil {
 		return nil, err
 	}
@@ -58,8 +58,8 @@ func UnmarshalCoord(id int64, data []byte) (node *element.Node, err error) {
 
 	node = &element.Node{}
 	node.Id = id
-	node.Long = intToCoord(long)
-	node.Lat = intToCoord(lat)
+	node.Long = IntToCoord(long)
+	node.Lat = IntToCoord(lat)
 	return node, nil
 }
 

cache/db.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"code.google.com/p/goprotobuf/proto"
 	bin "encoding/binary"
 	"github.com/jmhodges/levigo"
 	"goposm/binary"
@@ -38,6 +39,37 @@ func (p *Cache) PutCoord(node *element.Node) {
 	p.db.Put(p.wo, keyBuf, data)
 }
 
+func (p *Cache) PutCoords(nodes []element.Node) {
+	batch := levigo.NewWriteBatch()
+	defer batch.Close()
+
+	keyBuf := make([]byte, 8)
+	for _, node := range nodes {
+		bin.PutVarint(keyBuf, int64(node.Id))
+		data, err := binary.MarshalCoord(&node)
+		if err != nil {
+			panic(err)
+		}
+		batch.Put(keyBuf, data)
+	}
+	p.db.Write(p.wo, batch)
+}
+
+func (p *Cache) PutCoordsPacked(nodes []element.Node) {
+	if len(nodes) == 0 {
+		return
+	}
+	keyBuf := make([]byte, 8)
+	bin.PutVarint(keyBuf, int64(nodes[0].Id))
+
+	deltaCoords := packNodes(nodes)
+	data, err := proto.Marshal(deltaCoords)
+	if err != nil {
+		panic(err)
+	}
+	p.db.Put(p.wo, keyBuf, data)
+}
+
 func (p *Cache) GetCoord(id int64) *element.Node {
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(id))
@@ -93,6 +125,22 @@ func (p *Cache) PutWay(way *element.Way) {
 	p.db.Put(p.wo, keyBuf, data)
 }
 
+func (p *Cache) PutWays(ways []element.Way) {
+	batch := levigo.NewWriteBatch()
+	defer batch.Close()
+
+	keyBuf := make([]byte, 8)
+	for _, way := range ways {
+		bin.PutVarint(keyBuf, int64(way.Id))
+		data, err := binary.MarshalWay(&way)
+		if err != nil {
+			panic(err)
+		}
+		batch.Put(keyBuf, data)
+	}
+	p.db.Write(p.wo, batch)
+}
+
 func (p *Cache) GetWay(id int64) *element.Way {
 	keyBuf := make([]byte, 8)
 	bin.PutVarint(keyBuf, int64(id))

cache/delta.go

@@ -0,0 +1,29 @@
+package cache
+
+import (
+	"goposm/binary"
+	"goposm/element"
+)
+
+func packNodes(nodes []element.Node) *DeltaCoords {
+	var lastLon, lastLat int64
+	var lon, lat int64
+	var lastId, id int64
+	ids := make([]int64, len(nodes))
+	lons := make([]int64, len(nodes))
+	lats := make([]int64, len(nodes))
+
+	for i, nd := range nodes {
+		id = nd.Id
+		lon = int64(binary.CoordToInt(nd.Long))
+		lat = int64(binary.CoordToInt(nd.Lat))
+		ids[i] = id - lastId
+		lons[i] = lon - lastLon
+		lats[i] = lat - lastLat
+
+		lastId = id
+		lastLon = lon
+		lastLat = lat
+	}
+	return &DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
+}

cache/internal.pb.go

@@ -0,0 +1,49 @@
+// Code generated by protoc-gen-go.
+// source: internal.proto
+// DO NOT EDIT!
+
+package cache
+
+import proto "code.google.com/p/goprotobuf/proto"
+import json "encoding/json"
+import math "math"
+
+// Reference proto, json, and math imports to suppress error if they are not otherwise used.
+var _ = proto.Marshal
+var _ = &json.SyntaxError{}
+var _ = math.Inf
+
+type DeltaCoords struct {
+	Ids              []int64 `protobuf:"zigzag64,1,rep,packed,name=ids" json:"ids,omitempty"`
+	Lats             []int64 `protobuf:"zigzag64,2,rep,packed,name=lats" json:"lats,omitempty"`
+	Lons             []int64 `protobuf:"zigzag64,3,rep,packed,name=lons" json:"lons,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *DeltaCoords) Reset()         { *m = DeltaCoords{} }
+func (m *DeltaCoords) String() string { return proto.CompactTextString(m) }
+func (*DeltaCoords) ProtoMessage()    {}
+
+func (m *DeltaCoords) GetIds() []int64 {
+	if m != nil {
+		return m.Ids
+	}
+	return nil
+}
+
+func (m *DeltaCoords) GetLats() []int64 {
+	if m != nil {
+		return m.Lats
+	}
+	return nil
+}
+
+func (m *DeltaCoords) GetLons() []int64 {
+	if m != nil {
+		return m.Lons
+	}
+	return nil
+}
+
+func init() {
+}

cache/internal.proto

@@ -0,0 +1,8 @@
+package imposm.cache.internal;
+
+message DeltaCoords {
+   repeated sint64 ids = 1 [packed = true];
+   repeated sint64 lats = 2 [packed = true];
+   repeated sint64 lons = 3 [packed = true];
+}
+

parser.go

@@ -11,8 +11,8 @@ import (
 )
 
 func parse(filename string) {
-	nodes := make(chan element.Node)
-	ways := make(chan element.Way)
+	nodes := make(chan []element.Node)
+	ways := make(chan []element.Way)
 	relations := make(chan element.Relation)
 
 	positions := parser.PBFBlockPositions(filename)
@@ -29,41 +29,41 @@ func parse(filename string) {
 	}
 
 	waitCounter := sync.WaitGroup{}
+	wayCache := cache.NewCache("/tmp/goposm/way.cache")
+	defer wayCache.Close()
+	for i := 0; i < 2; i++ {
+		waitCounter.Add(1)
+		go func() {
+			wayCounter := 0
+			for ws := range ways {
+				wayCache.PutWays(ws)
+				wayCounter += 1
+			}
+			fmt.Println("ways", wayCounter)
+			waitCounter.Done()
+		}()
+	}
+	relCache := cache.NewCache("/tmp/goposm/relation.cache")
+	defer relCache.Close()
 	waitCounter.Add(1)
 	go func() {
-		cache := cache.NewCache("/tmp/goposm/way.cache")
-		defer cache.Close()
-
-		wayCounter := 0
-		for way := range ways {
-			cache.PutWay(&way)
-			wayCounter += 1
-		}
-		fmt.Println("ways", wayCounter)
-		waitCounter.Done()
-	}()
-	waitCounter.Add(1)
-	go func() {
-		cache := cache.NewCache("/tmp/goposm/relation.cache")
-		defer cache.Close()
-
 		relationCounter := 0
 		for rel := range relations {
-			cache.PutRelation(&rel)
+			relCache.PutRelation(&rel)
 			relationCounter += 1
 		}
 		fmt.Println("relations", relationCounter)
 		waitCounter.Done()
 	}()
 
-	cache := cache.NewCache("/tmp/goposm/node.cache")
-	defer cache.Close()
-	for i := 0; i < 4; i++ {
+	nodeCache := cache.NewCache("/tmp/goposm/node.cache")
+	defer nodeCache.Close()
+	for i := 0; i < 2; i++ {
 		waitCounter.Add(1)
 		go func() {
 			nodeCounter := 0
-			for node := range nodes {
-				cache.PutNode(&node)
+			for nds := range nodes {
+				nodeCache.PutCoordsPacked(nds)
 				nodeCounter += 1
 			}
 			fmt.Println("nodes", nodeCounter)

parser/pbf.go

@@ -212,7 +212,7 @@ func PBFBlockPositions(filename string) chan BlockPosition {
 	return pbf.BlockPositions()
 }
 
-func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Way, relations chan element.Relation) {
+func ParseBlock(pos BlockPosition, nodes chan []element.Node, ways chan []element.Way, relations chan element.Relation) {
 	block := ReadPrimitiveBlock(pos)
 	stringtable := NewStringTable(block.GetStringtable())
 
@@ -220,18 +220,18 @@ func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Wa
 		dense := group.GetDense()
 		if dense != nil {
 			parsedNodes := ReadDenseNodes(dense, block, stringtable)
-			for _, node := range parsedNodes {
-				nodes <- node
-			}
+			nodes <- parsedNodes
 		}
 		parsedNodes := ReadNodes(group.Nodes, block, stringtable)
-		for _, node := range parsedNodes {
-			nodes <- node
-		}
+		nodes <- parsedNodes
+		//for _, node := range parsedNodes {
+		//nodes <- node
+		//}
 		parsedWays := ReadWays(group.Ways, block, stringtable)
-		for _, way := range parsedWays {
-			ways <- way
-		}
+		ways <- parsedWays
+		//for _, way := range parsedWays {
+		//ways <- way
+		//}
 		parsedRelations := ReadRelations(group.Relations, block, stringtable)
 		for _, rel := range parsedRelations {
 			relations <- rel