From 74a388887d8f52d29540ec62cfd93ea9d514c2ef Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Thu, 11 Apr 2013 19:50:20 +0200 Subject: [PATCH] full parsing and caching; first delta encoding of coords --- binary/serialize.go | 12 +++++------ cache/db.go | 48 +++++++++++++++++++++++++++++++++++++++++++ cache/delta.go | 29 ++++++++++++++++++++++++++ cache/internal.pb.go | 49 ++++++++++++++++++++++++++++++++++++++++++++ cache/internal.proto | 8 ++++++++ parser.go | 48 +++++++++++++++++++++---------------------- parser/pbf.go | 20 +++++++++--------- 7 files changed, 174 insertions(+), 40 deletions(-) create mode 100644 cache/delta.go create mode 100644 cache/internal.pb.go create mode 100644 cache/internal.proto diff --git a/binary/serialize.go b/binary/serialize.go index 9b95629..99549c4 100644 --- a/binary/serialize.go +++ b/binary/serialize.go @@ -10,11 +10,11 @@ import ( const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0 -func coordToInt(coord float64) uint32 { +func CoordToInt(coord float64) uint32 { return uint32((coord + 180.0) * COORD_FACTOR) } -func intToCoord(coord uint32) float64 { +func IntToCoord(coord uint32) float64 { return float64((float64(coord) / COORD_FACTOR) - 180.0) } @@ -33,11 +33,11 @@ func MarshalCoord(node *element.Node) ([]byte, error) { data := make([]byte, 8) buf := bytes.NewBuffer(data) - err := bin.Write(buf, bin.LittleEndian, coordToInt(node.Long)) + err := bin.Write(buf, bin.LittleEndian, CoordToInt(node.Long)) if err != nil { return nil, err } - err = bin.Write(buf, bin.LittleEndian, coordToInt(node.Lat)) + err = bin.Write(buf, bin.LittleEndian, CoordToInt(node.Lat)) if err != nil { return nil, err } @@ -58,8 +58,8 @@ func UnmarshalCoord(id int64, data []byte) (node *element.Node, err error) { node = &element.Node{} node.Id = id - node.Long = intToCoord(long) - node.Lat = intToCoord(lat) + node.Long = IntToCoord(long) + node.Lat = IntToCoord(lat) return node, nil } diff --git a/cache/db.go b/cache/db.go index e4147aa..5ec41b9 100644 --- a/cache/db.go +++ b/cache/db.go @@ -1,6 +1,7 @@ package cache import ( + "code.google.com/p/goprotobuf/proto" bin "encoding/binary" "github.com/jmhodges/levigo" "goposm/binary" @@ -38,6 +39,37 @@ func (p *Cache) PutCoord(node *element.Node) { p.db.Put(p.wo, keyBuf, data) } +func (p *Cache) PutCoords(nodes []element.Node) { + batch := levigo.NewWriteBatch() + defer batch.Close() + + keyBuf := make([]byte, 8) + for _, node := range nodes { + bin.PutVarint(keyBuf, int64(node.Id)) + data, err := binary.MarshalCoord(&node) + if err != nil { + panic(err) + } + batch.Put(keyBuf, data) + } + p.db.Write(p.wo, batch) +} + +func (p *Cache) PutCoordsPacked(nodes []element.Node) { + if len(nodes) == 0 { + return + } + keyBuf := make([]byte, 8) + bin.PutVarint(keyBuf, int64(nodes[0].Id)) + + deltaCoords := packNodes(nodes) + data, err := proto.Marshal(deltaCoords) + if err != nil { + panic(err) + } + p.db.Put(p.wo, keyBuf, data) +} + func (p *Cache) GetCoord(id int64) *element.Node { keyBuf := make([]byte, 8) bin.PutVarint(keyBuf, int64(id)) @@ -93,6 +125,22 @@ func (p *Cache) PutWay(way *element.Way) { p.db.Put(p.wo, keyBuf, data) } +func (p *Cache) PutWays(ways []element.Way) { + batch := levigo.NewWriteBatch() + defer batch.Close() + + keyBuf := make([]byte, 8) + for _, way := range ways { + bin.PutVarint(keyBuf, int64(way.Id)) + data, err := binary.MarshalWay(&way) + if err != nil { + panic(err) + } + batch.Put(keyBuf, data) + } + p.db.Write(p.wo, batch) +} + func (p *Cache) GetWay(id int64) *element.Way { keyBuf := make([]byte, 8) bin.PutVarint(keyBuf, int64(id)) diff --git a/cache/delta.go b/cache/delta.go new file mode 100644 index 0000000..bf07bdc --- /dev/null +++ b/cache/delta.go @@ -0,0 +1,29 @@ +package cache + +import ( + "goposm/binary" + "goposm/element" +) + +func packNodes(nodes []element.Node) *DeltaCoords { + var lastLon, lastLat int64 + var lon, lat int64 + var lastId, id int64 + ids := make([]int64, len(nodes)) + lons := make([]int64, len(nodes)) + lats := make([]int64, len(nodes)) + + for i, nd := range nodes { + id = nd.Id + lon = int64(binary.CoordToInt(nd.Long)) + lat = int64(binary.CoordToInt(nd.Lat)) + ids[i] = id - lastId + lons[i] = lon - lastLon + lats[i] = lat - lastLat + + lastId = id + lastLon = lon + lastLat = lat + } + return &DeltaCoords{Ids: ids, Lats: lats, Lons: lons} +} diff --git a/cache/internal.pb.go b/cache/internal.pb.go new file mode 100644 index 0000000..c410b68 --- /dev/null +++ b/cache/internal.pb.go @@ -0,0 +1,49 @@ +// Code generated by protoc-gen-go. +// source: internal.proto +// DO NOT EDIT! + +package cache + +import proto "code.google.com/p/goprotobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +type DeltaCoords struct { + Ids []int64 `protobuf:"zigzag64,1,rep,packed,name=ids" json:"ids,omitempty"` + Lats []int64 `protobuf:"zigzag64,2,rep,packed,name=lats" json:"lats,omitempty"` + Lons []int64 `protobuf:"zigzag64,3,rep,packed,name=lons" json:"lons,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *DeltaCoords) Reset() { *m = DeltaCoords{} } +func (m *DeltaCoords) String() string { return proto.CompactTextString(m) } +func (*DeltaCoords) ProtoMessage() {} + +func (m *DeltaCoords) GetIds() []int64 { + if m != nil { + return m.Ids + } + return nil +} + +func (m *DeltaCoords) GetLats() []int64 { + if m != nil { + return m.Lats + } + return nil +} + +func (m *DeltaCoords) GetLons() []int64 { + if m != nil { + return m.Lons + } + return nil +} + +func init() { +} diff --git a/cache/internal.proto b/cache/internal.proto new file mode 100644 index 0000000..c67b37c --- /dev/null +++ b/cache/internal.proto @@ -0,0 +1,8 @@ +package imposm.cache.internal; + +message DeltaCoords { + repeated sint64 ids = 1 [packed = true]; + repeated sint64 lats = 2 [packed = true]; + repeated sint64 lons = 3 [packed = true]; +} + diff --git a/parser.go b/parser.go index 2bfe907..1dec5de 100644 --- a/parser.go +++ b/parser.go @@ -11,8 +11,8 @@ import ( ) func parse(filename string) { - nodes := make(chan element.Node) - ways := make(chan element.Way) + nodes := make(chan []element.Node) + ways := make(chan []element.Way) relations := make(chan element.Relation) positions := parser.PBFBlockPositions(filename) @@ -29,41 +29,41 @@ func parse(filename string) { } waitCounter := sync.WaitGroup{} + wayCache := cache.NewCache("/tmp/goposm/way.cache") + defer wayCache.Close() + for i := 0; i < 2; i++ { + waitCounter.Add(1) + go func() { + wayCounter := 0 + for ws := range ways { + wayCache.PutWays(ws) + wayCounter += 1 + } + fmt.Println("ways", wayCounter) + waitCounter.Done() + }() + } + relCache := cache.NewCache("/tmp/goposm/relation.cache") + defer relCache.Close() waitCounter.Add(1) go func() { - cache := cache.NewCache("/tmp/goposm/way.cache") - defer cache.Close() - - wayCounter := 0 - for way := range ways { - cache.PutWay(&way) - wayCounter += 1 - } - fmt.Println("ways", wayCounter) - waitCounter.Done() - }() - waitCounter.Add(1) - go func() { - cache := cache.NewCache("/tmp/goposm/relation.cache") - defer cache.Close() - relationCounter := 0 for rel := range relations { - cache.PutRelation(&rel) + relCache.PutRelation(&rel) relationCounter += 1 } fmt.Println("relations", relationCounter) waitCounter.Done() }() - cache := cache.NewCache("/tmp/goposm/node.cache") - defer cache.Close() - for i := 0; i < 4; i++ { + nodeCache := cache.NewCache("/tmp/goposm/node.cache") + defer nodeCache.Close() + for i := 0; i < 2; i++ { waitCounter.Add(1) go func() { nodeCounter := 0 - for node := range nodes { - cache.PutNode(&node) + for nds := range nodes { + nodeCache.PutCoordsPacked(nds) nodeCounter += 1 } fmt.Println("nodes", nodeCounter) diff --git a/parser/pbf.go b/parser/pbf.go index e8889f0..3c2b5e5 100644 --- a/parser/pbf.go +++ b/parser/pbf.go @@ -212,7 +212,7 @@ func PBFBlockPositions(filename string) chan BlockPosition { return pbf.BlockPositions() } -func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Way, relations chan element.Relation) { +func ParseBlock(pos BlockPosition, nodes chan []element.Node, ways chan []element.Way, relations chan element.Relation) { block := ReadPrimitiveBlock(pos) stringtable := NewStringTable(block.GetStringtable()) @@ -220,18 +220,18 @@ func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Wa dense := group.GetDense() if dense != nil { parsedNodes := ReadDenseNodes(dense, block, stringtable) - for _, node := range parsedNodes { - nodes <- node - } + nodes <- parsedNodes } parsedNodes := ReadNodes(group.Nodes, block, stringtable) - for _, node := range parsedNodes { - nodes <- node - } + nodes <- parsedNodes + //for _, node := range parsedNodes { + //nodes <- node + //} parsedWays := ReadWays(group.Ways, block, stringtable) - for _, way := range parsedWays { - ways <- way - } + ways <- parsedWays + //for _, way := range parsedWays { + //ways <- way + //} parsedRelations := ReadRelations(group.Relations, block, stringtable) for _, rel := range parsedRelations { relations <- rel