full parsing and caching; first delta encoding of coords
parent
79b528d53e
commit
74a388887d
|
@ -10,11 +10,11 @@ import (
|
||||||
|
|
||||||
const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
|
const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
|
||||||
|
|
||||||
func coordToInt(coord float64) uint32 {
|
func CoordToInt(coord float64) uint32 {
|
||||||
return uint32((coord + 180.0) * COORD_FACTOR)
|
return uint32((coord + 180.0) * COORD_FACTOR)
|
||||||
}
|
}
|
||||||
|
|
||||||
func intToCoord(coord uint32) float64 {
|
func IntToCoord(coord uint32) float64 {
|
||||||
return float64((float64(coord) / COORD_FACTOR) - 180.0)
|
return float64((float64(coord) / COORD_FACTOR) - 180.0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -33,11 +33,11 @@ func MarshalCoord(node *element.Node) ([]byte, error) {
|
||||||
data := make([]byte, 8)
|
data := make([]byte, 8)
|
||||||
|
|
||||||
buf := bytes.NewBuffer(data)
|
buf := bytes.NewBuffer(data)
|
||||||
err := bin.Write(buf, bin.LittleEndian, coordToInt(node.Long))
|
err := bin.Write(buf, bin.LittleEndian, CoordToInt(node.Long))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
err = bin.Write(buf, bin.LittleEndian, coordToInt(node.Lat))
|
err = bin.Write(buf, bin.LittleEndian, CoordToInt(node.Lat))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -58,8 +58,8 @@ func UnmarshalCoord(id int64, data []byte) (node *element.Node, err error) {
|
||||||
|
|
||||||
node = &element.Node{}
|
node = &element.Node{}
|
||||||
node.Id = id
|
node.Id = id
|
||||||
node.Long = intToCoord(long)
|
node.Long = IntToCoord(long)
|
||||||
node.Lat = intToCoord(lat)
|
node.Lat = IntToCoord(lat)
|
||||||
return node, nil
|
return node, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package cache
|
package cache
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"code.google.com/p/goprotobuf/proto"
|
||||||
bin "encoding/binary"
|
bin "encoding/binary"
|
||||||
"github.com/jmhodges/levigo"
|
"github.com/jmhodges/levigo"
|
||||||
"goposm/binary"
|
"goposm/binary"
|
||||||
|
@ -38,6 +39,37 @@ func (p *Cache) PutCoord(node *element.Node) {
|
||||||
p.db.Put(p.wo, keyBuf, data)
|
p.db.Put(p.wo, keyBuf, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Cache) PutCoords(nodes []element.Node) {
|
||||||
|
batch := levigo.NewWriteBatch()
|
||||||
|
defer batch.Close()
|
||||||
|
|
||||||
|
keyBuf := make([]byte, 8)
|
||||||
|
for _, node := range nodes {
|
||||||
|
bin.PutVarint(keyBuf, int64(node.Id))
|
||||||
|
data, err := binary.MarshalCoord(&node)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
batch.Put(keyBuf, data)
|
||||||
|
}
|
||||||
|
p.db.Write(p.wo, batch)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Cache) PutCoordsPacked(nodes []element.Node) {
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
keyBuf := make([]byte, 8)
|
||||||
|
bin.PutVarint(keyBuf, int64(nodes[0].Id))
|
||||||
|
|
||||||
|
deltaCoords := packNodes(nodes)
|
||||||
|
data, err := proto.Marshal(deltaCoords)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
p.db.Put(p.wo, keyBuf, data)
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Cache) GetCoord(id int64) *element.Node {
|
func (p *Cache) GetCoord(id int64) *element.Node {
|
||||||
keyBuf := make([]byte, 8)
|
keyBuf := make([]byte, 8)
|
||||||
bin.PutVarint(keyBuf, int64(id))
|
bin.PutVarint(keyBuf, int64(id))
|
||||||
|
@ -93,6 +125,22 @@ func (p *Cache) PutWay(way *element.Way) {
|
||||||
p.db.Put(p.wo, keyBuf, data)
|
p.db.Put(p.wo, keyBuf, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Cache) PutWays(ways []element.Way) {
|
||||||
|
batch := levigo.NewWriteBatch()
|
||||||
|
defer batch.Close()
|
||||||
|
|
||||||
|
keyBuf := make([]byte, 8)
|
||||||
|
for _, way := range ways {
|
||||||
|
bin.PutVarint(keyBuf, int64(way.Id))
|
||||||
|
data, err := binary.MarshalWay(&way)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
batch.Put(keyBuf, data)
|
||||||
|
}
|
||||||
|
p.db.Write(p.wo, batch)
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Cache) GetWay(id int64) *element.Way {
|
func (p *Cache) GetWay(id int64) *element.Way {
|
||||||
keyBuf := make([]byte, 8)
|
keyBuf := make([]byte, 8)
|
||||||
bin.PutVarint(keyBuf, int64(id))
|
bin.PutVarint(keyBuf, int64(id))
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import (
|
||||||
|
"goposm/binary"
|
||||||
|
"goposm/element"
|
||||||
|
)
|
||||||
|
|
||||||
|
func packNodes(nodes []element.Node) *DeltaCoords {
|
||||||
|
var lastLon, lastLat int64
|
||||||
|
var lon, lat int64
|
||||||
|
var lastId, id int64
|
||||||
|
ids := make([]int64, len(nodes))
|
||||||
|
lons := make([]int64, len(nodes))
|
||||||
|
lats := make([]int64, len(nodes))
|
||||||
|
|
||||||
|
for i, nd := range nodes {
|
||||||
|
id = nd.Id
|
||||||
|
lon = int64(binary.CoordToInt(nd.Long))
|
||||||
|
lat = int64(binary.CoordToInt(nd.Lat))
|
||||||
|
ids[i] = id - lastId
|
||||||
|
lons[i] = lon - lastLon
|
||||||
|
lats[i] = lat - lastLat
|
||||||
|
|
||||||
|
lastId = id
|
||||||
|
lastLon = lon
|
||||||
|
lastLat = lat
|
||||||
|
}
|
||||||
|
return &DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
// Code generated by protoc-gen-go.
|
||||||
|
// source: internal.proto
|
||||||
|
// DO NOT EDIT!
|
||||||
|
|
||||||
|
package cache
|
||||||
|
|
||||||
|
import proto "code.google.com/p/goprotobuf/proto"
|
||||||
|
import json "encoding/json"
|
||||||
|
import math "math"
|
||||||
|
|
||||||
|
// Reference proto, json, and math imports to suppress error if they are not otherwise used.
|
||||||
|
var _ = proto.Marshal
|
||||||
|
var _ = &json.SyntaxError{}
|
||||||
|
var _ = math.Inf
|
||||||
|
|
||||||
|
type DeltaCoords struct {
|
||||||
|
Ids []int64 `protobuf:"zigzag64,1,rep,packed,name=ids" json:"ids,omitempty"`
|
||||||
|
Lats []int64 `protobuf:"zigzag64,2,rep,packed,name=lats" json:"lats,omitempty"`
|
||||||
|
Lons []int64 `protobuf:"zigzag64,3,rep,packed,name=lons" json:"lons,omitempty"`
|
||||||
|
XXX_unrecognized []byte `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *DeltaCoords) Reset() { *m = DeltaCoords{} }
|
||||||
|
func (m *DeltaCoords) String() string { return proto.CompactTextString(m) }
|
||||||
|
func (*DeltaCoords) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (m *DeltaCoords) GetIds() []int64 {
|
||||||
|
if m != nil {
|
||||||
|
return m.Ids
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *DeltaCoords) GetLats() []int64 {
|
||||||
|
if m != nil {
|
||||||
|
return m.Lats
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *DeltaCoords) GetLons() []int64 {
|
||||||
|
if m != nil {
|
||||||
|
return m.Lons
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
}
|
|
@ -0,0 +1,8 @@
|
||||||
|
package imposm.cache.internal;
|
||||||
|
|
||||||
|
message DeltaCoords {
|
||||||
|
repeated sint64 ids = 1 [packed = true];
|
||||||
|
repeated sint64 lats = 2 [packed = true];
|
||||||
|
repeated sint64 lons = 3 [packed = true];
|
||||||
|
}
|
||||||
|
|
48
parser.go
48
parser.go
|
@ -11,8 +11,8 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func parse(filename string) {
|
func parse(filename string) {
|
||||||
nodes := make(chan element.Node)
|
nodes := make(chan []element.Node)
|
||||||
ways := make(chan element.Way)
|
ways := make(chan []element.Way)
|
||||||
relations := make(chan element.Relation)
|
relations := make(chan element.Relation)
|
||||||
|
|
||||||
positions := parser.PBFBlockPositions(filename)
|
positions := parser.PBFBlockPositions(filename)
|
||||||
|
@ -29,41 +29,41 @@ func parse(filename string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
waitCounter := sync.WaitGroup{}
|
waitCounter := sync.WaitGroup{}
|
||||||
|
wayCache := cache.NewCache("/tmp/goposm/way.cache")
|
||||||
|
defer wayCache.Close()
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
waitCounter.Add(1)
|
||||||
|
go func() {
|
||||||
|
wayCounter := 0
|
||||||
|
for ws := range ways {
|
||||||
|
wayCache.PutWays(ws)
|
||||||
|
wayCounter += 1
|
||||||
|
}
|
||||||
|
fmt.Println("ways", wayCounter)
|
||||||
|
waitCounter.Done()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
relCache := cache.NewCache("/tmp/goposm/relation.cache")
|
||||||
|
defer relCache.Close()
|
||||||
waitCounter.Add(1)
|
waitCounter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
cache := cache.NewCache("/tmp/goposm/way.cache")
|
|
||||||
defer cache.Close()
|
|
||||||
|
|
||||||
wayCounter := 0
|
|
||||||
for way := range ways {
|
|
||||||
cache.PutWay(&way)
|
|
||||||
wayCounter += 1
|
|
||||||
}
|
|
||||||
fmt.Println("ways", wayCounter)
|
|
||||||
waitCounter.Done()
|
|
||||||
}()
|
|
||||||
waitCounter.Add(1)
|
|
||||||
go func() {
|
|
||||||
cache := cache.NewCache("/tmp/goposm/relation.cache")
|
|
||||||
defer cache.Close()
|
|
||||||
|
|
||||||
relationCounter := 0
|
relationCounter := 0
|
||||||
for rel := range relations {
|
for rel := range relations {
|
||||||
cache.PutRelation(&rel)
|
relCache.PutRelation(&rel)
|
||||||
relationCounter += 1
|
relationCounter += 1
|
||||||
}
|
}
|
||||||
fmt.Println("relations", relationCounter)
|
fmt.Println("relations", relationCounter)
|
||||||
waitCounter.Done()
|
waitCounter.Done()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
cache := cache.NewCache("/tmp/goposm/node.cache")
|
nodeCache := cache.NewCache("/tmp/goposm/node.cache")
|
||||||
defer cache.Close()
|
defer nodeCache.Close()
|
||||||
for i := 0; i < 4; i++ {
|
for i := 0; i < 2; i++ {
|
||||||
waitCounter.Add(1)
|
waitCounter.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
nodeCounter := 0
|
nodeCounter := 0
|
||||||
for node := range nodes {
|
for nds := range nodes {
|
||||||
cache.PutNode(&node)
|
nodeCache.PutCoordsPacked(nds)
|
||||||
nodeCounter += 1
|
nodeCounter += 1
|
||||||
}
|
}
|
||||||
fmt.Println("nodes", nodeCounter)
|
fmt.Println("nodes", nodeCounter)
|
||||||
|
|
|
@ -212,7 +212,7 @@ func PBFBlockPositions(filename string) chan BlockPosition {
|
||||||
return pbf.BlockPositions()
|
return pbf.BlockPositions()
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Way, relations chan element.Relation) {
|
func ParseBlock(pos BlockPosition, nodes chan []element.Node, ways chan []element.Way, relations chan element.Relation) {
|
||||||
block := ReadPrimitiveBlock(pos)
|
block := ReadPrimitiveBlock(pos)
|
||||||
stringtable := NewStringTable(block.GetStringtable())
|
stringtable := NewStringTable(block.GetStringtable())
|
||||||
|
|
||||||
|
@ -220,18 +220,18 @@ func ParseBlock(pos BlockPosition, nodes chan element.Node, ways chan element.Wa
|
||||||
dense := group.GetDense()
|
dense := group.GetDense()
|
||||||
if dense != nil {
|
if dense != nil {
|
||||||
parsedNodes := ReadDenseNodes(dense, block, stringtable)
|
parsedNodes := ReadDenseNodes(dense, block, stringtable)
|
||||||
for _, node := range parsedNodes {
|
nodes <- parsedNodes
|
||||||
nodes <- node
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
parsedNodes := ReadNodes(group.Nodes, block, stringtable)
|
parsedNodes := ReadNodes(group.Nodes, block, stringtable)
|
||||||
for _, node := range parsedNodes {
|
nodes <- parsedNodes
|
||||||
nodes <- node
|
//for _, node := range parsedNodes {
|
||||||
}
|
//nodes <- node
|
||||||
|
//}
|
||||||
parsedWays := ReadWays(group.Ways, block, stringtable)
|
parsedWays := ReadWays(group.Ways, block, stringtable)
|
||||||
for _, way := range parsedWays {
|
ways <- parsedWays
|
||||||
ways <- way
|
//for _, way := range parsedWays {
|
||||||
}
|
//ways <- way
|
||||||
|
//}
|
||||||
parsedRelations := ReadRelations(group.Relations, block, stringtable)
|
parsedRelations := ReadRelations(group.Relations, block, stringtable)
|
||||||
for _, rel := range parsedRelations {
|
for _, rel := range parsedRelations {
|
||||||
relations <- rel
|
relations <- rel
|
||||||
|
|
Loading…
Reference in New Issue