vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

more hacking

master
Oliver Tonnhofer 2013-02-12 19:45:49 +01:00
parent bcabc7a339
commit 17484a68e9
12 changed files with 585 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
binary/serialize.go
View File

@ -2,7 +2,6 @@ package binary
import (
"code.google.com/p/goprotobuf/proto"
"goposm/element"
"goposm/model"
)
@ -11,10 +10,26 @@ import (
// msg string
// }
func tagsFromArray(arr []string) *element.Tags {
result := make(element.Tags)
for i := 0; i < len(arr); i += 2 {
result[arr[i]] = arr[i+1]
}
return &result
}
func tagsAsArray(tags *element.Tags) []string {
result := make([]string, 0, 2*len(*tags))
for key, val := range *tags {
result = append(result, key, val)
}
return result
}
func Marshal(elem interface{}) ([]byte, error) {
switch typedElem := elem.(type) {
case element.Node:
return MarshalNode(typedElem)
return MarshalNode(&typedElem)
default:
panic("invalid elem to marshal")
}
@ -22,10 +37,47 @@ func Marshal(elem interface{}) ([]byte, error) {
return []byte{}, nil
}
func MarshalNode(node element.Node) ([]byte, error) {
func MarshalNode(node *element.Node) ([]byte, error) {
pbfNode := &model.Node{}
foo := int64(node.Id)
pbfNode.Id = &foo
nodeId := node.Id
pbfNode.Id = &nodeId
pbfNode.FromWgsCoord(node.Long, node.Lat)
pbfNode.Tags = tagsAsArray(&node.Tags)
return proto.Marshal(pbfNode)
}
func UnmarshalNode(data []byte) (node *element.Node, err error) {
pbfNode := &model.Node{}
err = proto.Unmarshal(data, pbfNode)
if err != nil {
return nil, err
}
node = &element.Node{}
node.Id = *pbfNode.Id
node.Long, node.Lat = pbfNode.WgsCoord()
node.Tags = *tagsFromArray(pbfNode.Tags)
return node, nil
}
func MarshalWay(way *element.Way) ([]byte, error) {
pbfWay := &model.Way{}
pbfWay.Id = &way.Id
pbfWay.Nodes = way.Nodes
pbfWay.Tags = tagsAsArray(&way.Tags)
return proto.Marshal(pbfWay)
}
func UnmarshalWay(data []byte) (way *element.Way, err error) {
pbfWay := &model.Way{}
err = proto.Unmarshal(data, pbfWay)
if err != nil {
return nil, err
}
way = &element.Way{}
way.Id = *pbfWay.Id
way.Nodes = pbfWay.Nodes
way.Tags = *tagsFromArray(pbfWay.Tags)
return way, nil
}

75
binary/serialize_test.go Normal file
View File

@ -0,0 +1,75 @@
package binary
import (
"goposm/element"
"testing"
)
func compareNodes(a []int64, b []int64) bool {
if len(a) != len(b) {
return false
}
for i, v := range a {
if v != b[i] {
return false
}
}
return true
}
func TestMarshalNode(t *testing.T) {
node := &element.Node{}
node.Id = 12345
node.Tags = make(element.Tags)
node.Tags["name"] = "test"
node.Tags["place"] = "city"
data, _ := MarshalNode(node)
node, _ = UnmarshalNode(data)
if node.Id != 12345 {
t.Error("id does not match")
}
if node.Tags["name"] != "test" {
t.Error("name tag does not match")
}
if node.Tags["place"] != "city" {
t.Error("place tag does not match")
}
if len(node.Tags) != 2 {
t.Error("tags len does not match")
}
}
func TestMarshalWay(t *testing.T) {
way := &element.Way{}
way.Id = 12345
way.Tags = make(element.Tags)
way.Tags["name"] = "test"
way.Tags["highway"] = "trunk"
way.Nodes = append(way.Nodes, 1, 2, 3, 4)
data, _ := MarshalWay(way)
way, _ = UnmarshalWay(data)
if way.Id != 12345 {
t.Error("id does not match")
}
if way.Tags["name"] != "test" {
t.Error("name tag does not match")
}
if way.Tags["highway"] != "trunk" {
t.Error("highway tag does not match")
}
if len(way.Tags) != 2 {
t.Error("tags len does not match")
}
if !compareNodes(way.Nodes, []int64{1, 2, 3, 4}) {
t.Error("nodes do not match")
}
}

56
cache/db.go vendored Normal file
View File

@ -0,0 +1,56 @@
package cache
import (
bin "encoding/binary"
"github.com/jmhodges/levigo"
"goposm/binary"
"goposm/element"
)
type Cache struct {
db *levigo.DB
wo *levigo.WriteOptions
ro *levigo.ReadOptions
}
func NewCache(path string) *Cache {
result := &Cache{}
opts := levigo.NewOptions()
opts.SetCreateIfMissing(true)
db, err := levigo.Open(path, opts)
if err != nil {
panic("unable to open db")
}
result.db = db
result.wo = levigo.NewWriteOptions()
result.ro = levigo.NewReadOptions()
return result
}
func (p *Cache) PutCoord(node *element.Node) {
keyBuf := make([]byte, 8)
bin.PutVarint(keyBuf, int64(node.Id))
data, err := binary.MarshalNode(node)
if err != nil {
panic(err)
}
p.db.Put(p.wo, keyBuf, data)
}
func (p *Cache) GetCoord(id element.OSMID) *element.Node {
keyBuf := make([]byte, 8)
bin.PutVarint(keyBuf, int64(id))
data, err := p.db.Get(p.ro, keyBuf)
if err != nil {
panic(err)
}
node, err := binary.UnmarshalNode(data)
if err != nil {
panic(err)
}
return node
}
func (p *Cache) Close() {
p.db.Close()
}

40
cache/db_test.go vendored Normal file
View File

@ -0,0 +1,40 @@
package cache
import (
"goposm/element"
"io/ioutil"
"os"
"testing"
)
func TestCreateCache(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache := NewCache(cache_dir)
defer cache.Close()
if stat, err := os.Stat(cache_dir); err != nil || !stat.IsDir() {
t.Error("cache dir not created")
}
}
func TestReadWriteNode(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache := NewCache(cache_dir)
node := &element.Node{}
node.Id = 1
cache.PutCoord(node)
cache.Close()
cache = NewCache(cache_dir)
defer cache.Close()
data := cache.GetCoord(element.OSMID(1))
if data.Id != 1 {
t.Errorf("unexpected result of GetNode(1): %v", data)
}
}

22
element/element.go
View File

@ -1,39 +1,41 @@
package element
type Tags map[string]string
type OSMID int64
type OSMElem struct {
Id OSMID
Id int64
Tags Tags
}
type Node struct {
OSMElem
Id int64
Tags Tags
Lat float64
Long float64
}
type Way struct {
OSMElem
Nodes []OSMID
Id int64
Tags Tags
Nodes []int64
}
type MemberType int
const (
NODE MemberType = iota
WAY MemberType = iota
RELATION MemberType = iota
NODE MemberType = iota
WAY
RELATION
)
type Member struct {
Id OSMID
Id int64
Type MemberType
Role string
}
type Relation struct {
OSMElem
Id int64
Tags Tags
Members []Member
}

46
model/model.pb.go
View File

@ -1,5 +1,5 @@
// Code generated by protoc-gen-go.
// source: model/model.proto
// source: model.proto
// DO NOT EDIT!
package model
@ -53,11 +53,11 @@ func (x *RelationMember_MemberType) UnmarshalJSON(data []byte) error {
}
type Node struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Long *uint32 `protobuf:"varint,2,req,name=long" json:"long,omitempty"`
Lat *uint32 `protobuf:"varint,3,req,name=lat" json:"lat,omitempty"`
Tags []*Tag `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"`
XXX_unrecognized []byte `json:"-"`
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Long *uint32 `protobuf:"varint,2,req,name=long" json:"long,omitempty"`
Lat *uint32 `protobuf:"varint,3,req,name=lat" json:"lat,omitempty"`
Tags []string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (this *Node) Reset() { *this = Node{} }
@ -85,35 +85,11 @@ func (this *Node) GetLat() uint32 {
return 0
}
type Tag struct {
Key *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"`
Val *string `protobuf:"bytes,2,req,name=val" json:"val,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (this *Tag) Reset() { *this = Tag{} }
func (this *Tag) String() string { return proto.CompactTextString(this) }
func (*Tag) ProtoMessage() {}
func (this *Tag) GetKey() string {
if this != nil && this.Key != nil {
return *this.Key
}
return ""
}
func (this *Tag) GetVal() string {
if this != nil && this.Val != nil {
return *this.Val
}
return ""
}
type Way struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Tags []*Tag `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
Nodes []int64 `protobuf:"varint,3,rep,packed,name=nodes" json:"nodes,omitempty"`
XXX_unrecognized []byte `json:"-"`
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Tags []string `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
Nodes []int64 `protobuf:"varint,3,rep,packed,name=nodes" json:"nodes,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (this *Way) Reset() { *this = Way{} }
@ -129,7 +105,7 @@ func (this *Way) GetId() int64 {
type Relation struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Tags []*Tag `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
Tags []string `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
Members []*RelationMember `protobuf:"bytes,3,rep,name=members" json:"members,omitempty"`
XXX_unrecognized []byte `json:"-"`
}

12
model/model.proto
View File

@ -4,26 +4,20 @@ message Node {
required int64 id = 1;
required uint32 long = 2;
required uint32 lat= 3;
repeated Tag tags = 4;
}
message Tag {
required string key = 1;
required string val = 2;
repeated string tags = 4;
}
message Way {
required int64 id = 1;
repeated Tag tags = 2;
repeated string tags = 2;
repeated int64 nodes = 3 [packed = true];
}
message Relation {
required int64 id = 1;
repeated Tag tags = 2;
repeated string tags = 2;
repeated RelationMember members = 3;
}

206
parser.go
View File

@ -1,214 +1,12 @@
package main
import (
"code.google.com/p/goprotobuf/proto"
structs "encoding/binary"
"fmt"
// "goposm/osmpbf/fileformat"
"bytes"
"compress/zlib"
"goposm/binary"
"goposm/element"
// "goposm/model"
"io"
"log"
"goposm/parser"
"os"
"osmpbf"
)
type PBF struct {
file *os.File
filename string
offset int64
}
type BlockPosition struct {
filename string
offset int64
size int32
}
func Open(filename string) (f *PBF, err error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
f = &PBF{filename: filename, file: file}
return f, nil
}
func (pbf *PBF) NextDataPosition() (offset int64, size int32) {
header := pbf.nextBlobHeader()
size = header.GetDatasize()
offset = pbf.offset
pbf.offset += int64(size)
pbf.file.Seek(pbf.offset, 0)
if header.GetType() == "OSMHeader" {
return pbf.NextDataPosition()
}
return
}
func (pbf *PBF) BlockPositions() (positions chan BlockPosition) {
positions = make(chan BlockPosition)
go func() {
for {
offset, size := pbf.NextDataPosition()
if size == 0 {
close(positions)
return
}
positions <- BlockPosition{pbf.filename, offset, size}
}
}()
return
}
func (pbf *PBF) nextBlobHeaderSize() (size int32) {
pbf.offset += 4
structs.Read(pbf.file, structs.BigEndian, &size)
return
}
func (pbf *PBF) nextBlobHeader() *osmpbf.BlobHeader {
var blobHeader = &osmpbf.BlobHeader{}
size := pbf.nextBlobHeaderSize()
if size == 0 {
return blobHeader
}
data := make([]byte, size)
io.ReadFull(pbf.file, data)
err := proto.Unmarshal(data, blobHeader)
if err != nil {
log.Fatal("unmarshaling error (header): ", err)
}
pbf.offset += int64(size)
return blobHeader
}
func ReadPrimitiveBlock(file *os.File, offset int64, size int32) *osmpbf.PrimitiveBlock {
var block = &osmpbf.PrimitiveBlock{}
var blob = &osmpbf.Blob{}
blobData := make([]byte, size)
file.Seek(offset, 0)
io.ReadFull(file, blobData)
err := proto.Unmarshal(blobData, blob)
if err != nil {
log.Fatal("unmarshaling error blob: ", err)
}
buf := bytes.NewBuffer(blob.GetZlibData())
r, err := zlib.NewReader(buf)
if err != nil {
log.Fatal("zlib error: ", err)
}
raw := make([]byte, blob.GetRawSize())
io.ReadFull(r, raw)
if err != nil {
log.Fatal("zlib read error: ", err)
}
err = proto.Unmarshal(raw, block)
if err != nil {
log.Fatal("unmarshaling error: ", err)
}
return block
}
func DenseNodeTags(stringtable []string, keyvals []int32) (tags map[string]string, nextPos int) {
tags = make(map[string]string)
nextPos = 0
for {
keyId := keyvals[nextPos]
nextPos += 1
if keyId == 0 {
return
}
key := stringtable[keyId]
valId := keyvals[nextPos]
nextPos += 1
val := stringtable[valId]
tags[key] = val
}
return
}
const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
func coordToInt(coord float64) uint32 {
return uint32((coord + 180.0) * COORD_FACTOR)
}
func intToCoord(coord uint32) float64 {
return float64((float64(coord) / COORD_FACTOR) - 180.0)
}
func ReadDenseNodes(dense *osmpbf.DenseNodes, block *osmpbf.PrimitiveBlock) (nodes []element.Node) {
var lastId int64
var lastLon, lastLat int64
nodes = make([]element.Node, len(dense.Id))
granularity := int64(block.GetGranularity())
latOffset := block.GetLatOffset()
lonOffset := block.GetLonOffset()
coordScale := 0.000000001
for i := range nodes {
lastId += dense.Id[i]
lastLon += dense.Lon[i]
lastLat += dense.Lat[i]
nodes[i].Id = element.OSMID(lastId)
nodes[i].Long = (coordScale * float64(lonOffset+(granularity*lastLon)))
nodes[i].Lat = (coordScale * float64(latOffset+(granularity*lastLat)))
}
return nodes
}
func blockPositions(filename string) {
pbf, err := Open(filename)
if err != nil {
log.Fatal(err)
}
var nodesCounter, relationsCounter, waysCounter int
for pos := range pbf.BlockPositions() {
file, err := os.Open(pos.filename)
if err != nil {
log.Fatal(err)
}
block := ReadPrimitiveBlock(file, pos.offset, pos.size)
for _, group := range block.Primitivegroup {
dense := group.GetDense()
if dense != nil {
nodes := ReadDenseNodes(dense, block)
lon, lat := nodes[0].Long, nodes[0].Lat
data, _ := binary.Marshal(nodes[0])
fmt.Printf("len: %d", len(data))
fmt.Printf("%v", data)
fmt.Printf("%12d %10.8f %10.8f\n", nodes[0].Id, lon, lat)
nodesCounter += len(dense.Id)
}
nodesCounter += len(group.Nodes)
waysCounter += len(group.Ways)
relationsCounter += len(group.Relations)
}
}
fmt.Printf("nodes: %v\tways: %v\trelations:%v\n", nodesCounter, waysCounter, relationsCounter)
}
func main() {
blockPositions(os.Args[1])
parser.BlockPositions(os.Args[1])
fmt.Println("done")
// osmpbf
}

87
parser/lowlevel.go Normal file
View File

@ -0,0 +1,87 @@
package parser
import (
"bytes"
"code.google.com/p/goprotobuf/proto"
"compress/zlib"
structs "encoding/binary"
"io"
"log"
"os"
"osmpbf"
)
func ReadPrimitiveBlock(pos BlockPosition) *osmpbf.PrimitiveBlock {
file, err := os.Open(pos.filename)
if err != nil {
log.Panic(err)
}
defer file.Close()
var block = &osmpbf.PrimitiveBlock{}
var blob = &osmpbf.Blob{}
blobData := make([]byte, pos.size)
file.Seek(pos.offset, 0)
io.ReadFull(file, blobData)
err = proto.Unmarshal(blobData, blob)
if err != nil {
log.Panic("unmarshaling error blob: ", err)
}
buf := bytes.NewBuffer(blob.GetZlibData())
r, err := zlib.NewReader(buf)
if err != nil {
log.Panic("zlib error: ", err)
}
raw := make([]byte, blob.GetRawSize())
io.ReadFull(r, raw)
err = proto.Unmarshal(raw, block)
if err != nil {
log.Panic("unmarshaling error: ", err)
}
return block
}
func (pbf *PBF) BlockPositions() (positions chan BlockPosition) {
positions = make(chan BlockPosition)
go func() {
for {
offset, size := pbf.NextDataPosition()
if size == 0 {
close(positions)
return
}
positions <- BlockPosition{pbf.filename, offset, size}
}
}()
return
}
func (pbf *PBF) nextBlobHeaderSize() (size int32) {
pbf.offset += 4
structs.Read(pbf.file, structs.BigEndian, &size)
return
}
func (pbf *PBF) nextBlobHeader() *osmpbf.BlobHeader {
var blobHeader = &osmpbf.BlobHeader{}
size := pbf.nextBlobHeaderSize()
if size == 0 {
return blobHeader
}
data := make([]byte, size)
io.ReadFull(pbf.file, data)
err := proto.Unmarshal(data, blobHeader)
if err != nil {
log.Fatal("unmarshaling error (header): ", err)
}
pbf.offset += int64(size)
return blobHeader
}

BIN
parser/parser.test Executable file
View File

Binary file not shown.

163
parser/pbf.go Normal file
View File

@ -0,0 +1,163 @@
package parser
import (
"fmt"
"goposm/binary"
"goposm/element"
"log"
"os"
"osmpbf"
)
type PBF struct {
file *os.File
filename string
offset int64
}
type BlockPosition struct {
filename string
offset int64
size int32
}
func Open(filename string) (f *PBF, err error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
}
f = &PBF{filename: filename, file: file}
return f, nil
}
func (pbf *PBF) NextDataPosition() (offset int64, size int32) {
header := pbf.nextBlobHeader()
size = header.GetDatasize()
offset = pbf.offset
pbf.offset += int64(size)
pbf.file.Seek(pbf.offset, 0)
if header.GetType() == "OSMHeader" {
return pbf.NextDataPosition()
}
return
}
func DenseNodeTags(stringtable []string, keyvals []int32) (tags map[string]string, nextPos int) {
tags = make(map[string]string)
nextPos = 0
for {
keyId := keyvals[nextPos]
nextPos += 1
if keyId == 0 {
return
}
key := stringtable[keyId]
valId := keyvals[nextPos]
nextPos += 1
val := stringtable[valId]
tags[key] = val
}
return
}
const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
func coordToInt(coord float64) uint32 {
return uint32((coord + 180.0) * COORD_FACTOR)
}
func intToCoord(coord uint32) float64 {
return float64((float64(coord) / COORD_FACTOR) - 180.0)
}
func ReadDenseNodes(
dense *osmpbf.DenseNodes,
block *osmpbf.PrimitiveBlock,
stringtable *StringTable) (nodes []element.Node) {
var lastId int64
var lastLon, lastLat int64
nodes = make([]element.Node, len(dense.Id))
granularity := int64(block.GetGranularity())
latOffset := block.GetLatOffset()
lonOffset := block.GetLonOffset()
coordScale := 0.000000001
lastKeyValPos := 0
for i := range nodes {
lastId += dense.Id[i]
lastLon += dense.Lon[i]
lastLat += dense.Lat[i]
nodes[i].Id = element.OSMID(lastId)
nodes[i].Long = (coordScale * float64(lonOffset+(granularity*lastLon)))
nodes[i].Lat = (coordScale * float64(latOffset+(granularity*lastLat)))
if dense.KeysVals[lastKeyValPos] != 0 {
nodes[i].Tags = ParseDenseNodeTags(stringtable, &dense.KeysVals, &lastKeyValPos)
} else {
lastKeyValPos += 1
}
}
return nodes
}
func ParseDenseNodeTags(stringtable *StringTable, keysVals *[]int32, pos *int) map[string]string {
result := make(map[string]string)
for {
if *pos >= len(*keysVals) {
return result
}
key := (*keysVals)[*pos]
*pos += 1
if key == 0 {
return result
}
val := (*keysVals)[*pos]
*pos += 1
result[(*stringtable)[key]] = (*stringtable)[val]
}
return result
}
type StringTable []string
func NewStringTable(source *osmpbf.StringTable) *StringTable {
result := make(StringTable, len(source.S))
for i, bytes := range source.S {
result[i] = string(bytes)
}
return &result
}
func BlockPositions(filename string) {
pbf, err := Open(filename)
if err != nil {
log.Fatal(err)
}
var nodesCounter, relationsCounter, waysCounter int
for pos := range pbf.BlockPositions() {
block := ReadPrimitiveBlock(pos)
stringtable := NewStringTable(block.GetStringtable())
for _, group := range block.Primitivegroup {
dense := group.GetDense()
if dense != nil {
nodes := ReadDenseNodes(dense, block, stringtable)
lon, lat := nodes[0].Long, nodes[0].Lat
data, _ := binary.Marshal(nodes[0])
fmt.Printf("len: %d", len(data))
fmt.Printf("%v", data)
fmt.Printf("%12d %10.8f %10.8f\n", nodes[0].Id, lon, lat)
nodesCounter += len(dense.Id)
}
nodesCounter += len(group.Nodes)
waysCounter += len(group.Ways)
relationsCounter += len(group.Relations)
}
}
fmt.Printf("nodes: %v\tways: %v\trelations:%v\n", nodesCounter, waysCounter, relationsCounter)
}

79
parser/pbf_test.go Normal file
View File

@ -0,0 +1,79 @@
package parser
import (
"bytes"
"code.google.com/p/goprotobuf/proto"
"compress/zlib"
"fmt"
"io"
"log"
"os"
"osmpbf"
"testing"
)
func BenchmarkHello(b *testing.B) {
b.StopTimer()
pbf, err := Open("../azores.osm.pbf")
if err != nil {
panic(err)
}
for pos := range pbf.BlockPositions() {
fmt.Println(pos.size, pos.offset)
b.StartTimer()
for i := 0; i < b.N; i++ {
ReadPrimitiveBlock(pos)
}
return
// for {
// stringtable := NewStringTable(block.GetStringtable())
// for _, group := range block.Primitivegroup {
// dense := group.GetDense()
// ReadDenseNodes(dense, block, stringtable)
// }
// }
// return
}
}
func BenchmarkPrimitiveBlock(b *testing.B) {
b.StopTimer()
file, err := os.Open("../azores.osm.pbf")
if err != nil {
log.Panic(err)
}
defer file.Close()
var block = &osmpbf.PrimitiveBlock{}
var blob = &osmpbf.Blob{}
var size = 56092
var offset int64 = 197
blobData := make([]byte, size)
file.Seek(offset, 0)
io.ReadFull(file, blobData)
err = proto.Unmarshal(blobData, blob)
if err != nil {
log.Panic("unmarshaling error blob: ", err)
}
b.StartTimer()
for i := 0; i < b.N; i++ {
buf := bytes.NewBuffer(blob.GetZlibData())
r, err := zlib.NewReader(buf)
if err != nil {
log.Panic("zlib error: ", err)
}
raw := make([]byte, blob.GetRawSize())
io.ReadFull(r, raw)
err = proto.Unmarshal(raw, block)
if err != nil {
log.Panic("unmarshaling error: ", err)
}
}
}