197 lines
4.1 KiB
Go
197 lines
4.1 KiB
Go
package pbf
|
|
|
|
import (
|
|
"bytes"
|
|
"code.google.com/p/goprotobuf/proto"
|
|
"compress/zlib"
|
|
structs "encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/omniscale/imposm3/parser/pbf/osmpbf"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
)
|
|
|
|
type parserError struct {
|
|
message string
|
|
originalError error
|
|
}
|
|
|
|
func (e *parserError) Error() string {
|
|
return fmt.Sprintf("%s: %v", e.message, e.originalError)
|
|
}
|
|
|
|
func newParserError(message string, err error) *parserError {
|
|
return &parserError{message, err}
|
|
}
|
|
|
|
var supportedFeatured = map[string]bool{"OsmSchema-V0.6": true, "DenseNodes": true}
|
|
|
|
func readBlobData(pos Block) ([]byte, error) {
|
|
file, err := os.Open(pos.filename)
|
|
if err != nil {
|
|
return nil, newParserError("file open", err)
|
|
}
|
|
defer file.Close()
|
|
|
|
var blob = &osmpbf.Blob{}
|
|
|
|
blobData := make([]byte, pos.size)
|
|
file.Seek(pos.offset, 0)
|
|
io.ReadFull(file, blobData)
|
|
err = proto.Unmarshal(blobData, blob)
|
|
if err != nil {
|
|
return nil, newParserError("unmarshaling blob", err)
|
|
}
|
|
|
|
// pbf contains (uncompressed) raw or zlibdata
|
|
raw := blob.GetRaw()
|
|
if raw == nil {
|
|
buf := bytes.NewBuffer(blob.GetZlibData())
|
|
r, err := zlib.NewReader(buf)
|
|
if err != nil {
|
|
return nil, newParserError("zlib error", err)
|
|
}
|
|
raw = make([]byte, blob.GetRawSize())
|
|
_, err = io.ReadFull(r, raw)
|
|
if err != nil {
|
|
return nil, newParserError("zlib read error", err)
|
|
}
|
|
}
|
|
return raw, nil
|
|
}
|
|
|
|
func readPrimitiveBlock(pos Block) *osmpbf.PrimitiveBlock {
|
|
raw, err := readBlobData(pos)
|
|
if err != nil {
|
|
log.Panic(err)
|
|
}
|
|
block := &osmpbf.PrimitiveBlock{}
|
|
err = proto.Unmarshal(raw, block)
|
|
if err != nil {
|
|
log.Panic("unmarshaling error: ", err)
|
|
}
|
|
|
|
return block
|
|
}
|
|
|
|
func readAndParseHeaderBlock(pos Block) (*pbfHeader, error) {
|
|
raw, err := readBlobData(pos)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
header := &osmpbf.HeaderBlock{}
|
|
err = proto.Unmarshal(raw, header)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, feature := range header.RequiredFeatures {
|
|
if supportedFeatured[feature] != true {
|
|
return nil, errors.New("cannot parse file, feature " + feature + " not supported")
|
|
}
|
|
}
|
|
|
|
result := &pbfHeader{}
|
|
timestamp := header.GetOsmosisReplicationTimestamp()
|
|
result.Time = time.Unix(timestamp, 0 /* nanoseconds */)
|
|
return result, nil
|
|
}
|
|
|
|
type Pbf struct {
|
|
file *os.File
|
|
Filename string
|
|
offset int64
|
|
Header *pbfHeader
|
|
}
|
|
|
|
type pbfHeader struct {
|
|
Time time.Time
|
|
}
|
|
|
|
func Open(filename string) (f *Pbf, err error) {
|
|
file, err := os.Open(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
f = &Pbf{Filename: filename, file: file}
|
|
err = f.parseHeader()
|
|
if err != nil {
|
|
file.Close()
|
|
return nil, err
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
func (pbf *Pbf) Close() error {
|
|
return pbf.file.Close()
|
|
}
|
|
|
|
func (pbf *Pbf) parseHeader() error {
|
|
offset, size, header := pbf.nextBlock()
|
|
if header.GetType() != "OSMHeader" {
|
|
panic("invalid block type, expected OSMHeader, got " + header.GetType())
|
|
}
|
|
var err error
|
|
pbf.Header, err = readAndParseHeaderBlock(Block{pbf.Filename, offset, size})
|
|
return err
|
|
}
|
|
|
|
func (pbf *Pbf) nextBlock() (offset int64, size int32, header *osmpbf.BlobHeader) {
|
|
header = pbf.nextBlobHeader()
|
|
size = header.GetDatasize()
|
|
offset = pbf.offset
|
|
|
|
pbf.offset += int64(size)
|
|
pbf.file.Seek(pbf.offset, 0)
|
|
return offset, size, header
|
|
}
|
|
|
|
func (pbf *Pbf) BlockPositions() (positions chan Block) {
|
|
positions = make(chan Block, 8)
|
|
go func() {
|
|
for {
|
|
offset, size, header := pbf.nextBlock()
|
|
if size == 0 {
|
|
close(positions)
|
|
pbf.Close()
|
|
return
|
|
}
|
|
if header.GetType() != "OSMData" {
|
|
panic("invalid block type, expected OSMData, got " + header.GetType())
|
|
}
|
|
positions <- Block{pbf.Filename, offset, size}
|
|
}
|
|
}()
|
|
return
|
|
}
|
|
|
|
func (pbf *Pbf) nextBlobHeaderSize() (size int32) {
|
|
pbf.offset += 4
|
|
structs.Read(pbf.file, structs.BigEndian, &size)
|
|
return
|
|
}
|
|
|
|
func (pbf *Pbf) nextBlobHeader() *osmpbf.BlobHeader {
|
|
var blobHeader = &osmpbf.BlobHeader{}
|
|
|
|
size := pbf.nextBlobHeaderSize()
|
|
if size == 0 {
|
|
return blobHeader
|
|
}
|
|
|
|
data := make([]byte, size)
|
|
io.ReadFull(pbf.file, data)
|
|
|
|
err := proto.Unmarshal(data, blobHeader)
|
|
if err != nil {
|
|
log.Fatal("unmarshaling error (header): ", err)
|
|
}
|
|
|
|
pbf.offset += int64(size)
|
|
return blobHeader
|
|
}
|