2013-02-12 22:45:49 +04:00
|
|
|
package parser
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"goposm/element"
|
2013-05-06 18:49:48 +04:00
|
|
|
"goposm/parser/osmpbf"
|
2013-02-12 22:45:49 +04:00
|
|
|
"log"
|
|
|
|
"os"
|
|
|
|
)
|
|
|
|
|
|
|
|
type PBF struct {
|
|
|
|
file *os.File
|
|
|
|
filename string
|
|
|
|
offset int64
|
|
|
|
}
|
|
|
|
|
|
|
|
type BlockPosition struct {
|
2013-04-27 18:02:16 +04:00
|
|
|
Filename string
|
|
|
|
Offset int64
|
|
|
|
Size int32
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func Open(filename string) (f *PBF, err error) {
|
|
|
|
file, err := os.Open(filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
f = &PBF{filename: filename, file: file}
|
|
|
|
return f, nil
|
|
|
|
}
|
|
|
|
|
2013-04-03 00:07:27 +04:00
|
|
|
func (pbf *PBF) Close() error {
|
|
|
|
return pbf.file.Close()
|
|
|
|
}
|
|
|
|
|
2013-02-12 22:45:49 +04:00
|
|
|
func (pbf *PBF) NextDataPosition() (offset int64, size int32) {
|
|
|
|
header := pbf.nextBlobHeader()
|
|
|
|
size = header.GetDatasize()
|
|
|
|
offset = pbf.offset
|
|
|
|
|
|
|
|
pbf.offset += int64(size)
|
|
|
|
pbf.file.Seek(pbf.offset, 0)
|
|
|
|
|
|
|
|
if header.GetType() == "OSMHeader" {
|
|
|
|
return pbf.NextDataPosition()
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
const COORD_FACTOR float64 = 11930464.7083 // ((2<<31)-1)/360.0
|
|
|
|
|
|
|
|
func coordToInt(coord float64) uint32 {
|
|
|
|
return uint32((coord + 180.0) * COORD_FACTOR)
|
|
|
|
}
|
|
|
|
|
|
|
|
func intToCoord(coord uint32) float64 {
|
|
|
|
return float64((float64(coord) / COORD_FACTOR) - 180.0)
|
|
|
|
}
|
|
|
|
|
|
|
|
func ReadDenseNodes(
|
|
|
|
dense *osmpbf.DenseNodes,
|
|
|
|
block *osmpbf.PrimitiveBlock,
|
2013-05-21 10:52:14 +04:00
|
|
|
stringtable StringTable) (coords []element.Node, nodes []element.Node) {
|
2013-02-12 22:45:49 +04:00
|
|
|
|
|
|
|
var lastId int64
|
|
|
|
var lastLon, lastLat int64
|
2013-05-21 10:52:14 +04:00
|
|
|
coords = make([]element.Node, len(dense.Id))
|
|
|
|
nodes = make([]element.Node, 0, len(dense.Id)/8)
|
2013-02-12 22:45:49 +04:00
|
|
|
granularity := int64(block.GetGranularity())
|
|
|
|
latOffset := block.GetLatOffset()
|
|
|
|
lonOffset := block.GetLonOffset()
|
|
|
|
coordScale := 0.000000001
|
|
|
|
lastKeyValPos := 0
|
|
|
|
|
2013-05-21 10:52:14 +04:00
|
|
|
for i := range coords {
|
2013-02-12 22:45:49 +04:00
|
|
|
lastId += dense.Id[i]
|
|
|
|
lastLon += dense.Lon[i]
|
|
|
|
lastLat += dense.Lat[i]
|
2013-05-21 10:52:14 +04:00
|
|
|
coords[i].Id = lastId
|
|
|
|
coords[i].Long = (coordScale * float64(lonOffset+(granularity*lastLon)))
|
|
|
|
coords[i].Lat = (coordScale * float64(latOffset+(granularity*lastLat)))
|
2013-05-06 12:50:55 +04:00
|
|
|
if stringtable != nil && len(dense.KeysVals) > 0 {
|
2013-05-02 20:45:33 +04:00
|
|
|
if dense.KeysVals[lastKeyValPos] != 0 {
|
2013-05-21 10:52:14 +04:00
|
|
|
tags := ParseDenseNodeTags(stringtable, &dense.KeysVals, &lastKeyValPos)
|
|
|
|
if tags != nil {
|
|
|
|
if _, ok := tags["created_by"]; ok && len(tags) == 1 {
|
|
|
|
// don't add nodes with only created_by tag to nodes cache
|
|
|
|
} else {
|
|
|
|
nd := coords[i]
|
|
|
|
nd.Tags = tags
|
|
|
|
nodes = append(nodes, nd)
|
|
|
|
}
|
|
|
|
}
|
2013-05-02 20:45:33 +04:00
|
|
|
} else {
|
|
|
|
lastKeyValPos += 1
|
|
|
|
}
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
}
|
2013-05-21 10:52:14 +04:00
|
|
|
|
|
|
|
return coords, nodes
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
|
2013-04-03 00:07:27 +04:00
|
|
|
func ParseDenseNodeTags(stringtable StringTable, keysVals *[]int32, pos *int) map[string]string {
|
2013-02-12 22:45:49 +04:00
|
|
|
result := make(map[string]string)
|
|
|
|
for {
|
|
|
|
if *pos >= len(*keysVals) {
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
key := (*keysVals)[*pos]
|
|
|
|
*pos += 1
|
|
|
|
if key == 0 {
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
val := (*keysVals)[*pos]
|
|
|
|
*pos += 1
|
2013-04-03 00:07:27 +04:00
|
|
|
result[stringtable[key]] = stringtable[val]
|
|
|
|
}
|
|
|
|
}
|
2013-05-13 12:55:08 +04:00
|
|
|
|
2013-04-03 00:07:27 +04:00
|
|
|
func ParseTags(stringtable StringTable, keys []uint32, vals []uint32) map[string]string {
|
|
|
|
tags := make(map[string]string)
|
|
|
|
for i := 0; i < len(keys); i++ {
|
|
|
|
key := stringtable[keys[i]]
|
|
|
|
val := stringtable[vals[i]]
|
|
|
|
tags[key] = val
|
|
|
|
}
|
|
|
|
return tags
|
|
|
|
}
|
|
|
|
|
|
|
|
func ReadNodes(
|
|
|
|
nodes []*osmpbf.Node,
|
|
|
|
block *osmpbf.PrimitiveBlock,
|
2013-05-21 10:52:14 +04:00
|
|
|
stringtable StringTable) ([]element.Node, []element.Node) {
|
2013-04-03 00:07:27 +04:00
|
|
|
|
2013-05-21 10:52:14 +04:00
|
|
|
coords := make([]element.Node, len(nodes))
|
|
|
|
nds := make([]element.Node, 0, len(nodes)/8)
|
2013-04-03 00:07:27 +04:00
|
|
|
granularity := int64(block.GetGranularity())
|
|
|
|
latOffset := block.GetLatOffset()
|
|
|
|
lonOffset := block.GetLonOffset()
|
|
|
|
coordScale := 0.000000001
|
|
|
|
|
|
|
|
for i := range nodes {
|
|
|
|
id := *nodes[i].Id
|
|
|
|
lon := *nodes[i].Lon
|
|
|
|
lat := *nodes[i].Lat
|
2013-05-21 10:52:14 +04:00
|
|
|
coords[i].Id = id
|
|
|
|
coords[i].Long = (coordScale * float64(lonOffset+(granularity*lon)))
|
|
|
|
coords[i].Lat = (coordScale * float64(latOffset+(granularity*lat)))
|
2013-05-02 20:45:33 +04:00
|
|
|
if stringtable != nil {
|
2013-05-21 10:52:14 +04:00
|
|
|
tags := ParseTags(stringtable, nodes[i].Keys, nodes[i].Vals)
|
|
|
|
if tags != nil {
|
|
|
|
if _, ok := tags["created_by"]; ok && len(tags) == 1 {
|
|
|
|
// don't add nodes with only created_by tag to nodes cache
|
|
|
|
} else {
|
|
|
|
nd := coords[i]
|
|
|
|
nd.Tags = tags
|
|
|
|
nds = append(nds, nd)
|
|
|
|
}
|
|
|
|
}
|
2013-05-02 20:45:33 +04:00
|
|
|
}
|
2013-04-03 00:07:27 +04:00
|
|
|
}
|
2013-05-21 10:52:14 +04:00
|
|
|
return coords, nds
|
2013-04-03 00:07:27 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func ParseDeltaRefs(refs []int64) []int64 {
|
|
|
|
result := make([]int64, len(refs))
|
|
|
|
var lastRef int64
|
|
|
|
|
|
|
|
for i, refDelta := range refs {
|
|
|
|
lastRef += refDelta
|
|
|
|
result[i] = lastRef
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
|
|
|
func ReadWays(
|
|
|
|
ways []*osmpbf.Way,
|
|
|
|
block *osmpbf.PrimitiveBlock,
|
|
|
|
stringtable StringTable) []element.Way {
|
|
|
|
|
|
|
|
result := make([]element.Way, len(ways))
|
|
|
|
|
|
|
|
for i := range ways {
|
|
|
|
id := *ways[i].Id
|
|
|
|
result[i].Id = id
|
|
|
|
result[i].Tags = ParseTags(stringtable, ways[i].Keys, ways[i].Vals)
|
2013-04-24 00:30:41 +04:00
|
|
|
result[i].Refs = ParseDeltaRefs(ways[i].Refs)
|
2013-04-03 00:07:27 +04:00
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
|
|
|
func ParseRelationMembers(rel *osmpbf.Relation, stringtable StringTable) []element.Member {
|
|
|
|
result := make([]element.Member, len(rel.Memids))
|
|
|
|
|
|
|
|
var lastId int64
|
|
|
|
for i := range rel.Memids {
|
|
|
|
lastId += rel.Memids[i]
|
|
|
|
result[i].Id = lastId
|
|
|
|
result[i].Role = stringtable[rel.RolesSid[i]]
|
|
|
|
result[i].Type = element.MemberType(rel.Types[i])
|
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
|
|
|
func ReadRelations(
|
|
|
|
relations []*osmpbf.Relation,
|
|
|
|
block *osmpbf.PrimitiveBlock,
|
|
|
|
stringtable StringTable) []element.Relation {
|
|
|
|
|
|
|
|
result := make([]element.Relation, len(relations))
|
|
|
|
|
|
|
|
for i := range relations {
|
|
|
|
id := *relations[i].Id
|
|
|
|
result[i].Id = id
|
|
|
|
result[i].Tags = ParseTags(stringtable, relations[i].Keys, relations[i].Vals)
|
|
|
|
result[i].Members = ParseRelationMembers(relations[i], stringtable)
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
|
|
|
type StringTable []string
|
|
|
|
|
2013-04-03 00:07:27 +04:00
|
|
|
func NewStringTable(source *osmpbf.StringTable) StringTable {
|
2013-02-12 22:45:49 +04:00
|
|
|
result := make(StringTable, len(source.S))
|
|
|
|
for i, bytes := range source.S {
|
|
|
|
result[i] = string(bytes)
|
|
|
|
}
|
2013-04-03 00:07:27 +04:00
|
|
|
return result
|
|
|
|
}
|
|
|
|
|
|
|
|
func PBFBlockPositions(filename string) chan BlockPosition {
|
|
|
|
pbf, err := Open(filename)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return pbf.BlockPositions()
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
|
2013-05-02 20:45:33 +04:00
|
|
|
func ParseBlock(pos BlockPosition, coords chan []element.Node, nodes chan []element.Node, ways chan []element.Way, relations chan []element.Relation) {
|
2013-04-08 23:45:13 +04:00
|
|
|
block := ReadPrimitiveBlock(pos)
|
|
|
|
stringtable := NewStringTable(block.GetStringtable())
|
|
|
|
|
|
|
|
for _, group := range block.Primitivegroup {
|
|
|
|
dense := group.GetDense()
|
|
|
|
if dense != nil {
|
2013-05-21 10:52:14 +04:00
|
|
|
parsedCoords, parsedNodes := ReadDenseNodes(dense, block, stringtable)
|
|
|
|
if len(parsedCoords) > 0 {
|
|
|
|
coords <- parsedCoords
|
|
|
|
}
|
2013-04-20 18:50:23 +04:00
|
|
|
if len(parsedNodes) > 0 {
|
|
|
|
nodes <- parsedNodes
|
|
|
|
}
|
2013-04-08 23:45:13 +04:00
|
|
|
}
|
2013-05-21 10:52:14 +04:00
|
|
|
parsedCoords, parsedNodes := ReadNodes(group.Nodes, block, stringtable)
|
|
|
|
if len(parsedCoords) > 0 {
|
|
|
|
coords <- parsedCoords
|
|
|
|
}
|
2013-04-20 18:50:23 +04:00
|
|
|
if len(parsedNodes) > 0 {
|
|
|
|
nodes <- parsedNodes
|
|
|
|
}
|
2013-04-08 23:45:13 +04:00
|
|
|
parsedWays := ReadWays(group.Ways, block, stringtable)
|
2013-04-20 18:50:23 +04:00
|
|
|
if len(parsedWays) > 0 {
|
|
|
|
ways <- parsedWays
|
|
|
|
}
|
2013-04-08 23:45:13 +04:00
|
|
|
parsedRelations := ReadRelations(group.Relations, block, stringtable)
|
2013-04-20 18:50:23 +04:00
|
|
|
if len(parsedRelations) > 0 {
|
|
|
|
relations <- parsedRelations
|
2013-04-08 23:45:13 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2013-04-03 00:07:27 +04:00
|
|
|
func PBFStats(filename string) {
|
2013-02-12 22:45:49 +04:00
|
|
|
pbf, err := Open(filename)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
var nodesCounter, relationsCounter, waysCounter int
|
|
|
|
|
|
|
|
for pos := range pbf.BlockPositions() {
|
|
|
|
block := ReadPrimitiveBlock(pos)
|
|
|
|
stringtable := NewStringTable(block.GetStringtable())
|
|
|
|
|
|
|
|
for _, group := range block.Primitivegroup {
|
|
|
|
dense := group.GetDense()
|
|
|
|
if dense != nil {
|
2013-05-21 10:52:14 +04:00
|
|
|
_, _ = ReadDenseNodes(dense, block, stringtable)
|
2013-02-12 22:45:49 +04:00
|
|
|
nodesCounter += len(dense.Id)
|
|
|
|
}
|
2013-05-21 10:52:14 +04:00
|
|
|
_, _ = ReadNodes(group.Nodes, block, stringtable)
|
2013-02-12 22:45:49 +04:00
|
|
|
nodesCounter += len(group.Nodes)
|
|
|
|
waysCounter += len(group.Ways)
|
2013-04-03 00:07:27 +04:00
|
|
|
_ = ReadWays(group.Ways, block, stringtable)
|
2013-02-12 22:45:49 +04:00
|
|
|
relationsCounter += len(group.Relations)
|
2013-04-03 00:07:27 +04:00
|
|
|
_ = ReadRelations(group.Relations, block, stringtable)
|
|
|
|
|
2013-02-12 22:45:49 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
fmt.Printf("nodes: %v\tways: %v\trelations:%v\n", nodesCounter, waysCounter, relationsCounter)
|
|
|
|
}
|