vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

do not store Ids in cache value, already stored as the key

master
Oliver Tonnhofer 2013-06-05 07:47:28 +02:00
parent 384e3060af
commit f31323f975
8 changed files with 45 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cache/binary/messages.pb.go vendored
View File

@ -53,10 +53,9 @@ func (x *Relation_MemberType) UnmarshalJSON(data []byte) error {
}
type Node struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Long *uint32 `protobuf:"varint,2,req,name=long" json:"long,omitempty"`
Lat *uint32 `protobuf:"varint,3,req,name=lat" json:"lat,omitempty"`
Tags []string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"`
Long *uint32 `protobuf:"varint,1,req,name=long" json:"long,omitempty"`
Lat *uint32 `protobuf:"varint,2,req,name=lat" json:"lat,omitempty"`
Tags []string `protobuf:"bytes,3,rep,name=tags" json:"tags,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -64,13 +63,6 @@ func (m *Node) Reset() { *m = Node{} }
func (m *Node) String() string { return proto.CompactTextString(m) }
func (*Node) ProtoMessage() {}
func (m *Node) GetId() int64 {
if m != nil && m.Id != nil {
return *m.Id
}
return 0
}
func (m *Node) GetLong() uint32 {
if m != nil && m.Long != nil {
return *m.Long
@ -93,9 +85,8 @@ func (m *Node) GetTags() []string {
}
type Way struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Tags []string `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
Refs []int64 `protobuf:"varint,3,rep,packed,name=refs" json:"refs,omitempty"`
Tags []string `protobuf:"bytes,1,rep,name=tags" json:"tags,omitempty"`
Refs []int64 `protobuf:"varint,2,rep,packed,name=refs" json:"refs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -103,13 +94,6 @@ func (m *Way) Reset() { *m = Way{} }
func (m *Way) String() string { return proto.CompactTextString(m) }
func (*Way) ProtoMessage() {}
func (m *Way) GetId() int64 {
if m != nil && m.Id != nil {
return *m.Id
}
return 0
}
func (m *Way) GetTags() []string {
if m != nil {
return m.Tags
@ -125,11 +109,10 @@ func (m *Way) GetRefs() []int64 {
}
type Relation struct {
Id *int64 `protobuf:"varint,1,req,name=id" json:"id,omitempty"`
Tags []string `protobuf:"bytes,2,rep,name=tags" json:"tags,omitempty"`
MemberIds []int64 `protobuf:"varint,3,rep,name=member_ids" json:"member_ids,omitempty"`
MemberTypes []Relation_MemberType `protobuf:"varint,4,rep,name=member_types,enum=binary.Relation_MemberType" json:"member_types,omitempty"`
MemberRoles []string `protobuf:"bytes,5,rep,name=member_roles" json:"member_roles,omitempty"`
Tags []string `protobuf:"bytes,1,rep,name=tags" json:"tags,omitempty"`
MemberIds []int64 `protobuf:"varint,2,rep,name=member_ids" json:"member_ids,omitempty"`
MemberTypes []Relation_MemberType `protobuf:"varint,3,rep,name=member_types,enum=binary.Relation_MemberType" json:"member_types,omitempty"`
MemberRoles []string `protobuf:"bytes,4,rep,name=member_roles" json:"member_roles,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -137,13 +120,6 @@ func (m *Relation) Reset() { *m = Relation{} }
func (m *Relation) String() string { return proto.CompactTextString(m) }
func (*Relation) ProtoMessage() {}
func (m *Relation) GetId() int64 {
if m != nil && m.Id != nil {
return *m.Id
}
return 0
}
func (m *Relation) GetTags() []string {
if m != nil {
return m.Tags

23
cache/binary/messages.proto vendored
View File

@ -1,31 +1,26 @@
package binary;
message Node {
required int64 id = 1;
required uint32 long = 2;
required uint32 lat= 3;
repeated string tags = 4;
required uint32 long = 1;
required uint32 lat= 2;
repeated string tags = 3;
}
message Way {
required int64 id = 1;
repeated string tags = 2;
repeated int64 refs = 3 [packed = true];
repeated string tags = 1;
repeated int64 refs = 2 [packed = true];
}
message Relation {
required int64 id = 1;
repeated string tags = 2;
repeated int64 member_ids = 3;
repeated string tags = 1;
repeated int64 member_ids = 2;
enum MemberType {
NODE = 0;
WAY = 1;
RELATION = 2;
}
repeated MemberType member_types = 4;
repeated string member_roles = 5;
repeated MemberType member_types = 3;
repeated string member_roles = 4;
}
message DeltaCoords {

8
cache/binary/serialize.go vendored
View File

@ -62,8 +62,6 @@ func UnmarshalCoord(id int64, data []byte) (node *element.Node, err error) {
func MarshalNode(node *element.Node) ([]byte, error) {
pbfNode := &Node{}
nodeId := node.Id
pbfNode.Id = &nodeId
pbfNode.FromWgsCoord(node.Long, node.Lat)
pbfNode.Tags = node.TagsAsArray()
return proto.Marshal(pbfNode)
@ -77,7 +75,6 @@ func UnmarshalNode(data []byte) (node *element.Node, err error) {
}
node = &element.Node{}
node.Id = *pbfNode.Id
node.Long, node.Lat = pbfNode.WgsCoord()
node.TagsFromArray(pbfNode.Tags)
return node, nil
@ -105,7 +102,6 @@ func deltaUnpack(data []int64) {
func MarshalWay(way *element.Way) ([]byte, error) {
// TODO reuse Way to avoid make(Tags) for each way in TagsAsArray
pbfWay := &Way{}
pbfWay.Id = &way.Id
deltaPack(way.Refs)
pbfWay.Refs = way.Refs
pbfWay.Tags = way.TagsAsArray()
@ -120,7 +116,6 @@ func UnmarshalWay(data []byte) (way *element.Way, err error) {
}
way = &element.Way{}
way.Id = *pbfWay.Id
deltaUnpack(pbfWay.Refs)
way.Refs = pbfWay.Refs
way.TagsFromArray(pbfWay.Tags)
@ -129,8 +124,6 @@ func UnmarshalWay(data []byte) (way *element.Way, err error) {
func MarshalRelation(relation *element.Relation) ([]byte, error) {
pbfRelation := &Relation{}
pbfRelation.Id = &relation.Id
// TODO store members
pbfRelation.MemberIds = make([]int64, len(relation.Members))
pbfRelation.MemberTypes = make([]Relation_MemberType, len(relation.Members))
pbfRelation.MemberRoles = make([]string, len(relation.Members))
@ -151,7 +144,6 @@ func UnmarshalRelation(data []byte) (relation *element.Relation, err error) {
}
relation = &element.Relation{}
relation.Id = *pbfRelation.Id
relation.Members = make([]element.Member, len(pbfRelation.MemberIds))
for i, _ := range pbfRelation.MemberIds {
relation.Members[i].Id = pbfRelation.MemberIds[i]

13
cache/nodes.go vendored
View File

@ -64,11 +64,12 @@ func (p *NodesCache) GetNode(id int64) (*element.Node, error) {
if err != nil {
return nil, err
}
node.Id = id
return node, nil
}
func (p *NodesCache) Iter() chan *element.Node {
node := make(chan *element.Node)
nodes := make(chan *element.Node)
go func() {
ro := levigo.NewReadOptions()
ro.SetFillCache(false)
@ -76,13 +77,15 @@ func (p *NodesCache) Iter() chan *element.Node {
defer it.Close()
it.SeekToFirst()
for ; it.Valid(); it.Next() {
nodes, err := binary.UnmarshalNode(it.Value())
node, err := binary.UnmarshalNode(it.Value())
if err != nil {
panic(err)
}
node <- nodes
node.Id = idFromKeyBuf(it.Key())
nodes <- node
}
close(node)
close(nodes)
}()
return node
return nodes
}

4
cache/osm.go vendored
View File

@ -183,6 +183,10 @@ func idToKeyBuf(id int64) []byte {
return b[:8]
}
func idFromKeyBuf(buf []byte) int64 {
return int64(bin.BigEndian.Uint64(buf))
}
func (p *Cache) Close() {
p.db.Close()
}

BIN
cache/query/query vendored Executable file
View File

Binary file not shown.

13
cache/relations.go vendored
View File

@ -47,7 +47,7 @@ func (p *RelationsCache) PutRelations(rels []element.Relation) error {
}
func (p *RelationsCache) Iter() chan *element.Relation {
rel := make(chan *element.Relation)
rels := make(chan *element.Relation)
go func() {
ro := levigo.NewReadOptions()
ro.SetFillCache(false)
@ -55,15 +55,17 @@ func (p *RelationsCache) Iter() chan *element.Relation {
defer it.Close()
it.SeekToFirst()
for ; it.Valid(); it.Next() {
relation, err := binary.UnmarshalRelation(it.Value())
rel, err := binary.UnmarshalRelation(it.Value())
if err != nil {
panic(err)
}
rel <- relation
rel.Id = idFromKeyBuf(it.Key())
rels <- rel
}
close(rel)
close(rels)
}()
return rel
return rels
}
func (p *RelationsCache) GetRelation(id int64) (*element.Relation, error) {
@ -79,5 +81,6 @@ func (p *RelationsCache) GetRelation(id int64) (*element.Relation, error) {
if err != nil {
return nil, err
}
relation.Id = id
return relation, err
}

36
cache/ways.go vendored
View File

@ -8,13 +8,10 @@ import (
type WaysCache struct {
Cache
toWrite chan []element.Way
}
func NewWaysCache(path string) (*WaysCache, error) {
cache := WaysCache{}
cache.toWrite = make(chan []element.Way)
go cache.wayWriter()
err := cache.open(path)
if err != nil {
return nil, err
@ -46,27 +43,6 @@ func (p *WaysCache) PutWays(ways []element.Way) error {
return p.db.Write(p.wo, batch)
}
func (p *WaysCache) _PutWays(ways []element.Way) {
p.toWrite <- ways
}
func (p *WaysCache) wayWriter() {
for ways := range p.toWrite {
batch := levigo.NewWriteBatch()
defer batch.Close()
for _, way := range ways {
keyBuf := idToKeyBuf(way.Id)
data, err := binary.MarshalWay(&way)
if err != nil {
panic(err)
}
batch.Put(keyBuf, data)
}
_ = p.db.Write(p.wo, batch)
}
}
func (p *WaysCache) GetWay(id int64) (*element.Way, error) {
keyBuf := idToKeyBuf(id)
data, err := p.db.Get(p.ro, keyBuf)
@ -80,11 +56,12 @@ func (p *WaysCache) GetWay(id int64) (*element.Way, error) {
if err != nil {
return nil, err
}
way.Id = id
return way, nil
}
func (p *WaysCache) Iter() chan *element.Way {
way := make(chan *element.Way, 1024)
ways := make(chan *element.Way, 1024)
go func() {
ro := levigo.NewReadOptions()
ro.SetFillCache(false)
@ -92,15 +69,16 @@ func (p *WaysCache) Iter() chan *element.Way {
defer it.Close()
it.SeekToFirst()
for ; it.Valid(); it.Next() {
ways, err := binary.UnmarshalWay(it.Value())
way, err := binary.UnmarshalWay(it.Value())
if err != nil {
panic(err)
}
way <- ways
way.Id = idFromKeyBuf(it.Key())
ways <- way
}
close(way)
close(ways)
}()
return way
return ways
}
func (self *WaysCache) FillMembers(members []element.Member) error {