2013-04-11 21:50:20 +04:00
|
|
|
package cache
|
|
|
|
|
|
|
|
import (
|
2013-04-14 16:50:38 +04:00
|
|
|
"container/list"
|
2013-05-17 17:10:59 +04:00
|
|
|
"goposm/cache/binary"
|
2013-04-11 21:50:20 +04:00
|
|
|
"goposm/element"
|
2013-05-05 22:14:39 +04:00
|
|
|
"sort"
|
2013-04-14 16:50:38 +04:00
|
|
|
"sync"
|
2013-04-11 21:50:20 +04:00
|
|
|
)
|
|
|
|
|
2013-05-05 22:14:39 +04:00
|
|
|
type Nodes []element.Node
|
|
|
|
|
|
|
|
func (s Nodes) Len() int { return len(s) }
|
|
|
|
func (s Nodes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
|
|
|
func (s Nodes) Less(i, j int) bool { return s[i].Id < s[j].Id }
|
|
|
|
|
2013-05-17 17:16:24 +04:00
|
|
|
func packNodes(nodes []element.Node) *binary.DeltaCoords {
|
2013-04-11 21:50:20 +04:00
|
|
|
var lastLon, lastLat int64
|
|
|
|
var lon, lat int64
|
2013-04-15 23:54:48 +04:00
|
|
|
var lastId int64
|
2013-04-11 21:50:20 +04:00
|
|
|
ids := make([]int64, len(nodes))
|
|
|
|
lons := make([]int64, len(nodes))
|
|
|
|
lats := make([]int64, len(nodes))
|
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
i := 0
|
2013-05-05 22:14:39 +04:00
|
|
|
for _, nd := range nodes {
|
2013-04-11 21:50:20 +04:00
|
|
|
lon = int64(binary.CoordToInt(nd.Long))
|
|
|
|
lat = int64(binary.CoordToInt(nd.Lat))
|
2013-05-05 22:14:39 +04:00
|
|
|
ids[i] = nd.Id - lastId
|
2013-04-11 21:50:20 +04:00
|
|
|
lons[i] = lon - lastLon
|
|
|
|
lats[i] = lat - lastLat
|
|
|
|
|
2013-05-05 22:14:39 +04:00
|
|
|
lastId = nd.Id
|
2013-04-11 21:50:20 +04:00
|
|
|
lastLon = lon
|
|
|
|
lastLat = lat
|
2013-04-15 23:54:48 +04:00
|
|
|
i++
|
2013-04-11 21:50:20 +04:00
|
|
|
}
|
2013-05-17 17:16:24 +04:00
|
|
|
return &binary.DeltaCoords{Ids: ids, Lats: lats, Lons: lons}
|
2013-04-11 21:50:20 +04:00
|
|
|
}
|
2013-04-14 16:50:38 +04:00
|
|
|
|
2013-05-17 17:16:24 +04:00
|
|
|
func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []element.Node {
|
2013-05-06 09:28:02 +04:00
|
|
|
if len(deltaCoords.Ids) > cap(nodes) {
|
|
|
|
nodes = make([]element.Node, len(deltaCoords.Ids))
|
|
|
|
} else {
|
|
|
|
nodes = nodes[:len(deltaCoords.Ids)]
|
|
|
|
}
|
2013-04-14 16:50:38 +04:00
|
|
|
|
|
|
|
var lastLon, lastLat int64
|
|
|
|
var lon, lat int64
|
|
|
|
var lastId, id int64
|
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
for i := 0; i < len(deltaCoords.Ids); i++ {
|
2013-04-14 16:50:38 +04:00
|
|
|
id = lastId + deltaCoords.Ids[i]
|
2013-05-08 17:44:34 +04:00
|
|
|
lon = lastLon + deltaCoords.Lons[i]
|
|
|
|
lat = lastLat + deltaCoords.Lats[i]
|
2013-05-05 22:14:39 +04:00
|
|
|
nodes[i] = element.Node{
|
2013-04-15 23:54:48 +04:00
|
|
|
OSMElem: element.OSMElem{Id: int64(id)},
|
|
|
|
Long: binary.IntToCoord(uint32(lon)),
|
|
|
|
Lat: binary.IntToCoord(uint32(lat)),
|
|
|
|
}
|
2013-04-14 16:50:38 +04:00
|
|
|
|
|
|
|
lastId = id
|
|
|
|
lastLon = lon
|
|
|
|
lastLat = lat
|
|
|
|
}
|
|
|
|
return nodes
|
|
|
|
}
|
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
type CoordsBunch struct {
|
|
|
|
sync.Mutex
|
|
|
|
id int64
|
2013-05-05 22:14:39 +04:00
|
|
|
coords []element.Node
|
2013-04-15 23:54:48 +04:00
|
|
|
elem *list.Element
|
|
|
|
needsWrite bool
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-21 15:06:49 +04:00
|
|
|
func (b *CoordsBunch) GetCoord(id int64) (*element.Node, error) {
|
|
|
|
idx := sort.Search(len(b.coords), func(i int) bool {
|
|
|
|
return b.coords[i].Id >= id
|
|
|
|
})
|
|
|
|
if idx < len(b.coords) && b.coords[idx].Id == id {
|
|
|
|
return &b.coords[idx], nil
|
|
|
|
}
|
|
|
|
return nil, NotFound
|
|
|
|
}
|
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
type DeltaCoordsCache struct {
|
|
|
|
Cache
|
2013-05-13 10:08:52 +04:00
|
|
|
lruList *list.List
|
|
|
|
table map[int64]*CoordsBunch
|
|
|
|
capacity int64
|
|
|
|
linearImport bool
|
|
|
|
mu sync.Mutex
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-04-16 23:14:19 +04:00
|
|
|
func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
|
2013-04-15 23:54:48 +04:00
|
|
|
coordsCache := DeltaCoordsCache{}
|
2013-04-16 23:14:19 +04:00
|
|
|
err := coordsCache.open(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-04-15 23:54:48 +04:00
|
|
|
coordsCache.lruList = list.New()
|
|
|
|
coordsCache.table = make(map[int64]*CoordsBunch)
|
2013-05-13 12:40:18 +04:00
|
|
|
// mem req for cache approx. capacity*deltaCacheBunchSize*30
|
2013-05-06 13:09:21 +04:00
|
|
|
coordsCache.capacity = 1024 * 8
|
2013-04-16 23:14:19 +04:00
|
|
|
return &coordsCache, nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-13 10:08:52 +04:00
|
|
|
func (self *DeltaCoordsCache) SetLinearImport(v bool) {
|
|
|
|
self.linearImport = v
|
|
|
|
}
|
|
|
|
|
2013-05-14 11:26:28 +04:00
|
|
|
func (self *DeltaCoordsCache) Flush() {
|
2013-05-02 23:11:24 +04:00
|
|
|
for bunchId, bunch := range self.table {
|
2013-04-15 23:54:48 +04:00
|
|
|
if bunch.needsWrite {
|
2013-05-02 23:11:24 +04:00
|
|
|
self.putCoordsPacked(bunchId, bunch.coords)
|
2013-04-15 23:54:48 +04:00
|
|
|
}
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-05-14 11:26:28 +04:00
|
|
|
}
|
|
|
|
func (self *DeltaCoordsCache) Close() {
|
|
|
|
self.Flush()
|
2013-04-15 23:54:48 +04:00
|
|
|
self.Cache.Close()
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
func (self *DeltaCoordsCache) GetCoord(id int64) (*element.Node, error) {
|
2013-05-02 23:11:24 +04:00
|
|
|
bunchId := getBunchId(id)
|
2013-05-16 17:48:22 +04:00
|
|
|
bunch, err := self.getBunch(bunchId)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-04-15 23:54:48 +04:00
|
|
|
defer bunch.Unlock()
|
2013-05-21 15:06:49 +04:00
|
|
|
return bunch.GetCoord(id)
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
|
2013-05-05 22:14:39 +04:00
|
|
|
if way == nil {
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil
|
2013-05-05 22:14:39 +04:00
|
|
|
}
|
2013-05-02 22:37:31 +04:00
|
|
|
way.Nodes = make([]element.Node, len(way.Refs))
|
2013-05-21 15:06:49 +04:00
|
|
|
|
|
|
|
var err error
|
|
|
|
var bunch *CoordsBunch
|
|
|
|
var bunchId, lastBunchId int64
|
|
|
|
lastBunchId = -1
|
|
|
|
|
2013-05-02 22:37:31 +04:00
|
|
|
for i, id := range way.Refs {
|
2013-05-21 15:06:49 +04:00
|
|
|
bunchId = getBunchId(id)
|
|
|
|
// re-use bunches
|
|
|
|
if bunchId != lastBunchId {
|
|
|
|
if bunch != nil {
|
|
|
|
bunch.Unlock()
|
|
|
|
}
|
|
|
|
bunch, err = self.getBunch(bunchId)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lastBunchId = bunchId
|
|
|
|
|
|
|
|
nd, err := bunch.GetCoord(id)
|
2013-05-16 17:48:22 +04:00
|
|
|
if err != nil {
|
2013-05-21 15:06:49 +04:00
|
|
|
bunch.Unlock()
|
2013-05-16 17:48:22 +04:00
|
|
|
return err
|
2013-05-05 22:14:39 +04:00
|
|
|
}
|
2013-05-16 17:48:22 +04:00
|
|
|
way.Nodes[i] = *nd
|
2013-05-02 22:37:31 +04:00
|
|
|
}
|
2013-05-21 15:06:49 +04:00
|
|
|
if bunch != nil {
|
|
|
|
bunch.Unlock()
|
|
|
|
}
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil
|
2013-05-02 22:37:31 +04:00
|
|
|
}
|
|
|
|
|
2013-05-07 09:51:50 +04:00
|
|
|
// PutCoords puts nodes into cache.
|
|
|
|
// nodes need to be sorted by Id.
|
2013-05-16 17:48:22 +04:00
|
|
|
func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
|
2013-05-02 23:11:24 +04:00
|
|
|
var start, currentBunchId int64
|
|
|
|
currentBunchId = getBunchId(nodes[0].Id)
|
2013-04-15 23:54:48 +04:00
|
|
|
start = 0
|
2013-05-07 11:14:58 +04:00
|
|
|
totalNodes := len(nodes)
|
2013-04-15 23:54:48 +04:00
|
|
|
for i, node := range nodes {
|
2013-05-02 23:11:24 +04:00
|
|
|
bunchId := getBunchId(node.Id)
|
|
|
|
if bunchId != currentBunchId {
|
2013-05-13 12:40:18 +04:00
|
|
|
if self.linearImport && int64(i) > deltaCacheBunchSize && int64(i) < int64(totalNodes)-deltaCacheBunchSize {
|
2013-05-07 11:14:58 +04:00
|
|
|
// no need to handle concurrent updates to the same
|
2013-05-13 12:40:18 +04:00
|
|
|
// bunch if we are not at the boundary of a deltaCacheBunchSize
|
2013-05-08 17:45:03 +04:00
|
|
|
self.putCoordsPacked(currentBunchId, nodes[start:i])
|
2013-05-07 11:14:58 +04:00
|
|
|
} else {
|
2013-05-16 17:48:22 +04:00
|
|
|
bunch, err := self.getBunch(currentBunchId)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2013-05-07 11:14:58 +04:00
|
|
|
bunch.coords = append(bunch.coords, nodes[start:i]...)
|
2013-05-13 09:39:00 +04:00
|
|
|
sort.Sort(Nodes(bunch.coords))
|
2013-05-07 11:14:58 +04:00
|
|
|
bunch.needsWrite = true
|
|
|
|
bunch.Unlock()
|
|
|
|
}
|
2013-05-02 23:11:24 +04:00
|
|
|
currentBunchId = bunchId
|
2013-04-15 23:54:48 +04:00
|
|
|
start = int64(i)
|
|
|
|
}
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-05-16 17:48:22 +04:00
|
|
|
bunch, err := self.getBunch(currentBunchId)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2013-05-05 22:14:39 +04:00
|
|
|
bunch.coords = append(bunch.coords, nodes[start:]...)
|
2013-05-13 09:39:00 +04:00
|
|
|
sort.Sort(Nodes(bunch.coords))
|
2013-04-15 23:54:48 +04:00
|
|
|
bunch.needsWrite = true
|
|
|
|
bunch.Unlock()
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
func (p *DeltaCoordsCache) putCoordsPacked(bunchId int64, nodes []element.Node) error {
|
2013-04-15 23:54:48 +04:00
|
|
|
if len(nodes) == 0 {
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-05-10 13:11:29 +04:00
|
|
|
keyBuf := idToKeyBuf(bunchId)
|
2013-04-14 16:50:38 +04:00
|
|
|
|
2013-05-21 13:34:14 +04:00
|
|
|
data := binary.MarshalDeltaNodes(nodes, nil)
|
|
|
|
|
|
|
|
err := p.db.Put(p.wo, keyBuf, data)
|
2013-05-13 09:39:38 +04:00
|
|
|
if err != nil {
|
2013-05-16 17:48:22 +04:00
|
|
|
return err
|
2013-05-13 09:39:38 +04:00
|
|
|
}
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
func (p *DeltaCoordsCache) getCoordsPacked(bunchId int64, nodes []element.Node) ([]element.Node, error) {
|
2013-05-10 13:11:29 +04:00
|
|
|
keyBuf := idToKeyBuf(bunchId)
|
2013-04-14 16:50:38 +04:00
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
data, err := p.db.Get(p.ro, keyBuf)
|
|
|
|
if err != nil {
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil, err
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-05-05 22:14:39 +04:00
|
|
|
if data == nil {
|
2013-05-06 09:28:02 +04:00
|
|
|
// clear before returning
|
2013-05-16 17:48:22 +04:00
|
|
|
return nodes[:0], nil
|
2013-05-05 22:14:39 +04:00
|
|
|
}
|
2013-05-21 13:34:14 +04:00
|
|
|
nodes, err = binary.UnmarshalDeltaNodes(data, nodes)
|
2013-04-15 23:54:48 +04:00
|
|
|
if err != nil {
|
2013-05-16 17:48:22 +04:00
|
|
|
return nil, err
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-04-15 23:54:48 +04:00
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
return nodes, nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-04-15 23:54:48 +04:00
|
|
|
func getBunchId(nodeId int64) int64 {
|
2013-05-13 12:40:18 +04:00
|
|
|
return nodeId / deltaCacheBunchSize
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
|
|
|
|
2013-05-21 15:24:55 +04:00
|
|
|
var (
|
|
|
|
freeNodes = make(chan []element.Node, 4)
|
|
|
|
)
|
|
|
|
|
2013-05-16 17:48:22 +04:00
|
|
|
func (self *DeltaCoordsCache) getBunch(bunchId int64) (*CoordsBunch, error) {
|
2013-04-15 23:54:48 +04:00
|
|
|
self.mu.Lock()
|
|
|
|
defer self.mu.Unlock()
|
2013-05-02 23:11:24 +04:00
|
|
|
bunch, ok := self.table[bunchId]
|
2013-05-06 09:28:02 +04:00
|
|
|
var nodes []element.Node
|
2013-04-14 16:50:38 +04:00
|
|
|
if !ok {
|
2013-05-02 23:11:24 +04:00
|
|
|
elem := self.lruList.PushFront(bunchId)
|
2013-05-21 15:24:55 +04:00
|
|
|
select {
|
|
|
|
case nodes = <-freeNodes:
|
|
|
|
default:
|
2013-05-13 12:40:18 +04:00
|
|
|
nodes = make([]element.Node, 0, deltaCacheBunchSize)
|
2013-05-06 09:28:02 +04:00
|
|
|
}
|
2013-05-16 17:48:22 +04:00
|
|
|
nodes, err := self.getCoordsPacked(bunchId, nodes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2013-05-05 22:14:39 +04:00
|
|
|
bunch = &CoordsBunch{id: bunchId, coords: nodes, elem: elem}
|
2013-05-02 23:11:24 +04:00
|
|
|
self.table[bunchId] = bunch
|
2013-04-15 23:54:48 +04:00
|
|
|
} else {
|
|
|
|
self.lruList.MoveToFront(bunch.elem)
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-04-15 23:54:48 +04:00
|
|
|
bunch.Lock()
|
|
|
|
self.CheckCapacity()
|
2013-05-16 17:48:22 +04:00
|
|
|
return bunch, nil
|
2013-04-14 16:50:38 +04:00
|
|
|
}
|
2013-04-15 23:54:48 +04:00
|
|
|
|
|
|
|
func (self *DeltaCoordsCache) CheckCapacity() {
|
|
|
|
for int64(len(self.table)) > self.capacity {
|
|
|
|
elem := self.lruList.Back()
|
2013-05-02 23:11:24 +04:00
|
|
|
bunchId := self.lruList.Remove(elem).(int64)
|
|
|
|
bunch := self.table[bunchId]
|
2013-05-05 22:14:39 +04:00
|
|
|
bunch.elem = nil
|
2013-04-15 23:54:48 +04:00
|
|
|
if bunch.needsWrite {
|
2013-05-02 23:11:24 +04:00
|
|
|
self.putCoordsPacked(bunchId, bunch.coords)
|
2013-04-15 23:54:48 +04:00
|
|
|
}
|
2013-05-21 15:24:55 +04:00
|
|
|
select {
|
|
|
|
case freeNodes <- bunch.coords:
|
|
|
|
default:
|
|
|
|
}
|
2013-05-02 23:11:24 +04:00
|
|
|
delete(self.table, bunchId)
|
2013-04-15 23:54:48 +04:00
|
|
|
}
|
|
|
|
}
|