vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into diffbunch 

Conflicts:
	cache/diff.go
	cache/diff_test.go
master
Oliver Tonnhofer 2013-07-04 17:33:35 +02:00
parent 80aaa01dc8 d62f11ff97
commit 0f09649775
19 changed files with 490 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cache/binary/nodes.go vendored
View File

@ -1,25 +1,14 @@
package binary
import "code.google.com/p/goprotobuf/proto"
import "log"
func (this *Node) WgsCoord() (lon float64, lat float64) {
func (this *Node) wgsCoord() (lon float64, lat float64) {
lon = IntToCoord(this.GetLong())
lat = IntToCoord(this.GetLat())
return
}
func (this *Node) FromWgsCoord(lon float64, lat float64) {
func (this *Node) fromWgsCoord(lon float64, lat float64) {
longInt := CoordToInt(lon)
latInt := CoordToInt(lat)
this.Long = &longInt
this.Lat = &latInt
}
func (this *Way) Marshal() []byte {
data, err := proto.Marshal(this)
if err != nil {
log.Fatal("marshaling error: ", err)
}
return data
}

54
cache/binary/serialize.go vendored
View File

@ -1,9 +1,7 @@
package binary
import (
"bytes"
"code.google.com/p/goprotobuf/proto"
bin "encoding/binary"
"goposm/element"
)
@ -26,44 +24,10 @@ func Marshal(elem interface{}) ([]byte, error) {
}
}
func MarshalCoord(node *element.Node) ([]byte, error) {
data := make([]byte, 8)
buf := bytes.NewBuffer(data)
err := bin.Write(buf, bin.LittleEndian, CoordToInt(node.Long))
if err != nil {
return nil, err
}
err = bin.Write(buf, bin.LittleEndian, CoordToInt(node.Lat))
if err != nil {
return nil, err
}
return data, nil
}
func UnmarshalCoord(id int64, data []byte) (node *element.Node, err error) {
var long, lat uint32
buf := bytes.NewBuffer(data)
err = bin.Read(buf, bin.LittleEndian, &long)
if err != nil {
return nil, err
}
err = bin.Read(buf, bin.LittleEndian, &lat)
if err != nil {
return nil, err
}
node = &element.Node{}
node.Id = id
node.Long = IntToCoord(long)
node.Lat = IntToCoord(lat)
return node, nil
}
func MarshalNode(node *element.Node) ([]byte, error) {
pbfNode := &Node{}
pbfNode.FromWgsCoord(node.Long, node.Lat)
pbfNode.Tags = TagsAsArray(node.Tags)
pbfNode.fromWgsCoord(node.Long, node.Lat)
pbfNode.Tags = tagsAsArray(node.Tags)
return proto.Marshal(pbfNode)
}
@ -75,8 +39,8 @@ func UnmarshalNode(data []byte) (node *element.Node, err error) {
}
node = &element.Node{}
node.Long, node.Lat = pbfNode.WgsCoord()
node.Tags = TagsFromArray(pbfNode.Tags)
node.Long, node.Lat = pbfNode.wgsCoord()
node.Tags = tagsFromArray(pbfNode.Tags)
return node, nil
}
@ -100,11 +64,11 @@ func deltaUnpack(data []int64) {
}
func MarshalWay(way *element.Way) ([]byte, error) {
// TODO reuse Way to avoid make(Tags) for each way in TagsAsArray
// TODO reuse Way to avoid make(Tags) for each way in tagsAsArray
pbfWay := &Way{}
deltaPack(way.Refs)
pbfWay.Refs = way.Refs
pbfWay.Tags = TagsAsArray(way.Tags)
pbfWay.Tags = tagsAsArray(way.Tags)
return proto.Marshal(pbfWay)
}
@ -118,7 +82,7 @@ func UnmarshalWay(data []byte) (way *element.Way, err error) {
way = &element.Way{}
deltaUnpack(pbfWay.Refs)
way.Refs = pbfWay.Refs
way.Tags = TagsFromArray(pbfWay.Tags)
way.Tags = tagsFromArray(pbfWay.Tags)
return way, nil
}
@ -132,7 +96,7 @@ func MarshalRelation(relation *element.Relation) ([]byte, error) {
pbfRelation.MemberTypes[i] = Relation_MemberType(m.Type)
pbfRelation.MemberRoles[i] = m.Role
}
pbfRelation.Tags = TagsAsArray(relation.Tags)
pbfRelation.Tags = tagsAsArray(relation.Tags)
return proto.Marshal(pbfRelation)
}
@ -151,6 +115,6 @@ func UnmarshalRelation(data []byte) (relation *element.Relation, err error) {
relation.Members[i].Role = pbfRelation.MemberRoles[i]
}
//relation.Nodes = pbfRelation.Node
relation.Tags = TagsFromArray(pbfRelation.Tags)
relation.Tags = tagsFromArray(pbfRelation.Tags)
return relation, nil
}

4
cache/binary/tags.go vendored
View File

@ -65,7 +65,7 @@ func addCommonKey(key string) {
nextKeyCodePoint += 1
}
func TagsFromArray(arr []string) element.Tags {
func tagsFromArray(arr []string) element.Tags {
if len(arr) == 0 {
return element.Tags{}
}
@ -89,7 +89,7 @@ func TagsFromArray(arr []string) element.Tags {
return result
}
func TagsAsArray(tags element.Tags) []string {
func tagsAsArray(tags element.Tags) []string {
if len(tags) == 0 {
return nil
}

4
cache/binary/tags_test.go vendored
View File

@ -7,7 +7,7 @@ import (
func TestTagsAsAndFromArray(t *testing.T) {
tags := element.Tags{"name": "foo", "highway": "residential", "oneway": "yes"}
array := TagsAsArray(tags)
array := tagsAsArray(tags)
if len(array) != 3 {
t.Fatal("invalid length", array)
@ -21,7 +21,7 @@ func TestTagsAsAndFromArray(t *testing.T) {
}
}
tags = TagsFromArray(array)
tags = tagsFromArray(array)
if len(tags) != 3 {
t.Fatal("invalid length", tags)
}

28
cache/config.go vendored
View File

@ -7,7 +7,7 @@ import (
"os"
)
type CacheOptions struct {
type cacheOptions struct {
CacheSizeM int
MaxOpenFiles int
BlockRestartInterval int
@ -15,19 +15,19 @@ type CacheOptions struct {
BlockSizeK int
}
type CoordsCacheOptions struct {
CacheOptions
type coordsCacheOptions struct {
cacheOptions
BunchSize int
BunchCacheCapacity int
}
type OSMCacheOptions struct {
Coords CoordsCacheOptions
Ways CacheOptions
Nodes CacheOptions
Relations CacheOptions
InsertedWays CacheOptions
CoordsIndex CacheOptions
WaysIndex CacheOptions
type osmCacheOptions struct {
Coords coordsCacheOptions
Ways cacheOptions
Nodes cacheOptions
Relations cacheOptions
InsertedWays cacheOptions
CoordsIndex cacheOptions
WaysIndex cacheOptions
}
const defaultConfig = `
@ -86,10 +86,10 @@ const defaultConfig = `
}
`
var osmCacheOptions OSMCacheOptions
var globalCacheOptions osmCacheOptions
func init() {
err := json.Unmarshal([]byte(defaultConfig), &osmCacheOptions)
err := json.Unmarshal([]byte(defaultConfig), &globalCacheOptions)
if err != nil {
panic(err)
}
@ -100,7 +100,7 @@ func init() {
if err != nil {
log.Println("Unable to read cache config:", err)
}
err = json.Unmarshal(data, &osmCacheOptions)
err = json.Unmarshal(data, &globalCacheOptions)
if err != nil {
log.Println("Unable to parse cache config:", err)
}

16
cache/db_test.go vendored
View File

@ -12,7 +12,7 @@ func TestCreateCache(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewNodesCache(cache_dir)
cache, err := newNodesCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -27,7 +27,7 @@ func TestReadWriteNode(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewNodesCache(cache_dir)
cache, err := newNodesCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -39,7 +39,7 @@ func TestReadWriteNode(t *testing.T) {
cache.PutNode(node)
cache.Close()
cache, err = NewNodesCache(cache_dir)
cache, err = newNodesCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -61,7 +61,7 @@ func TestReadWriteWay(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewWaysCache(cache_dir)
cache, err := newWaysCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -74,7 +74,7 @@ func TestReadWriteWay(t *testing.T) {
cache.PutWay(way)
cache.Close()
cache, err = NewWaysCache(cache_dir)
cache, err = newWaysCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -96,7 +96,7 @@ func TestReadMissingWay(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewWaysCache(cache_dir)
cache, err := newWaysCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -114,7 +114,7 @@ func BenchmarkWriteWay(b *testing.B) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewWaysCache(cache_dir)
cache, err := newWaysCache(cache_dir)
if err != nil {
b.Fatal()
}
@ -136,7 +136,7 @@ func BenchmarkReadWay(b *testing.B) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewWaysCache(cache_dir)
cache, err := newWaysCache(cache_dir)
if err != nil {
b.Fatal()
}

38
cache/delta.go vendored
View File

@ -8,11 +8,11 @@ import (
"sync"
)
type Nodes []element.Node
type byId []element.Node
func (s Nodes) Len() int { return len(s) }
func (s Nodes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s Nodes) Less(i, j int) bool { return s[i].Id < s[j].Id }
func (s byId) Len() int { return len(s) }
func (s byId) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byId) Less(i, j int) bool { return s[i].Id < s[j].Id }
func packNodes(nodes []element.Node) *binary.DeltaCoords {
var lastLon, lastLat int64
@ -66,7 +66,7 @@ func unpackNodes(deltaCoords *binary.DeltaCoords, nodes []element.Node) []elemen
return nodes
}
type CoordsBunch struct {
type coordsBunch struct {
sync.Mutex
id int64
coords []element.Node
@ -74,7 +74,7 @@ type CoordsBunch struct {
needsWrite bool
}
func (b *CoordsBunch) GetCoord(id int64) (*element.Node, error) {
func (b *coordsBunch) GetCoord(id int64) (*element.Node, error) {
idx := sort.Search(len(b.coords), func(i int) bool {
return b.coords[i].Id >= id
})
@ -85,9 +85,9 @@ func (b *CoordsBunch) GetCoord(id int64) (*element.Node, error) {
}
type DeltaCoordsCache struct {
Cache
cache
lruList *list.List
table map[int64]*CoordsBunch
table map[int64]*coordsBunch
capacity int64
linearImport bool
mu sync.Mutex
@ -95,18 +95,18 @@ type DeltaCoordsCache struct {
readOnly bool
}
func NewDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
func newDeltaCoordsCache(path string) (*DeltaCoordsCache, error) {
coordsCache := DeltaCoordsCache{}
coordsCache.options = &osmCacheOptions.Coords.CacheOptions
coordsCache.options = &globalCacheOptions.Coords.cacheOptions
err := coordsCache.open(path)
if err != nil {
return nil, err
}
coordsCache.bunchSize = int64(osmCacheOptions.Coords.BunchSize)
coordsCache.bunchSize = int64(globalCacheOptions.Coords.BunchSize)
coordsCache.lruList = list.New()
coordsCache.table = make(map[int64]*CoordsBunch)
coordsCache.table = make(map[int64]*coordsBunch)
// mem req for cache approx. capacity*bunchSize*40
coordsCache.capacity = int64(osmCacheOptions.Coords.BunchCacheCapacity)
coordsCache.capacity = int64(globalCacheOptions.Coords.BunchCacheCapacity)
return &coordsCache, nil
}
@ -123,7 +123,7 @@ func (self *DeltaCoordsCache) Flush() {
}
func (self *DeltaCoordsCache) Close() {
self.Flush()
self.Cache.Close()
self.cache.Close()
}
func (self *DeltaCoordsCache) SetReadOnly(val bool) {
@ -151,7 +151,7 @@ func (self *DeltaCoordsCache) FillWay(way *element.Way) error {
way.Nodes = make([]element.Node, len(way.Refs))
var err error
var bunch *CoordsBunch
var bunch *coordsBunch
var bunchId, lastBunchId int64
lastBunchId = -1
@ -202,7 +202,7 @@ func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
return err
}
bunch.coords = append(bunch.coords, nodes[start:i]...)
sort.Sort(Nodes(bunch.coords))
sort.Sort(byId(bunch.coords))
bunch.needsWrite = true
bunch.Unlock()
}
@ -215,7 +215,7 @@ func (self *DeltaCoordsCache) PutCoords(nodes []element.Node) error {
return err
}
bunch.coords = append(bunch.coords, nodes[start:]...)
sort.Sort(Nodes(bunch.coords))
sort.Sort(byId(bunch.coords))
bunch.needsWrite = true
bunch.Unlock()
return nil
@ -279,7 +279,7 @@ var (
freeNodes = make(chan []element.Node, 4)
)
func (self *DeltaCoordsCache) getBunch(bunchId int64) (*CoordsBunch, error) {
func (self *DeltaCoordsCache) getBunch(bunchId int64) (*coordsBunch, error) {
self.mu.Lock()
bunch, ok := self.table[bunchId]
var nodes []element.Node
@ -291,7 +291,7 @@ func (self *DeltaCoordsCache) getBunch(bunchId int64) (*CoordsBunch, error) {
default:
nodes = make([]element.Node, 0, self.bunchSize)
}
bunch = &CoordsBunch{id: bunchId, coords: nil, elem: elem}
bunch = &coordsBunch{id: bunchId, coords: nil, elem: elem}
needsGet = true
self.table[bunchId] = bunch
} else {

6
cache/delta_test.go vendored
View File

@ -23,7 +23,7 @@ func TestReadWriteDeltaCoords(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewDeltaCoordsCache(cache_dir)
cache, err := newDeltaCoordsCache(cache_dir)
if err != nil {
t.Fatal()
}
@ -38,13 +38,13 @@ func TestReadWriteDeltaCoords(t *testing.T) {
// add nodes in batches of ten
for i := 0; i <= len(nodes)-10; i = i + 10 {
// sort each batch as required by PutCoords
sort.Sort(Nodes(nodes[i : i+10]))
sort.Sort(byId(nodes[i : i+10]))
cache.PutCoords(nodes[i : i+10])
}
cache.Close()
cache, err = NewDeltaCoordsCache(cache_dir)
cache, err = newDeltaCoordsCache(cache_dir)
if err != nil {
t.Fatal()
}

236
cache/diff.go vendored
View File

@ -1,22 +1,28 @@
package cache
import (
"bytes"
"encoding/binary"
"github.com/jmhodges/levigo"
"goposm/element"
"io"
"log"
"os"
"path/filepath"
"runtime"
"sort"
"sync"
)
type Refs []int64
type byInt64 []int64
func (a Refs) Len() int { return len(a) }
func (a Refs) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a Refs) Less(i, j int) bool { return a[i] < a[j] }
func (a byInt64) Len() int { return len(a) }
func (a byInt64) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byInt64) Less(i, j int) bool { return a[i] < a[j] }
type DiffCache struct {
Dir string
Coords *BunchRefCache
Coords *CoordsRefIndex
Ways *WaysRefIndex
opened bool
}
@ -39,7 +45,7 @@ func NewDiffCache(dir string) *DiffCache {
func (c *DiffCache) Open() error {
var err error
c.Coords, err = NewBunchRefCache(filepath.Join(c.Dir, "coords_index"), &osmCacheOptions.CoordsIndex)
c.Coords, err = NewCoordsRefIndex(filepath.Join(c.Dir, "coords_index"))
if err != nil {
c.Close()
return err
@ -80,8 +86,8 @@ func (c *DiffCache) Remove() error {
}
type RefIndex struct {
Cache
cache map[int64][]int64
cache
buffer map[int64][]int64
write chan map[int64][]int64
add chan idRef
mu sync.Mutex
@ -90,10 +96,10 @@ type RefIndex struct {
}
type CoordsRefIndex struct {
BunchRefCache
RefIndex
}
type WaysRefIndex struct {
BunchRefCache
RefIndex
}
type idRef struct {
@ -101,7 +107,7 @@ type idRef struct {
ref int64
}
const cacheSize = 1024
const cacheSize = 64 * 1024
var refCaches chan map[int64][]int64
@ -109,8 +115,29 @@ func init() {
refCaches = make(chan map[int64][]int64, 1)
}
func NewRefIndex(path string, opts *cacheOptions) (*RefIndex, error) {
index := RefIndex{}
index.options = opts
err := index.open(path)
if err != nil {
return nil, err
}
index.write = make(chan map[int64][]int64, 2)
index.buffer = make(map[int64][]int64, cacheSize)
index.add = make(chan idRef, 1024)
index.waitWrite = &sync.WaitGroup{}
index.waitAdd = &sync.WaitGroup{}
index.waitWrite.Add(1)
index.waitAdd.Add(1)
go index.writer()
go index.dispatch()
return &index, nil
}
func NewCoordsRefIndex(dir string) (*CoordsRefIndex, error) {
cache, err := NewBunchRefCache(dir, &osmCacheOptions.CoordsIndex)
cache, err := NewRefIndex(dir, &globalCacheOptions.CoordsIndex)
if err != nil {
return nil, err
}
@ -118,13 +145,49 @@ func NewCoordsRefIndex(dir string) (*CoordsRefIndex, error) {
}
func NewWaysRefIndex(dir string) (*WaysRefIndex, error) {
cache, err := NewBunchRefCache(dir, &osmCacheOptions.WaysIndex)
cache, err := NewRefIndex(dir, &globalCacheOptions.WaysIndex)
if err != nil {
return nil, err
}
return &WaysRefIndex{*cache}, nil
}
func (index *RefIndex) writer() {
for buffer := range index.write {
if err := index.writeRefs(buffer); err != nil {
log.Println("error while writing ref index", err)
}
}
index.waitWrite.Done()
}
func (index *RefIndex) Close() {
close(index.add)
index.waitAdd.Wait()
close(index.write)
index.waitWrite.Wait()
index.cache.Close()
}
func (index *RefIndex) dispatch() {
for idRef := range index.add {
index.addToCache(idRef.id, idRef.ref)
if len(index.buffer) >= cacheSize {
index.write <- index.buffer
select {
case index.buffer = <-refCaches:
default:
index.buffer = make(map[int64][]int64, cacheSize)
}
}
}
if len(index.buffer) > 0 {
index.write <- index.buffer
index.buffer = nil
}
index.waitAdd.Done()
}
func (index *CoordsRefIndex) AddFromWay(way *element.Way) {
for _, node := range way.Nodes {
index.add <- idRef{node.Id, way.Id}
@ -139,14 +202,99 @@ func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member)
}
}
func (index *RefIndex) addToCache(id, ref int64) {
refs, ok := index.buffer[id]
if !ok {
refs = make([]int64, 0, 1)
}
refs = insertRefs(refs, ref)
index.buffer[id] = refs
}
type writeRefItem struct {
key []byte
data []byte
}
type loadRefItem struct {
id int64
refs []int64
}
func (index *RefIndex) writeRefs(idRefs map[int64][]int64) error {
batch := levigo.NewWriteBatch()
defer batch.Close()
wg := sync.WaitGroup{}
putc := make(chan writeRefItem)
loadc := make(chan loadRefItem)
for i := 0; i < runtime.NumCPU(); i++ {
wg.Add(1)
go func() {
for item := range loadc {
keyBuf := idToKeyBuf(item.id)
putc <- writeRefItem{
keyBuf,
index.loadAppendMarshal(keyBuf, item.refs),
}
}
wg.Done()
}()
}
go func() {
for id, refs := range idRefs {
loadc <- loadRefItem{id, refs}
}
close(loadc)
wg.Wait()
close(putc)
}()
for item := range putc {
batch.Put(item.key, item.data)
}
go func() {
for k, _ := range idRefs {
delete(idRefs, k)
}
select {
case refCaches <- idRefs:
}
}()
return index.db.Write(index.wo, batch)
}
func (index *RefIndex) loadAppendMarshal(keyBuf []byte, newRefs []int64) []byte {
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
panic(err)
}
var refs []int64
if data != nil {
refs = UnmarshalRefs(data)
}
if refs == nil {
refs = newRefs
} else {
refs = append(refs, newRefs...)
sort.Sort(byInt64(refs))
}
data = MarshalRefs(refs)
return data
}
func insertRefs(refs []int64, ref int64) []int64 {
i := sort.Search(len(refs), func(i int) bool {
return refs[i] >= ref
})
if i < len(refs) && refs[i] >= ref {
if refs[i] == ref {
return refs
}
refs = append(refs, 0)
copy(refs[i+1:], refs[i:])
refs[i] = ref
@ -155,3 +303,59 @@ func insertRefs(refs []int64, ref int64) []int64 {
}
return refs
}
func (index *RefIndex) Get(id int64) []int64 {
keyBuf := idToKeyBuf(id)
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
panic(err)
}
var refs []int64
if data != nil {
refs = UnmarshalRefs(data)
if err != nil {
panic(err)
}
}
return refs
}
func UnmarshalRefs(buf []byte) []int64 {
refs := make([]int64, 0, 8)
r := bytes.NewBuffer(buf)
lastRef := int64(0)
for {
ref, err := binary.ReadVarint(r)
if err == io.EOF {
break
}
if err != nil {
log.Println("error while unmarshaling refs:", err)
break
}
ref = lastRef + ref
refs = append(refs, ref)
lastRef = ref
}
return refs
}
func MarshalRefs(refs []int64) []byte {
buf := make([]byte, len(refs)*4+binary.MaxVarintLen64)
lastRef := int64(0)
nextPos := 0
for _, ref := range refs {
if len(buf)-nextPos < binary.MaxVarintLen64 {
tmp := make([]byte, len(buf)*2)
copy(tmp, buf)
buf = tmp
}
nextPos += binary.PutVarint(buf[nextPos:], ref-lastRef)
lastRef = ref
}
return buf[:nextPos]
}

156
cache/diff_test.go vendored
View File

@ -46,23 +46,41 @@ func TestInsertRefs(t *testing.T) {
}
func TestMarshalRefs(t *testing.T) {
refs := []int64{1890166659, -1890166659, 0, 1890166, 1890167, 1890167, 1890165}
buf := MarshalRefs(refs)
t.Log(len(refs), len(buf))
result := UnmarshalRefs(buf)
if len(result) != len(refs) {
t.Fatal(result)
}
for i, ref := range refs {
if result[i] != ref {
t.Fatal(result)
}
}
}
func TestWriteDiff(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewBunchRefCache(cache_dir, &osmCacheOptions.CoordsIndex)
cache, err := NewRefIndex(cache_dir, &globalCacheOptions.CoordsIndex)
if err != nil {
t.Fatal()
}
for w := 0; w < 5; w++ {
for n := 0; n < 200; n++ {
cache.cache.add(cache.getBunchId(int64(n)), int64(n), int64(w))
cache.addToCache(int64(n), int64(w))
}
}
cache.Close()
cache, err = NewBunchRefCache(cache_dir, &osmCacheOptions.CoordsIndex)
cache, err = NewRefIndex(cache_dir, &globalCacheOptions.CoordsIndex)
if err != nil {
t.Fatal(err)
}
@ -74,54 +92,7 @@ func TestWriteDiff(t *testing.T) {
t.Fatal(refs)
}
}
}
func TestMarshalBunch(t *testing.T) {
bunch := []IdRef{
{123923123, []int64{1213123}},
{123923133, []int64{1231237}},
{123924123, []int64{912412210, 912412213}},
{123924129, []int64{812412213}},
{123924130, []int64{91241213}},
{123924132, []int64{912412210, 9124213, 212412210}},
}
buf := MarshalBunch(bunch)
newBunch := UnmarshalBunch(buf)
t.Log(len(buf), float64(len(buf))/6.0)
if len(newBunch) != 6 {
t.Fatal(newBunch)
}
if newBunch[0].id != 123923123 || newBunch[0].refs[0] != 1213123 {
t.Fatal(newBunch[0])
}
if newBunch[1].id != 123923133 || newBunch[1].refs[0] != 1231237 {
t.Fatal(newBunch[1])
}
if newBunch[2].id != 123924123 || newBunch[2].refs[0] != 912412210 || newBunch[2].refs[1] != 912412213 {
t.Fatal(newBunch[2])
}
if newBunch[5].id != 123924132 || newBunch[5].refs[2] != 212412210 {
t.Fatal(newBunch[5])
}
}
func BenchmarkMarshalBunch(b *testing.B) {
bunch := []IdRef{
{123923123, []int64{1213123}},
{123923133, []int64{1231237}},
{123924123, []int64{912412210, 912412213}},
{123924129, []int64{812412213}},
{123924130, []int64{91241213}},
{123924132, []int64{912412210, 9124213, 212412210}},
}
for i := 0; i < b.N; i++ {
buf := MarshalBunch(bunch)
UnmarshalBunch(buf)
}
}
func BenchmarkWriteDiff(b *testing.B) {
@ -129,7 +100,7 @@ func BenchmarkWriteDiff(b *testing.B) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")
defer os.RemoveAll(cache_dir)
cache, err := NewBunchRefCache(cache_dir, &osmCacheOptions.CoordsIndex)
cache, err := NewRefIndex(cache_dir, &globalCacheOptions.CoordsIndex)
if err != nil {
b.Fatal()
}
@ -139,90 +110,9 @@ func BenchmarkWriteDiff(b *testing.B) {
for i := 0; i < b.N; i++ {
for w := 0; w < 5; w++ {
for n := 0; n < 200; n++ {
cache.cache.add(cache.getBunchId(int64(n)), int64(n), int64(w))
cache.addToCache(int64(n), int64(w))
}
}
}
}
func TestMergeIdRefs(t *testing.T) {
bunch := []IdRef{}
bunch = mergeBunch(bunch, []IdRef{IdRef{50, []int64{1}}})
if b := bunch[0]; b.id != 50 || b.refs[0] != 1 {
t.Fatal(bunch)
}
// before
bunch = mergeBunch(bunch, []IdRef{IdRef{40, []int64{3}}})
if b := bunch[0]; b.id != 40 || b.refs[0] != 3 {
t.Fatal(bunch)
}
// after
bunch = mergeBunch(bunch, []IdRef{IdRef{70, []int64{4}}})
if b := bunch[2]; b.id != 70 || b.refs[0] != 4 {
t.Fatal(bunch)
}
// in between
bunch = mergeBunch(bunch, []IdRef{IdRef{60, []int64{5}}})
if b := bunch[2]; b.id != 60 || b.refs[0] != 5 {
t.Fatal(bunch)
}
// same
bunch = mergeBunch(bunch, []IdRef{IdRef{50, []int64{0, 5}}})
if b := bunch[1]; b.id != 50 || b.refs[0] != 0 ||
b.refs[1] != 1 || b.refs[2] != 5 {
t.Fatal(bunch)
}
if len(bunch) != 4 {
t.Fatal(bunch)
}
}
func TestIdRefBunches(t *testing.T) {
bunches := make(IdRefBunches)
bunches.add(1, 100, 999)
if r := bunches[1].idRefs[0]; r.id != 100 || r.refs[0] != 999 {
t.Fatal(bunches)
}
// before
bunches.add(1, 99, 888)
if r := bunches[1].idRefs[0]; r.id != 99 || r.refs[0] != 888 {
t.Fatal(bunches)
}
// after
bunches.add(1, 102, 777)
if r := bunches[1].idRefs[2]; r.id != 102 || r.refs[0] != 777 {
t.Fatal(bunches)
}
// in between
bunches.add(1, 101, 666)
if r := bunches[1].idRefs[2]; r.id != 101 || r.refs[0] != 666 {
t.Fatal(bunches)
}
// same id
bunches.add(1, 100, 998)
if r := bunches[1].idRefs[1]; r.id != 100 || r.refs[0] != 998 || r.refs[1] != 999 {
t.Fatal(bunches)
}
if len(bunches) != 1 {
t.Fatal(bunches)
}
if bunches[1].id != 1 {
t.Fatal(bunches)
}
if len(bunches[1].idRefs) != 4 {
t.Fatal(bunches)
}
}

30
cache/diffbunch.go vendored
View File

@ -61,8 +61,8 @@ func init() {
// BunchRefCache
type BunchRefCache struct {
Cache
cache IdRefBunches
cache
buffer IdRefBunches
write chan IdRefBunches
add chan idRef
mu sync.Mutex
@ -70,7 +70,7 @@ type BunchRefCache struct {
waitWrite *sync.WaitGroup
}
func NewBunchRefCache(path string, opts *CacheOptions) (*BunchRefCache, error) {
func NewBunchRefCache(path string, opts *cacheOptions) (*BunchRefCache, error) {
index := BunchRefCache{}
index.options = opts
err := index.open(path)
@ -78,7 +78,7 @@ func NewBunchRefCache(path string, opts *CacheOptions) (*BunchRefCache, error) {
return nil, err
}
index.write = make(chan IdRefBunches, 2)
index.cache = make(IdRefBunches, cacheSize)
index.buffer = make(IdRefBunches, cacheSize)
index.add = make(chan idRef, 1024)
index.waitWrite = &sync.WaitGroup{}
@ -92,8 +92,8 @@ func NewBunchRefCache(path string, opts *CacheOptions) (*BunchRefCache, error) {
}
func (index *BunchRefCache) writer() {
for cache := range index.write {
if err := index.writeRefs(cache); err != nil {
for buffer := range index.write {
if err := index.writeRefs(buffer); err != nil {
log.Println("error while writing ref index", err)
}
}
@ -105,24 +105,24 @@ func (index *BunchRefCache) Close() {
index.waitAdd.Wait()
close(index.write)
index.waitWrite.Wait()
index.Cache.Close()
index.cache.Close()
}
func (index *BunchRefCache) dispatch() {
for idRef := range index.add {
index.cache.add(index.getBunchId(idRef.id), idRef.id, idRef.ref)
if len(index.cache) >= cacheSize {
index.write <- index.cache
index.buffer.add(index.getBunchId(idRef.id), idRef.id, idRef.ref)
if len(index.buffer) >= cacheSize {
index.write <- index.buffer
select {
case index.cache = <-IdRefBunchesPool:
case index.buffer = <-IdRefBunchesPool:
default:
index.cache = make(IdRefBunches, cacheSize)
index.buffer = make(IdRefBunches, cacheSize)
}
}
}
if len(index.cache) > 0 {
index.write <- index.cache
index.cache = nil
if len(index.buffer) > 0 {
index.write <- index.buffer
index.buffer = nil
}
index.waitAdd.Done()
}

6
cache/nodes.go vendored
View File

@ -7,12 +7,12 @@ import (
)
type NodesCache struct {
Cache
cache
}
func NewNodesCache(path string) (*NodesCache, error) {
func newNodesCache(path string) (*NodesCache, error) {
cache := NodesCache{}
cache.options = &osmCacheOptions.Nodes
cache.options = &globalCacheOptions.Nodes
err := cache.open(path)
if err != nil {
return nil, err

44
cache/osm.go vendored
View File

@ -13,7 +13,7 @@ var (
)
type OSMCache struct {
Dir string
dir string
Coords *DeltaCoordsCache
Ways *WaysCache
Nodes *NodesCache
@ -46,35 +46,35 @@ func (c *OSMCache) Close() {
}
func NewOSMCache(dir string) *OSMCache {
cache := &OSMCache{Dir: dir}
cache := &OSMCache{dir: dir}
return cache
}
func (c *OSMCache) Open() error {
err := os.MkdirAll(c.Dir, 0755)
err := os.MkdirAll(c.dir, 0755)
if err != nil {
return err
}
c.Coords, err = NewDeltaCoordsCache(filepath.Join(c.Dir, "coords"))
c.Coords, err = newDeltaCoordsCache(filepath.Join(c.dir, "coords"))
if err != nil {
return err
}
c.Nodes, err = NewNodesCache(filepath.Join(c.Dir, "nodes"))
c.Nodes, err = newNodesCache(filepath.Join(c.dir, "nodes"))
if err != nil {
c.Close()
return err
}
c.Ways, err = NewWaysCache(filepath.Join(c.Dir, "ways"))
c.Ways, err = newWaysCache(filepath.Join(c.dir, "ways"))
if err != nil {
c.Close()
return err
}
c.Relations, err = NewRelationsCache(filepath.Join(c.Dir, "relations"))
c.Relations, err = newRelationsCache(filepath.Join(c.dir, "relations"))
if err != nil {
c.Close()
return err
}
c.InsertedWays, err = NewInsertedWaysCache(filepath.Join(c.Dir, "inserted_ways"))
c.InsertedWays, err = newInsertedWaysCache(filepath.Join(c.dir, "inserted_ways"))
if err != nil {
c.Close()
return err
@ -87,19 +87,19 @@ func (c *OSMCache) Exists() bool {
if c.opened {
return true
}
if _, err := os.Stat(filepath.Join(c.Dir, "coords")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(c.dir, "coords")); !os.IsNotExist(err) {
return true
}
if _, err := os.Stat(filepath.Join(c.Dir, "nodes")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(c.dir, "nodes")); !os.IsNotExist(err) {
return true
}
if _, err := os.Stat(filepath.Join(c.Dir, "ways")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(c.dir, "ways")); !os.IsNotExist(err) {
return true
}
if _, err := os.Stat(filepath.Join(c.Dir, "relations")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(c.dir, "relations")); !os.IsNotExist(err) {
return true
}
if _, err := os.Stat(filepath.Join(c.Dir, "inserted_ways")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(c.dir, "inserted_ways")); !os.IsNotExist(err) {
return true
}
return false
@ -109,33 +109,33 @@ func (c *OSMCache) Remove() error {
if c.opened {
c.Close()
}
if err := os.RemoveAll(filepath.Join(c.Dir, "coords")); err != nil {
if err := os.RemoveAll(filepath.Join(c.dir, "coords")); err != nil {
return err
}
if err := os.RemoveAll(filepath.Join(c.Dir, "nodes")); err != nil {
if err := os.RemoveAll(filepath.Join(c.dir, "nodes")); err != nil {
return err
}
if err := os.RemoveAll(filepath.Join(c.Dir, "ways")); err != nil {
if err := os.RemoveAll(filepath.Join(c.dir, "ways")); err != nil {
return err
}
if err := os.RemoveAll(filepath.Join(c.Dir, "relations")); err != nil {
if err := os.RemoveAll(filepath.Join(c.dir, "relations")); err != nil {
return err
}
if err := os.RemoveAll(filepath.Join(c.Dir, "inserted_ways")); err != nil {
if err := os.RemoveAll(filepath.Join(c.dir, "inserted_ways")); err != nil {
return err
}
return nil
}
type Cache struct {
type cache struct {
db *levigo.DB
options *CacheOptions
options *cacheOptions
cache *levigo.Cache
wo *levigo.WriteOptions
ro *levigo.ReadOptions
}
func (c *Cache) open(path string) error {
func (c *cache) open(path string) error {
opts := levigo.NewOptions()
opts.SetCreateIfMissing(true)
if c.options.CacheSizeM > 0 {
@ -175,7 +175,7 @@ func idFromKeyBuf(buf []byte) int64 {
return int64(bin.BigEndian.Uint64(buf))
}
func (c *Cache) Close() {
func (c *cache) Close() {
if c.db != nil {
c.db.Close()
c.db = nil

6
cache/relations.go vendored
View File

@ -7,12 +7,12 @@ import (
)
type RelationsCache struct {
Cache
cache
}
func NewRelationsCache(path string) (*RelationsCache, error) {
func newRelationsCache(path string) (*RelationsCache, error) {
cache := RelationsCache{}
cache.options = &osmCacheOptions.Relations
cache.options = &globalCacheOptions.Relations
err := cache.open(path)
if err != nil {
return nil, err

12
cache/ways.go vendored
View File

@ -7,12 +7,12 @@ import (
)
type WaysCache struct {
Cache
cache
}
func NewWaysCache(path string) (*WaysCache, error) {
func newWaysCache(path string) (*WaysCache, error) {
cache := WaysCache{}
cache.options = &osmCacheOptions.Ways
cache.options = &globalCacheOptions.Ways
err := cache.open(path)
if err != nil {
return nil, err
@ -100,12 +100,12 @@ func (self *WaysCache) FillMembers(members []element.Member) error {
}
type InsertedWaysCache struct {
Cache
cache
}
func NewInsertedWaysCache(path string) (*InsertedWaysCache, error) {
func newInsertedWaysCache(path string) (*InsertedWaysCache, error) {
cache := InsertedWaysCache{}
cache.options = &osmCacheOptions.InsertedWays
cache.options = &globalCacheOptions.InsertedWays
err := cache.open(path)
if err != nil {

4
database/database.go
View File

@ -45,6 +45,10 @@ type Deleter interface {
Delete(string, int64) error
}
type Optimizer interface {
Optimize() error
}
var databases map[string]func(Config, *mapping.Mapping) (DB, error)
func init() {

191
database/postgis/postgis.go
View File

@ -11,7 +11,6 @@ import (
"runtime"
"strings"
"sync"
"time"
)
var log = logging.NewLogger("PostGIS")
@ -189,73 +188,24 @@ func (pg *PostGIS) Finish() error {
worker = 1
}
time.Sleep(0 * time.Second)
p := newWorkerPool(worker, len(pg.Tables))
p := newWorkerPool(worker, len(pg.Tables)+len(pg.GeneralizedTables))
for tableName, tbl := range pg.Tables {
tableName := pg.Prefix + tableName
table := tbl
p.in <- func() error {
for _, col := range table.Columns {
if col.Type.Name() == "GEOMETRY" {
sql := fmt.Sprintf(`CREATE INDEX "%s_geom" ON "%s"."%s" USING GIST ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating geometry index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
if col.FieldType.Name == "id" {
sql := fmt.Sprintf(`CREATE INDEX "%s_osm_id_idx" ON "%s"."%s" USING BTREE ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating OSM id index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
}
return nil
return createIndex(pg, tableName, table.Columns)
}
}
err := p.wait()
if err != nil {
return err
}
p = newWorkerPool(worker, len(pg.GeneralizedTables))
for tableName, tbl := range pg.GeneralizedTables {
tableName := pg.Prefix + tableName
table := tbl
p.in <- func() error {
for _, col := range table.Source.Columns {
if col.Type.Name() == "GEOMETRY" {
sql := fmt.Sprintf(`CREATE INDEX "%s_geom" ON "%s"."%s" USING GIST ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating geometry index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
if col.FieldType.Name == "id" {
sql := fmt.Sprintf(`CREATE INDEX "%s_osm_id_idx" ON "%s"."%s" USING BTREE ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating OSM id index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
}
return nil
return createIndex(pg, tableName, table.Source.Columns)
}
}
err = p.wait()
err := p.wait()
if err != nil {
return err
}
@ -263,30 +213,30 @@ func (pg *PostGIS) Finish() error {
return nil
}
func (pg *PostGIS) checkGeneralizedTableSources() {
for name, table := range pg.GeneralizedTables {
if source, ok := pg.Tables[table.SourceName]; ok {
table.Source = source
} else if source, ok := pg.GeneralizedTables[table.SourceName]; ok {
table.SourceGeneralized = source
} else {
log.Printf("missing source '%s' for generalized table '%s'\n",
table.SourceName, name)
func createIndex(pg *PostGIS, tableName string, columns []ColumnSpec) error {
for _, col := range columns {
if col.Type.Name() == "GEOMETRY" {
sql := fmt.Sprintf(`CREATE INDEX "%s_geom" ON "%s"."%s" USING GIST ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating geometry index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
}
filled := true
for filled {
filled = false
for _, table := range pg.GeneralizedTables {
if table.Source == nil {
if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil {
table.Source = source.Source
}
filled = true
if col.FieldType.Name == "id" {
sql := fmt.Sprintf(`CREATE INDEX "%s_osm_id_idx" ON "%s"."%s" USING BTREE ("%s")`,
tableName, pg.Schema, tableName, col.Name)
step := log.StartStep(fmt.Sprintf("Creating OSM id index on %s", tableName))
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
}
}
return nil
}
func (pg *PostGIS) Generalize() error {
@ -391,6 +341,66 @@ func (pg *PostGIS) generalizeTable(table *GeneralizedTableSpec) error {
return nil
}
// Optimize clusters tables on new GeoHash index.
func (pg *PostGIS) Optimize() error {
defer log.StopStep(log.StartStep(fmt.Sprintf("Clustering on geometry")))
worker := int(runtime.NumCPU() / 2)
if worker < 1 {
worker = 1
}
p := newWorkerPool(worker, len(pg.Tables)+len(pg.GeneralizedTables))
for tableName, tbl := range pg.Tables {
tableName := pg.Prefix + tableName
table := tbl
p.in <- func() error {
return clusterTable(pg, tableName, table.Srid, table.Columns)
}
}
for tableName, tbl := range pg.GeneralizedTables {
tableName := pg.Prefix + tableName
table := tbl
p.in <- func() error {
return clusterTable(pg, tableName, table.Source.Srid, table.Source.Columns)
}
}
err := p.wait()
if err != nil {
return err
}
return nil
}
func clusterTable(pg *PostGIS, tableName string, srid int, columns []ColumnSpec) error {
for _, col := range columns {
if col.Type.Name() == "GEOMETRY" {
step := log.StartStep(fmt.Sprintf("Indexing %s on geohash", tableName))
sql := fmt.Sprintf(`CREATE INDEX "%s_geom_geohash" ON "%s"."%s" (ST_GeoHash(ST_Transform(ST_SetSRID(Box2D(%s), %d), 4326)))`,
tableName, pg.Schema, tableName, col.Name, srid)
_, err := pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
step = log.StartStep(fmt.Sprintf("Clustering %s on geohash", tableName))
sql = fmt.Sprintf(`CLUSTER "%s_geom_geohash" ON "%s"."%s"`,
tableName, pg.Schema, tableName)
_, err = pg.Db.Exec(sql)
log.StopStep(step)
if err != nil {
return err
}
break
}
}
return nil
}
type PostGIS struct {
Db *sql.DB
Schema string
@ -591,7 +601,7 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
for name, table := range m.GeneralizedTables {
db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table)
}
db.checkGeneralizedTableSources()
db.prepareGeneralizedTableSources()
err = db.Open()
if err != nil {
@ -600,6 +610,35 @@ func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
return db, nil
}
// prepareGeneralizedTableSources checks if all generalized table have an
// existing source and sets .Source to the original source (works even
// when source is allready generalized).
func (pg *PostGIS) prepareGeneralizedTableSources() {
for name, table := range pg.GeneralizedTables {
if source, ok := pg.Tables[table.SourceName]; ok {
table.Source = source
} else if source, ok := pg.GeneralizedTables[table.SourceName]; ok {
table.SourceGeneralized = source
} else {
log.Printf("missing source '%s' for generalized table '%s'\n",
table.SourceName, name)
}
}
// set source table until all generalized tables have a source
for filled := true; filled; {
filled = false
for _, table := range pg.GeneralizedTables {
if table.Source == nil {
if source, ok := pg.GeneralizedTables[table.SourceName]; ok && source.Source != nil {
table.Source = source.Source
}
filled = true
}
}
}
}
func init() {
database.Register("postgres", New)
database.Register("postgis", New)

8
geom/clipper/clipper.go
View File

@ -193,6 +193,14 @@ func (clipper *Clipper) Clip(geom *geos.Geom) ([]*geos.Geom, error) {
return mergeGeometries(g, intersections, geomType), nil
}
func (clipper *Clipper) Intersects(g *geos.Geos, geom *geos.Geom) bool {
hits := g.IndexQuery(clipper.index, geom)
if len(hits) == 0 {
return false
}
return true
}
func flattenPolygons(g *geos.Geos, geoms []*geos.Geom) []*geos.Geom {
var result []*geos.Geom
for _, geom := range geoms {

13
goposm.go
View File

@ -30,6 +30,7 @@ var (
appendcache = flag.Bool("appendcache", false, "append cache")
read = flag.String("read", "", "read")
write = flag.Bool("write", false, "write")
optimize = flag.Bool("optimize", false, "optimize")
connection = flag.String("connection", "", "connection parameters")
diff = flag.Bool("diff", false, "enable diff support")
mappingFile = flag.String("mapping", "", "mapping file")
@ -137,7 +138,7 @@ func main() {
srid := 3857 // TODO
if *write || *deployProduction || *revertDeploy || *removeBackup {
if *write || *deployProduction || *revertDeploy || *removeBackup || *optimize {
connType := database.ConnectionType(*connection)
conf := database.Config{
Type: connType,
@ -267,6 +268,16 @@ func main() {
log.StopStep(stepImport)
}
if *optimize {
if db, ok := db.(database.Optimizer); ok {
if err := db.Optimize(); err != nil {
die(err)
}
} else {
die("database not optimizable")
}
}
if *deployProduction {
if db, ok := db.(database.Deployer); ok {
if err := db.Deploy(); err != nil {