vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

more refactoring

master
Oliver Tonnhofer 2013-08-06 17:22:31 +02:00
parent ef3db821be
commit d6fcfc6c23
5 changed files with 250 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cache/binary/diff_test.go vendored
View File

@ -30,7 +30,7 @@ func TestmarshalBunch(t *testing.T) {
if newBunch[1].Id != 123923133 || newBunch[1].Refs[0] != 1231237 {
t.Fatal(newBunch[1])
}
if newBunch[2].Id != 123924123 || newBunch[2].Refs[0] != 912412210 || newBunch[2].refs[1] != 912412213 {
if newBunch[2].Id != 123924123 || newBunch[2].Refs[0] != 912412210 || newBunch[2].Refs[1] != 912412213 {
t.Fatal(newBunch[2])
}
if newBunch[5].Id != 123924132 || newBunch[5].Refs[2] != 212412210 {
@ -39,7 +39,7 @@ func TestmarshalBunch(t *testing.T) {
}
func BenchmarkMarshalBunch(b *testing.B) {
bunch := []idRefs{
bunch := []element.IdRefs{
{123923123, []int64{1213123}},
{123923133, []int64{1231237}},
{123924123, []int64{912412210, 912412213}},

327
cache/diff.go vendored
View File

@ -86,32 +86,6 @@ func (c *DiffCache) Remove() error {
const bufferSize = 64 * 1024
func insertRefs(refs []int64, ref int64) []int64 {
i := sort.Search(len(refs), func(i int) bool {
return refs[i] >= ref
})
if i < len(refs) && refs[i] >= ref {
if refs[i] > ref {
refs = append(refs, 0)
copy(refs[i+1:], refs[i:])
refs[i] = ref
} // else already inserted
} else {
refs = append(refs, ref)
}
return refs
}
func deleteRefs(refs []int64, ref int64) []int64 {
i := sort.Search(len(refs), func(i int) bool {
return refs[i] >= ref
})
if i < len(refs) && refs[i] == ref {
refs = append(refs[:i], refs[i+1:]...)
}
return refs
}
type idRef struct {
id int64
ref int64
@ -128,7 +102,7 @@ type idRefBunches map[int64]idRefBunch
func (bunches *idRefBunches) add(bunchId, id, ref int64) {
idRefs := bunches.getCreate(bunchId, id)
idRefs.Refs = insertRefs(idRefs.Refs, ref)
idRefs.Add(ref)
}
func (bunches *idRefBunches) getCreate(bunchId, id int64) *element.IdRefs {
@ -212,28 +186,6 @@ func newRefIndex(path string, opts *cacheOptions) (*bunchRefCache, error) {
return &index, nil
}
func (index *bunchRefCache) SetLinearImport(val bool) {
if val != !index.linearImport {
panic("programming error, linear import already set")
}
if val {
index.waitWrite.Add(1)
index.waitAdd.Add(1)
go index.writer()
go index.dispatch()
index.linearImport = true
} else {
close(index.addc)
index.waitAdd.Wait()
close(index.write)
index.waitWrite.Wait()
index.linearImport = false
}
}
type CoordsRefIndex struct {
bunchRefCache
}
@ -270,6 +222,155 @@ func (index *bunchRefCache) Close() {
index.cache.Close()
}
func (index *bunchRefCache) Get(id int64) []int64 {
if index.linearImport {
panic("programming error: get not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
panic(err)
}
if data != nil {
for _, idRef := range binary.UnmarshalIdRefsBunch(data) {
if idRef.Id == id {
return idRef.Refs
}
}
}
return nil
}
func (index *bunchRefCache) Add(id, ref int64) error {
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
var idRefs []element.IdRefs
if data != nil {
idRefs = binary.UnmarshalIdRefsBunch(data)
}
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.getCreate(id)
idRef.Add(ref)
data = binary.MarshalIdRefsBunch(idRefBunch.idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
func (index *bunchRefCache) DeleteRef(id, ref int64) error {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
if data != nil {
idRefs := binary.UnmarshalIdRefsBunch(data)
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.get(id)
if idRef != nil {
idRef.Delete(ref)
data := binary.MarshalIdRefsBunch(idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
}
return nil
}
func (index *bunchRefCache) Delete(id int64) error {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
if data != nil {
idRefs := binary.UnmarshalIdRefsBunch(data)
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.get(id)
if idRef != nil {
idRef.Refs = []int64{}
data := binary.MarshalIdRefsBunch(idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
}
return nil
}
func (index *CoordsRefIndex) AddFromWay(way *element.Way) {
for _, node := range way.Nodes {
if index.linearImport {
index.addc <- idRef{id: node.Id, ref: way.Id}
} else {
index.Add(node.Id, way.Id)
}
}
}
func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
for _, node := range way.Nodes {
index.DeleteRef(node.Id, way.Id)
}
}
func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) {
for _, member := range members {
if member.Type == element.WAY {
if index.linearImport {
index.addc <- idRef{id: member.Id, ref: relId}
} else {
index.Add(member.Id, relId)
}
}
}
}
// SetLinearImport optimizes the cache for write operations.
// Get/Delete operations will panic during linear import.
func (index *bunchRefCache) SetLinearImport(val bool) {
if val == index.linearImport {
// already in this mode
return
}
if val {
index.waitWrite.Add(1)
index.waitAdd.Add(1)
go index.writer()
go index.dispatch()
index.linearImport = true
} else {
close(index.addc)
index.waitAdd.Wait()
close(index.write)
index.waitWrite.Wait()
index.linearImport = false
}
}
func (index *bunchRefCache) writer() {
for buffer := range index.write {
if err := index.writeRefs(buffer); err != nil {
@ -369,7 +470,7 @@ NextIdRef:
bunch = append(bunch[:i], bunch[i+1:]...)
} else { // otherwise add refs
for _, r := range newIdRefs.Refs {
bunch[i].Refs = insertRefs(bunch[i].Refs, r)
bunch[i].Add(r)
}
}
lastIdx = i
@ -395,6 +496,8 @@ NextIdRef:
return bunch
}
// loadMergeMarshal loads an existing bunch, merges the IdRefs and
// marshals the result again.
func (index *bunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []element.IdRefs) []byte {
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
@ -416,127 +519,3 @@ func (index *bunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []element.I
data = binary.MarshalIdRefsBunch(bunch)
return data
}
func (index *bunchRefCache) Get(id int64) []int64 {
if index.linearImport {
panic("programming error: get not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
panic(err)
}
if data != nil {
for _, idRef := range binary.UnmarshalIdRefsBunch(data) {
if idRef.Id == id {
return idRef.Refs
}
}
}
return nil
}
func (index *bunchRefCache) Add(id, ref int64) error {
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
var idRefs []element.IdRefs
if data != nil {
idRefs = binary.UnmarshalIdRefsBunch(data)
}
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.getCreate(id)
idRef.Refs = insertRefs(idRef.Refs, ref)
data = binary.MarshalIdRefsBunch(idRefBunch.idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
func (index *bunchRefCache) DeleteRef(id, ref int64) error {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
if data != nil {
idRefs := binary.UnmarshalIdRefsBunch(data)
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.get(id)
if idRef != nil {
idRef.Refs = deleteRefs(idRef.Refs, ref)
data := binary.MarshalIdRefsBunch(idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
}
return nil
}
func (index *bunchRefCache) Delete(id int64) error {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
keyBuf := idToKeyBuf(index.getBunchId(id))
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
return err
}
if data != nil {
idRefs := binary.UnmarshalIdRefsBunch(data)
idRefBunch := idRefBunch{index.getBunchId(id), idRefs}
idRef := idRefBunch.get(id)
if idRef != nil {
idRef.Refs = []int64{}
data := binary.MarshalIdRefsBunch(idRefs)
return index.db.Put(index.wo, keyBuf, data)
}
}
return nil
}
func (index *CoordsRefIndex) AddFromWay(way *element.Way) {
for _, node := range way.Nodes {
if index.linearImport {
index.addc <- idRef{id: node.Id, ref: way.Id}
} else {
index.Add(node.Id, way.Id)
}
}
}
func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) {
if index.linearImport {
panic("programming error: delete not supported in linearImport mode")
}
for _, node := range way.Nodes {
index.DeleteRef(node.Id, way.Id)
}
}
func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) {
for _, member := range members {
if member.Type == element.WAY {
if index.linearImport {
index.addc <- idRef{id: member.Id, ref: relId}
} else {
index.Add(member.Id, relId)
}
}
}
}

40
cache/diff_test.go vendored
View File

@ -8,46 +8,6 @@ import (
"goposm/element"
)
func TestInsertRefs(t *testing.T) {
refs := make([]int64, 0, 1)
refs = insertRefs(refs, 1)
if refs[0] != 1 {
t.Fatal(refs)
}
refs = insertRefs(refs, 10)
if refs[0] != 1 || refs[1] != 10 {
t.Fatal(refs)
}
// insert twice
refs = insertRefs(refs, 10)
if refs[0] != 1 || refs[1] != 10 || len(refs) != 2 {
t.Fatal(refs)
}
// insert before
refs = insertRefs(refs, 0)
if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 {
t.Fatal(refs)
}
// insert after
refs = insertRefs(refs, 12)
if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 || refs[3] != 12 {
t.Fatal(refs)
}
// insert between
refs = insertRefs(refs, 11)
if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 || refs[3] != 11 || refs[4] != 12 {
t.Fatal(refs)
}
}
func TestDiffCache(t *testing.T) {
cache_dir, _ := ioutil.TempDir("", "goposm_test")

26
element/element.go
View File

@ -1,6 +1,8 @@
package element
import (
"sort"
"goposm/geom/geos"
)
@ -63,3 +65,27 @@ type IdRefs struct {
Id int64
Refs []int64
}
func (idRefs *IdRefs) Add(ref int64) {
i := sort.Search(len(idRefs.Refs), func(i int) bool {
return idRefs.Refs[i] >= ref
})
if i < len(idRefs.Refs) && idRefs.Refs[i] >= ref {
if idRefs.Refs[i] > ref {
idRefs.Refs = append(idRefs.Refs, 0)
copy(idRefs.Refs[i+1:], idRefs.Refs[i:])
idRefs.Refs[i] = ref
} // else already inserted
} else {
idRefs.Refs = append(idRefs.Refs, ref)
}
}
func (idRefs *IdRefs) Delete(ref int64) {
i := sort.Search(len(idRefs.Refs), func(i int) bool {
return idRefs.Refs[i] >= ref
})
if i < len(idRefs.Refs) && idRefs.Refs[i] == ref {
idRefs.Refs = append(idRefs.Refs[:i], idRefs.Refs[i+1:]...)
}
}

69
element/element_test.go Normal file
View File

@ -0,0 +1,69 @@
package element
import (
"testing"
)
func TestIdRefs(t *testing.T) {
idRefs := IdRefs{}
idRefs.Add(1)
if idRefs.Refs[0] != 1 {
t.Fatal(idRefs)
}
idRefs.Add(10)
if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 {
t.Fatal(idRefs)
}
// insert twice
idRefs.Add(10)
if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 || len(idRefs.Refs) != 2 {
t.Fatal(idRefs)
}
// insert before
idRefs.Add(0)
if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 {
t.Fatal(idRefs)
}
// insert after
idRefs.Add(12)
if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 12 {
t.Fatal(idRefs)
}
// insert between
idRefs.Add(11)
if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 11 || idRefs.Refs[4] != 12 {
t.Fatal(idRefs)
}
// delete between
idRefs.Delete(11)
if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 12 {
t.Fatal(idRefs)
}
// delete end
idRefs.Delete(12)
if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 {
t.Fatal(idRefs)
}
// delete begin
idRefs.Delete(0)
if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 {
t.Fatal(idRefs)
}
// delete missing
idRefs.Delete(99)
if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 || len(idRefs.Refs) != 2 {
t.Fatal(idRefs)
}
}