vitalif
/
imposm3
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactored bunchCache; add more tests

master
Oliver Tonnhofer 2013-06-26 21:54:20 +02:00
parent 64b2689ab8
commit 94b5999756
2 changed files with 129 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
cache/diff_test.go vendored
View File

@ -192,6 +192,44 @@ func TestWriteDiffBunch(t *testing.T) {
}
func TestMergeIdRefs(t *testing.T) {
bunch := []IdRef{}
bunch = mergeBunch(bunch, []IdRef{IdRef{50, []int64{1}}})
if b := bunch[0]; b.id != 50 || b.refs[0] != 1 {
t.Fatal(bunch)
}
// before
bunch = mergeBunch(bunch, []IdRef{IdRef{40, []int64{3}}})
if b := bunch[0]; b.id != 40 || b.refs[0] != 3 {
t.Fatal(bunch)
}
// after
bunch = mergeBunch(bunch, []IdRef{IdRef{70, []int64{4}}})
if b := bunch[2]; b.id != 70 || b.refs[0] != 4 {
t.Fatal(bunch)
}
// in between
bunch = mergeBunch(bunch, []IdRef{IdRef{60, []int64{5}}})
if b := bunch[2]; b.id != 60 || b.refs[0] != 5 {
t.Fatal(bunch)
}
// same
bunch = mergeBunch(bunch, []IdRef{IdRef{50, []int64{0, 5}}})
if b := bunch[1]; b.id != 50 || b.refs[0] != 0 ||
b.refs[1] != 1 || b.refs[2] != 5 {
t.Fatal(bunch)
}
if len(bunch) != 4 {
t.Fatal(bunch)
}
}
func TestIdRefBunches(t *testing.T) {
bunches := make(IdRefBunches)
bunches.add(1, 100, 999)
@ -200,12 +238,6 @@ func TestIdRefBunches(t *testing.T) {
t.Fatal(bunches)
}
// same id
bunches.add(1, 100, 998)
if r := bunches[1].idRefs[0]; r.id != 100 || r.refs[0] != 998 || r.refs[1] != 999 {
t.Fatal(bunches)
}
// before
bunches.add(1, 99, 888)
if r := bunches[1].idRefs[0]; r.id != 99 || r.refs[0] != 888 {
@ -224,6 +256,12 @@ func TestIdRefBunches(t *testing.T) {
t.Fatal(bunches)
}
// same id
bunches.add(1, 100, 998)
if r := bunches[1].idRefs[1]; r.id != 100 || r.refs[0] != 998 || r.refs[1] != 999 {
t.Fatal(bunches)
}
if len(bunches) != 1 {
t.Fatal(bunches)
}

154
cache/diffbunch.go vendored
View File

@ -11,6 +11,55 @@ import (
"sync"
)
type IdRef struct {
id int64
refs []int64
}
// IdRefBunch stores multiple IdRefs
type IdRefBunch struct {
id int64 // the bunch id
idRefs []IdRef
}
// IdRefBunches can hold multiple IdRefBunch
type IdRefBunches map[int64]IdRefBunch
func (bunches *IdRefBunches) add(bunchId, id, ref int64) {
bunch, ok := (*bunches)[bunchId]
if !ok {
bunch = IdRefBunch{id: bunchId}
}
var idRef *IdRef
i := sort.Search(len(bunch.idRefs), func(i int) bool {
return bunch.idRefs[i].id >= id
})
if i < len(bunch.idRefs) && bunch.idRefs[i].id >= id {
if bunch.idRefs[i].id == id {
idRef = &bunch.idRefs[i]
} else {
bunch.idRefs = append(bunch.idRefs, IdRef{})
copy(bunch.idRefs[i+1:], bunch.idRefs[i:])
bunch.idRefs[i] = IdRef{id: id}
idRef = &bunch.idRefs[i]
}
} else {
bunch.idRefs = append(bunch.idRefs, IdRef{id: id})
idRef = &bunch.idRefs[len(bunch.idRefs)-1]
}
idRef.refs = insertRefs(idRef.refs, ref)
(*bunches)[bunchId] = bunch
}
var IdRefBunchesPool chan IdRefBunches
func init() {
IdRefBunchesPool = make(chan IdRefBunches, 1)
}
// BunchRefCache
type BunchRefCache struct {
Cache
cache IdRefBunches
@ -21,24 +70,6 @@ type BunchRefCache struct {
waitWrite *sync.WaitGroup
}
var IdRefBunchesCache chan IdRefBunches
func init() {
IdRefBunchesCache = make(chan IdRefBunches, 1)
}
type IdRef struct {
id int64
refs []int64
}
type IdRefBunch struct {
id int64
idRefs []IdRef
}
type IdRefBunches map[int64]IdRefBunch
func NewBunchRefCache(path string, opts *CacheOptions) (*BunchRefCache, error) {
index := BunchRefCache{}
index.options = opts
@ -83,7 +114,7 @@ func (index *BunchRefCache) dispatch() {
if len(index.cache) >= cacheSize {
index.write <- index.cache
select {
case index.cache = <-IdRefBunchesCache:
case index.cache = <-IdRefBunchesPool:
default:
index.cache = make(IdRefBunches, cacheSize)
}
@ -106,34 +137,6 @@ func (index *BunchRefCache) getBunchId(id int64) int64 {
return id / 64
}
func (bunches *IdRefBunches) add(bunchId, id, ref int64) {
bunch, ok := (*bunches)[bunchId]
if !ok {
bunch = IdRefBunch{id: bunchId}
}
var idRef *IdRef
i := sort.Search(len(bunch.idRefs), func(i int) bool {
return bunch.idRefs[i].id >= id
})
if i < len(bunch.idRefs) && bunch.idRefs[i].id >= id {
if bunch.idRefs[i].id == id {
idRef = &bunch.idRefs[i]
} else {
bunch.idRefs = append(bunch.idRefs, IdRef{})
copy(bunch.idRefs[i+1:], bunch.idRefs[i:])
bunch.idRefs[i] = IdRef{id: id}
idRef = &bunch.idRefs[i]
}
} else {
bunch.idRefs = append(bunch.idRefs, IdRef{id: id})
idRef = &bunch.idRefs[len(bunch.idRefs)-1]
}
idRef.refs = insertRefs(idRef.refs, ref)
(*bunches)[bunchId] = bunch
}
type loadBunchItem struct {
bunchId int64
bunch IdRefBunch
@ -184,12 +187,44 @@ func (index *BunchRefCache) writeRefs(idRefs IdRefBunches) error {
delete(idRefs, k)
}
select {
case IdRefBunchesCache <- idRefs:
case IdRefBunchesPool <- idRefs:
}
}()
return index.db.Write(index.wo, batch)
}
func mergeBunch(bunch, newBunch []IdRef) []IdRef {
lastIdx := 0
NextIdRef:
// for each new IdRef...
for _, newIdRefs := range newBunch {
// search place in bunch
for i := lastIdx; i < len(bunch); i++ {
if bunch[i].id == newIdRefs.id {
// id already present, add refs
for _, r := range newIdRefs.refs {
bunch[i].refs = insertRefs(bunch[i].refs, r)
}
lastIdx = i
break NextIdRef
}
if bunch[i].id > newIdRefs.id {
// insert before
bunch = append(bunch, IdRef{})
copy(bunch[i+1:], bunch[i:])
bunch[i] = newIdRefs
lastIdx = i
break NextIdRef
}
}
// insert at the end
bunch = append(bunch, newIdRefs)
lastIdx = len(bunch) - 1
}
return bunch
}
func (index *BunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []IdRef) []byte {
data, err := index.db.Get(index.ro, keyBuf)
if err != nil {
@ -205,26 +240,7 @@ func (index *BunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []IdRef) []
if bunch == nil {
bunch = newBunch
} else {
var last int
for _, newIdRefs := range newBunch {
for i := last; i < len(bunch); i++ {
if bunch[i].id == newIdRefs.id {
for _, r := range newIdRefs.refs {
bunch[i].refs = insertRefs(bunch[i].refs, r)
}
last = i
break
}
if bunch[i].id >= newIdRefs.id {
// insert before
bunch = append(bunch, IdRef{})
copy(bunch[i+1:], bunch[i:])
bunch[i] = newIdRefs
last = i
break
}
}
}
bunch = mergeBunch(bunch, newBunch)
}
data = MarshalBunch(bunch)