From d6fcfc6c236e5251aeed1a4a379a7bb1581d3712 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Tue, 6 Aug 2013 17:22:31 +0200 Subject: [PATCH] more refactoring --- cache/binary/diff_test.go | 4 +- cache/diff.go | 327 ++++++++++++++++++-------------------- cache/diff_test.go | 40 ----- element/element.go | 26 +++ element/element_test.go | 69 ++++++++ 5 files changed, 250 insertions(+), 216 deletions(-) create mode 100644 element/element_test.go diff --git a/cache/binary/diff_test.go b/cache/binary/diff_test.go index 43f9a80..5241fd6 100644 --- a/cache/binary/diff_test.go +++ b/cache/binary/diff_test.go @@ -30,7 +30,7 @@ func TestmarshalBunch(t *testing.T) { if newBunch[1].Id != 123923133 || newBunch[1].Refs[0] != 1231237 { t.Fatal(newBunch[1]) } - if newBunch[2].Id != 123924123 || newBunch[2].Refs[0] != 912412210 || newBunch[2].refs[1] != 912412213 { + if newBunch[2].Id != 123924123 || newBunch[2].Refs[0] != 912412210 || newBunch[2].Refs[1] != 912412213 { t.Fatal(newBunch[2]) } if newBunch[5].Id != 123924132 || newBunch[5].Refs[2] != 212412210 { @@ -39,7 +39,7 @@ func TestmarshalBunch(t *testing.T) { } func BenchmarkMarshalBunch(b *testing.B) { - bunch := []idRefs{ + bunch := []element.IdRefs{ {123923123, []int64{1213123}}, {123923133, []int64{1231237}}, {123924123, []int64{912412210, 912412213}}, diff --git a/cache/diff.go b/cache/diff.go index 6cbf91e..ca05c1c 100644 --- a/cache/diff.go +++ b/cache/diff.go @@ -86,32 +86,6 @@ func (c *DiffCache) Remove() error { const bufferSize = 64 * 1024 -func insertRefs(refs []int64, ref int64) []int64 { - i := sort.Search(len(refs), func(i int) bool { - return refs[i] >= ref - }) - if i < len(refs) && refs[i] >= ref { - if refs[i] > ref { - refs = append(refs, 0) - copy(refs[i+1:], refs[i:]) - refs[i] = ref - } // else already inserted - } else { - refs = append(refs, ref) - } - return refs -} - -func deleteRefs(refs []int64, ref int64) []int64 { - i := sort.Search(len(refs), func(i int) bool { - return refs[i] >= ref - }) - if i < len(refs) && refs[i] == ref { - refs = append(refs[:i], refs[i+1:]...) - } - return refs -} - type idRef struct { id int64 ref int64 @@ -128,7 +102,7 @@ type idRefBunches map[int64]idRefBunch func (bunches *idRefBunches) add(bunchId, id, ref int64) { idRefs := bunches.getCreate(bunchId, id) - idRefs.Refs = insertRefs(idRefs.Refs, ref) + idRefs.Add(ref) } func (bunches *idRefBunches) getCreate(bunchId, id int64) *element.IdRefs { @@ -212,28 +186,6 @@ func newRefIndex(path string, opts *cacheOptions) (*bunchRefCache, error) { return &index, nil } -func (index *bunchRefCache) SetLinearImport(val bool) { - if val != !index.linearImport { - panic("programming error, linear import already set") - } - if val { - index.waitWrite.Add(1) - index.waitAdd.Add(1) - - go index.writer() - go index.dispatch() - - index.linearImport = true - } else { - close(index.addc) - index.waitAdd.Wait() - close(index.write) - index.waitWrite.Wait() - - index.linearImport = false - } -} - type CoordsRefIndex struct { bunchRefCache } @@ -270,6 +222,155 @@ func (index *bunchRefCache) Close() { index.cache.Close() } +func (index *bunchRefCache) Get(id int64) []int64 { + if index.linearImport { + panic("programming error: get not supported in linearImport mode") + } + keyBuf := idToKeyBuf(index.getBunchId(id)) + + data, err := index.db.Get(index.ro, keyBuf) + if err != nil { + panic(err) + } + + if data != nil { + for _, idRef := range binary.UnmarshalIdRefsBunch(data) { + if idRef.Id == id { + return idRef.Refs + } + } + } + return nil +} + +func (index *bunchRefCache) Add(id, ref int64) error { + keyBuf := idToKeyBuf(index.getBunchId(id)) + + data, err := index.db.Get(index.ro, keyBuf) + if err != nil { + return err + } + + var idRefs []element.IdRefs + if data != nil { + idRefs = binary.UnmarshalIdRefsBunch(data) + } + + idRefBunch := idRefBunch{index.getBunchId(id), idRefs} + idRef := idRefBunch.getCreate(id) + idRef.Add(ref) + + data = binary.MarshalIdRefsBunch(idRefBunch.idRefs) + + return index.db.Put(index.wo, keyBuf, data) +} + +func (index *bunchRefCache) DeleteRef(id, ref int64) error { + if index.linearImport { + panic("programming error: delete not supported in linearImport mode") + } + + keyBuf := idToKeyBuf(index.getBunchId(id)) + + data, err := index.db.Get(index.ro, keyBuf) + if err != nil { + return err + } + + if data != nil { + idRefs := binary.UnmarshalIdRefsBunch(data) + idRefBunch := idRefBunch{index.getBunchId(id), idRefs} + idRef := idRefBunch.get(id) + if idRef != nil { + idRef.Delete(ref) + data := binary.MarshalIdRefsBunch(idRefs) + return index.db.Put(index.wo, keyBuf, data) + } + } + return nil +} + +func (index *bunchRefCache) Delete(id int64) error { + if index.linearImport { + panic("programming error: delete not supported in linearImport mode") + } + + keyBuf := idToKeyBuf(index.getBunchId(id)) + + data, err := index.db.Get(index.ro, keyBuf) + if err != nil { + return err + } + + if data != nil { + idRefs := binary.UnmarshalIdRefsBunch(data) + idRefBunch := idRefBunch{index.getBunchId(id), idRefs} + idRef := idRefBunch.get(id) + if idRef != nil { + idRef.Refs = []int64{} + data := binary.MarshalIdRefsBunch(idRefs) + return index.db.Put(index.wo, keyBuf, data) + } + } + return nil +} + +func (index *CoordsRefIndex) AddFromWay(way *element.Way) { + for _, node := range way.Nodes { + if index.linearImport { + index.addc <- idRef{id: node.Id, ref: way.Id} + } else { + index.Add(node.Id, way.Id) + } + } +} + +func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) { + if index.linearImport { + panic("programming error: delete not supported in linearImport mode") + } + for _, node := range way.Nodes { + index.DeleteRef(node.Id, way.Id) + } +} + +func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) { + for _, member := range members { + if member.Type == element.WAY { + if index.linearImport { + index.addc <- idRef{id: member.Id, ref: relId} + } else { + index.Add(member.Id, relId) + } + } + } +} + +// SetLinearImport optimizes the cache for write operations. +// Get/Delete operations will panic during linear import. +func (index *bunchRefCache) SetLinearImport(val bool) { + if val == index.linearImport { + // already in this mode + return + } + if val { + index.waitWrite.Add(1) + index.waitAdd.Add(1) + + go index.writer() + go index.dispatch() + + index.linearImport = true + } else { + close(index.addc) + index.waitAdd.Wait() + close(index.write) + index.waitWrite.Wait() + + index.linearImport = false + } +} + func (index *bunchRefCache) writer() { for buffer := range index.write { if err := index.writeRefs(buffer); err != nil { @@ -369,7 +470,7 @@ NextIdRef: bunch = append(bunch[:i], bunch[i+1:]...) } else { // otherwise add refs for _, r := range newIdRefs.Refs { - bunch[i].Refs = insertRefs(bunch[i].Refs, r) + bunch[i].Add(r) } } lastIdx = i @@ -395,6 +496,8 @@ NextIdRef: return bunch } +// loadMergeMarshal loads an existing bunch, merges the IdRefs and +// marshals the result again. func (index *bunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []element.IdRefs) []byte { data, err := index.db.Get(index.ro, keyBuf) if err != nil { @@ -416,127 +519,3 @@ func (index *bunchRefCache) loadMergeMarshal(keyBuf []byte, newBunch []element.I data = binary.MarshalIdRefsBunch(bunch) return data } - -func (index *bunchRefCache) Get(id int64) []int64 { - if index.linearImport { - panic("programming error: get not supported in linearImport mode") - } - keyBuf := idToKeyBuf(index.getBunchId(id)) - - data, err := index.db.Get(index.ro, keyBuf) - if err != nil { - panic(err) - } - - if data != nil { - for _, idRef := range binary.UnmarshalIdRefsBunch(data) { - if idRef.Id == id { - return idRef.Refs - } - } - } - return nil -} - -func (index *bunchRefCache) Add(id, ref int64) error { - keyBuf := idToKeyBuf(index.getBunchId(id)) - - data, err := index.db.Get(index.ro, keyBuf) - if err != nil { - return err - } - - var idRefs []element.IdRefs - if data != nil { - idRefs = binary.UnmarshalIdRefsBunch(data) - } - - idRefBunch := idRefBunch{index.getBunchId(id), idRefs} - idRef := idRefBunch.getCreate(id) - idRef.Refs = insertRefs(idRef.Refs, ref) - - data = binary.MarshalIdRefsBunch(idRefBunch.idRefs) - - return index.db.Put(index.wo, keyBuf, data) -} - -func (index *bunchRefCache) DeleteRef(id, ref int64) error { - if index.linearImport { - panic("programming error: delete not supported in linearImport mode") - } - - keyBuf := idToKeyBuf(index.getBunchId(id)) - - data, err := index.db.Get(index.ro, keyBuf) - if err != nil { - return err - } - - if data != nil { - idRefs := binary.UnmarshalIdRefsBunch(data) - idRefBunch := idRefBunch{index.getBunchId(id), idRefs} - idRef := idRefBunch.get(id) - if idRef != nil { - idRef.Refs = deleteRefs(idRef.Refs, ref) - data := binary.MarshalIdRefsBunch(idRefs) - return index.db.Put(index.wo, keyBuf, data) - } - } - return nil -} - -func (index *bunchRefCache) Delete(id int64) error { - if index.linearImport { - panic("programming error: delete not supported in linearImport mode") - } - - keyBuf := idToKeyBuf(index.getBunchId(id)) - - data, err := index.db.Get(index.ro, keyBuf) - if err != nil { - return err - } - - if data != nil { - idRefs := binary.UnmarshalIdRefsBunch(data) - idRefBunch := idRefBunch{index.getBunchId(id), idRefs} - idRef := idRefBunch.get(id) - if idRef != nil { - idRef.Refs = []int64{} - data := binary.MarshalIdRefsBunch(idRefs) - return index.db.Put(index.wo, keyBuf, data) - } - } - return nil -} - -func (index *CoordsRefIndex) AddFromWay(way *element.Way) { - for _, node := range way.Nodes { - if index.linearImport { - index.addc <- idRef{id: node.Id, ref: way.Id} - } else { - index.Add(node.Id, way.Id) - } - } -} - -func (index *CoordsRefIndex) DeleteFromWay(way *element.Way) { - if index.linearImport { - panic("programming error: delete not supported in linearImport mode") - } - for _, node := range way.Nodes { - index.DeleteRef(node.Id, way.Id) - } -} - -func (index *WaysRefIndex) AddFromMembers(relId int64, members []element.Member) { - for _, member := range members { - if member.Type == element.WAY { - if index.linearImport { - index.addc <- idRef{id: member.Id, ref: relId} - } else { - index.Add(member.Id, relId) - } - } - } -} diff --git a/cache/diff_test.go b/cache/diff_test.go index d79279d..d59351a 100644 --- a/cache/diff_test.go +++ b/cache/diff_test.go @@ -8,46 +8,6 @@ import ( "goposm/element" ) -func TestInsertRefs(t *testing.T) { - - refs := make([]int64, 0, 1) - - refs = insertRefs(refs, 1) - if refs[0] != 1 { - t.Fatal(refs) - } - - refs = insertRefs(refs, 10) - if refs[0] != 1 || refs[1] != 10 { - t.Fatal(refs) - } - - // insert twice - refs = insertRefs(refs, 10) - if refs[0] != 1 || refs[1] != 10 || len(refs) != 2 { - t.Fatal(refs) - } - - // insert before - refs = insertRefs(refs, 0) - if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 { - t.Fatal(refs) - } - - // insert after - refs = insertRefs(refs, 12) - if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 || refs[3] != 12 { - t.Fatal(refs) - } - - // insert between - refs = insertRefs(refs, 11) - if refs[0] != 0 || refs[1] != 1 || refs[2] != 10 || refs[3] != 11 || refs[4] != 12 { - t.Fatal(refs) - } - -} - func TestDiffCache(t *testing.T) { cache_dir, _ := ioutil.TempDir("", "goposm_test") diff --git a/element/element.go b/element/element.go index 915bb29..f21ab6b 100644 --- a/element/element.go +++ b/element/element.go @@ -1,6 +1,8 @@ package element import ( + "sort" + "goposm/geom/geos" ) @@ -63,3 +65,27 @@ type IdRefs struct { Id int64 Refs []int64 } + +func (idRefs *IdRefs) Add(ref int64) { + i := sort.Search(len(idRefs.Refs), func(i int) bool { + return idRefs.Refs[i] >= ref + }) + if i < len(idRefs.Refs) && idRefs.Refs[i] >= ref { + if idRefs.Refs[i] > ref { + idRefs.Refs = append(idRefs.Refs, 0) + copy(idRefs.Refs[i+1:], idRefs.Refs[i:]) + idRefs.Refs[i] = ref + } // else already inserted + } else { + idRefs.Refs = append(idRefs.Refs, ref) + } +} + +func (idRefs *IdRefs) Delete(ref int64) { + i := sort.Search(len(idRefs.Refs), func(i int) bool { + return idRefs.Refs[i] >= ref + }) + if i < len(idRefs.Refs) && idRefs.Refs[i] == ref { + idRefs.Refs = append(idRefs.Refs[:i], idRefs.Refs[i+1:]...) + } +} diff --git a/element/element_test.go b/element/element_test.go new file mode 100644 index 0000000..832118e --- /dev/null +++ b/element/element_test.go @@ -0,0 +1,69 @@ +package element + +import ( + "testing" +) + +func TestIdRefs(t *testing.T) { + + idRefs := IdRefs{} + + idRefs.Add(1) + if idRefs.Refs[0] != 1 { + t.Fatal(idRefs) + } + + idRefs.Add(10) + if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 { + t.Fatal(idRefs) + } + + // insert twice + idRefs.Add(10) + if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 || len(idRefs.Refs) != 2 { + t.Fatal(idRefs) + } + + // insert before + idRefs.Add(0) + if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 { + t.Fatal(idRefs) + } + + // insert after + idRefs.Add(12) + if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 12 { + t.Fatal(idRefs) + } + + // insert between + idRefs.Add(11) + if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 11 || idRefs.Refs[4] != 12 { + t.Fatal(idRefs) + } + + // delete between + idRefs.Delete(11) + if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 || idRefs.Refs[3] != 12 { + t.Fatal(idRefs) + } + + // delete end + idRefs.Delete(12) + if idRefs.Refs[0] != 0 || idRefs.Refs[1] != 1 || idRefs.Refs[2] != 10 { + t.Fatal(idRefs) + } + + // delete begin + idRefs.Delete(0) + if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 { + t.Fatal(idRefs) + } + + // delete missing + idRefs.Delete(99) + if idRefs.Refs[0] != 1 || idRefs.Refs[1] != 10 || len(idRefs.Refs) != 2 { + t.Fatal(idRefs) + } + +}