parent
318e9c766f
commit
2b9bfda1d5
|
@ -1,7 +0,0 @@
|
||||||
package bolt
|
|
||||||
|
|
||||||
// maxMapSize represents the largest mmap size supported by Bolt.
|
|
||||||
const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
|
||||||
const maxAllocSize = 0x7FFFFFFF
|
|
|
@ -1,7 +0,0 @@
|
||||||
package bolt
|
|
||||||
|
|
||||||
// maxMapSize represents the largest mmap size supported by Bolt.
|
|
||||||
const maxMapSize = 0x7FFFFFFF // 2GB
|
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
|
||||||
const maxAllocSize = 0xFFFFFFF
|
|
|
@ -5,3 +5,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
const maxAllocSize = 0xFFFFFFF
|
const maxAllocSize = 0xFFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -0,0 +1,10 @@
|
||||||
|
package bolt
|
||||||
|
|
||||||
|
// maxMapSize represents the largest mmap size supported by Bolt.
|
||||||
|
const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||||
|
|
||||||
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
|
const maxAllocSize = 0x7FFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -0,0 +1,28 @@
|
||||||
|
package bolt
|
||||||
|
|
||||||
|
import "unsafe"
|
||||||
|
|
||||||
|
// maxMapSize represents the largest mmap size supported by Bolt.
|
||||||
|
const maxMapSize = 0x7FFFFFFF // 2GB
|
||||||
|
|
||||||
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
|
const maxAllocSize = 0xFFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned bool
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Simple check to see whether this arch handles unaligned load/stores
|
||||||
|
// correctly.
|
||||||
|
|
||||||
|
// ARM9 and older devices require load/stores to be from/to aligned
|
||||||
|
// addresses. If not, the lower 2 bits are cleared and that address is
|
||||||
|
// read in a jumbled up order.
|
||||||
|
|
||||||
|
// See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html
|
||||||
|
|
||||||
|
raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11}
|
||||||
|
val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2))
|
||||||
|
|
||||||
|
brokenUnaligned = val != 0x11222211
|
||||||
|
}
|
|
@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
const maxAllocSize = 0x7FFFFFFF
|
const maxAllocSize = 0x7FFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
const maxAllocSize = 0x7FFFFFFF
|
const maxAllocSize = 0x7FFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
const maxAllocSize = 0x7FFFFFFF
|
const maxAllocSize = 0x7FFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
||||||
|
|
||||||
// maxAllocSize is the size used when creating array pointers.
|
// maxAllocSize is the size used when creating array pointers.
|
||||||
const maxAllocSize = 0x7FFFFFFF
|
const maxAllocSize = 0x7FFFFFFF
|
||||||
|
|
||||||
|
// Are unaligned load/stores broken on this arch?
|
||||||
|
var brokenUnaligned = false
|
|
@ -89,7 +89,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro
|
||||||
func funlock(db *DB) error {
|
func funlock(db *DB) error {
|
||||||
err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{})
|
err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{})
|
||||||
db.lockfile.Close()
|
db.lockfile.Close()
|
||||||
os.Remove(db.path+lockExt)
|
os.Remove(db.path + lockExt)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -130,9 +130,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket {
|
||||||
func (b *Bucket) openBucket(value []byte) *Bucket {
|
func (b *Bucket) openBucket(value []byte) *Bucket {
|
||||||
var child = newBucket(b.tx)
|
var child = newBucket(b.tx)
|
||||||
|
|
||||||
|
// If unaligned load/stores are broken on this arch and value is
|
||||||
|
// unaligned simply clone to an aligned byte array.
|
||||||
|
unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0
|
||||||
|
|
||||||
|
if unaligned {
|
||||||
|
value = cloneBytes(value)
|
||||||
|
}
|
||||||
|
|
||||||
// If this is a writable transaction then we need to copy the bucket entry.
|
// If this is a writable transaction then we need to copy the bucket entry.
|
||||||
// Read-only transactions can point directly at the mmap entry.
|
// Read-only transactions can point directly at the mmap entry.
|
||||||
if b.tx.writable {
|
if b.tx.writable && !unaligned {
|
||||||
child.bucket = &bucket{}
|
child.bucket = &bucket{}
|
||||||
*child.bucket = *(*bucket)(unsafe.Pointer(&value[0]))
|
*child.bucket = *(*bucket)(unsafe.Pointer(&value[0]))
|
||||||
} else {
|
} else {
|
||||||
|
@ -167,9 +175,8 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) {
|
||||||
if bytes.Equal(key, k) {
|
if bytes.Equal(key, k) {
|
||||||
if (flags & bucketLeafFlag) != 0 {
|
if (flags & bucketLeafFlag) != 0 {
|
||||||
return nil, ErrBucketExists
|
return nil, ErrBucketExists
|
||||||
} else {
|
|
||||||
return nil, ErrIncompatibleValue
|
|
||||||
}
|
}
|
||||||
|
return nil, ErrIncompatibleValue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create empty, inline bucket.
|
// Create empty, inline bucket.
|
||||||
|
@ -329,6 +336,28 @@ func (b *Bucket) Delete(key []byte) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sequence returns the current integer for the bucket without incrementing it.
|
||||||
|
func (b *Bucket) Sequence() uint64 { return b.bucket.sequence }
|
||||||
|
|
||||||
|
// SetSequence updates the sequence number for the bucket.
|
||||||
|
func (b *Bucket) SetSequence(v uint64) error {
|
||||||
|
if b.tx.db == nil {
|
||||||
|
return ErrTxClosed
|
||||||
|
} else if !b.Writable() {
|
||||||
|
return ErrTxNotWritable
|
||||||
|
}
|
||||||
|
|
||||||
|
// Materialize the root node if it hasn't been already so that the
|
||||||
|
// bucket will be saved during commit.
|
||||||
|
if b.rootNode == nil {
|
||||||
|
_ = b.node(b.root, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Increment and return the sequence.
|
||||||
|
b.bucket.sequence = v
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// NextSequence returns an autoincrementing integer for the bucket.
|
// NextSequence returns an autoincrementing integer for the bucket.
|
||||||
func (b *Bucket) NextSequence() (uint64, error) {
|
func (b *Bucket) NextSequence() (uint64, error) {
|
||||||
if b.tx.db == nil {
|
if b.tx.db == nil {
|
96
cmd/vendor/github.com/boltdb/bolt/db.go → cmd/vendor/github.com/coreos/bbolt/db.go
generated
vendored
96
cmd/vendor/github.com/boltdb/bolt/db.go → cmd/vendor/github.com/coreos/bbolt/db.go
generated
vendored
|
@ -8,6 +8,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
@ -61,6 +62,11 @@ type DB struct {
|
||||||
// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
|
// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
|
||||||
NoSync bool
|
NoSync bool
|
||||||
|
|
||||||
|
// When true, skips syncing freelist to disk. This improves the database
|
||||||
|
// write performance under normal operation, but requires a full database
|
||||||
|
// re-sync during recovery.
|
||||||
|
NoFreelistSync bool
|
||||||
|
|
||||||
// When true, skips the truncate call when growing the database.
|
// When true, skips the truncate call when growing the database.
|
||||||
// Setting this to true is only safe on non-ext3/ext4 systems.
|
// Setting this to true is only safe on non-ext3/ext4 systems.
|
||||||
// Skipping truncation avoids preallocation of hard drive space and
|
// Skipping truncation avoids preallocation of hard drive space and
|
||||||
|
@ -156,6 +162,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
|
||||||
}
|
}
|
||||||
db.NoGrowSync = options.NoGrowSync
|
db.NoGrowSync = options.NoGrowSync
|
||||||
db.MmapFlags = options.MmapFlags
|
db.MmapFlags = options.MmapFlags
|
||||||
|
db.NoFreelistSync = options.NoFreelistSync
|
||||||
|
|
||||||
// Set default values for later DB operations.
|
// Set default values for later DB operations.
|
||||||
db.MaxBatchSize = DefaultMaxBatchSize
|
db.MaxBatchSize = DefaultMaxBatchSize
|
||||||
|
@ -232,9 +239,14 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read in the freelist.
|
if db.NoFreelistSync {
|
||||||
db.freelist = newFreelist()
|
db.freelist = newFreelist()
|
||||||
db.freelist.read(db.page(db.meta().freelist))
|
db.freelist.readIDs(db.freepages())
|
||||||
|
} else {
|
||||||
|
// Read in the freelist.
|
||||||
|
db.freelist = newFreelist()
|
||||||
|
db.freelist.read(db.page(db.meta().freelist))
|
||||||
|
}
|
||||||
|
|
||||||
// Mark the database as opened and return.
|
// Mark the database as opened and return.
|
||||||
return db, nil
|
return db, nil
|
||||||
|
@ -526,21 +538,36 @@ func (db *DB) beginRWTx() (*Tx, error) {
|
||||||
t := &Tx{writable: true}
|
t := &Tx{writable: true}
|
||||||
t.init(db)
|
t.init(db)
|
||||||
db.rwtx = t
|
db.rwtx = t
|
||||||
|
db.freePages()
|
||||||
|
return t, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Free any pages associated with closed read-only transactions.
|
// freePages releases any pages associated with closed read-only transactions.
|
||||||
var minid txid = 0xFFFFFFFFFFFFFFFF
|
func (db *DB) freePages() {
|
||||||
for _, t := range db.txs {
|
// Free all pending pages prior to earliest open transaction.
|
||||||
if t.meta.txid < minid {
|
sort.Sort(txsById(db.txs))
|
||||||
minid = t.meta.txid
|
minid := txid(0xFFFFFFFFFFFFFFFF)
|
||||||
}
|
if len(db.txs) > 0 {
|
||||||
|
minid = db.txs[0].meta.txid
|
||||||
}
|
}
|
||||||
if minid > 0 {
|
if minid > 0 {
|
||||||
db.freelist.release(minid - 1)
|
db.freelist.release(minid - 1)
|
||||||
}
|
}
|
||||||
|
// Release unused txid extents.
|
||||||
return t, nil
|
for _, t := range db.txs {
|
||||||
|
db.freelist.releaseRange(minid, t.meta.txid-1)
|
||||||
|
minid = t.meta.txid + 1
|
||||||
|
}
|
||||||
|
db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF))
|
||||||
|
// Any page both allocated and freed in an extent is safe to release.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type txsById []*Tx
|
||||||
|
|
||||||
|
func (t txsById) Len() int { return len(t) }
|
||||||
|
func (t txsById) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||||
|
func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid }
|
||||||
|
|
||||||
// removeTx removes a transaction from the database.
|
// removeTx removes a transaction from the database.
|
||||||
func (db *DB) removeTx(tx *Tx) {
|
func (db *DB) removeTx(tx *Tx) {
|
||||||
// Release the read lock on the mmap.
|
// Release the read lock on the mmap.
|
||||||
|
@ -552,7 +579,10 @@ func (db *DB) removeTx(tx *Tx) {
|
||||||
// Remove the transaction.
|
// Remove the transaction.
|
||||||
for i, t := range db.txs {
|
for i, t := range db.txs {
|
||||||
if t == tx {
|
if t == tx {
|
||||||
db.txs = append(db.txs[:i], db.txs[i+1:]...)
|
last := len(db.txs) - 1
|
||||||
|
db.txs[i] = db.txs[last]
|
||||||
|
db.txs[last] = nil
|
||||||
|
db.txs = db.txs[:last]
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -823,7 +853,7 @@ func (db *DB) meta() *meta {
|
||||||
}
|
}
|
||||||
|
|
||||||
// allocate returns a contiguous block of memory starting at a given page.
|
// allocate returns a contiguous block of memory starting at a given page.
|
||||||
func (db *DB) allocate(count int) (*page, error) {
|
func (db *DB) allocate(txid txid, count int) (*page, error) {
|
||||||
// Allocate a temporary buffer for the page.
|
// Allocate a temporary buffer for the page.
|
||||||
var buf []byte
|
var buf []byte
|
||||||
if count == 1 {
|
if count == 1 {
|
||||||
|
@ -835,7 +865,7 @@ func (db *DB) allocate(count int) (*page, error) {
|
||||||
p.overflow = uint32(count - 1)
|
p.overflow = uint32(count - 1)
|
||||||
|
|
||||||
// Use pages from the freelist if they are available.
|
// Use pages from the freelist if they are available.
|
||||||
if p.id = db.freelist.allocate(count); p.id != 0 {
|
if p.id = db.freelist.allocate(txid, count); p.id != 0 {
|
||||||
return p, nil
|
return p, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -890,6 +920,38 @@ func (db *DB) IsReadOnly() bool {
|
||||||
return db.readOnly
|
return db.readOnly
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (db *DB) freepages() []pgid {
|
||||||
|
tx, err := db.beginTx()
|
||||||
|
defer func() {
|
||||||
|
err = tx.Rollback()
|
||||||
|
if err != nil {
|
||||||
|
panic("freepages: failed to rollback tx")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
panic("freepages: failed to open read only tx")
|
||||||
|
}
|
||||||
|
|
||||||
|
reachable := make(map[pgid]*page)
|
||||||
|
nofreed := make(map[pgid]bool)
|
||||||
|
ech := make(chan error)
|
||||||
|
go func() {
|
||||||
|
for e := range ech {
|
||||||
|
panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
tx.checkBucket(&tx.root, reachable, nofreed, ech)
|
||||||
|
close(ech)
|
||||||
|
|
||||||
|
var fids []pgid
|
||||||
|
for i := pgid(2); i < db.meta().pgid; i++ {
|
||||||
|
if _, ok := reachable[i]; !ok {
|
||||||
|
fids = append(fids, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fids
|
||||||
|
}
|
||||||
|
|
||||||
// Options represents the options that can be set when opening a database.
|
// Options represents the options that can be set when opening a database.
|
||||||
type Options struct {
|
type Options struct {
|
||||||
// Timeout is the amount of time to wait to obtain a file lock.
|
// Timeout is the amount of time to wait to obtain a file lock.
|
||||||
|
@ -900,6 +962,10 @@ type Options struct {
|
||||||
// Sets the DB.NoGrowSync flag before memory mapping the file.
|
// Sets the DB.NoGrowSync flag before memory mapping the file.
|
||||||
NoGrowSync bool
|
NoGrowSync bool
|
||||||
|
|
||||||
|
// Do not sync freelist to disk. This improves the database write performance
|
||||||
|
// under normal operation, but requires a full database re-sync during recovery.
|
||||||
|
NoFreelistSync bool
|
||||||
|
|
||||||
// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
|
// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
|
||||||
// grab a shared lock (UNIX).
|
// grab a shared lock (UNIX).
|
||||||
ReadOnly bool
|
ReadOnly bool
|
||||||
|
@ -952,7 +1018,7 @@ func (s *Stats) Sub(other *Stats) Stats {
|
||||||
diff.PendingPageN = s.PendingPageN
|
diff.PendingPageN = s.PendingPageN
|
||||||
diff.FreeAlloc = s.FreeAlloc
|
diff.FreeAlloc = s.FreeAlloc
|
||||||
diff.FreelistInuse = s.FreelistInuse
|
diff.FreelistInuse = s.FreelistInuse
|
||||||
diff.TxN = other.TxN - s.TxN
|
diff.TxN = s.TxN - other.TxN
|
||||||
diff.TxStats = s.TxStats.Sub(&other.TxStats)
|
diff.TxStats = s.TxStats.Sub(&other.TxStats)
|
||||||
return diff
|
return diff
|
||||||
}
|
}
|
0
cmd/vendor/github.com/boltdb/bolt/doc.go → cmd/vendor/github.com/coreos/bbolt/doc.go
generated
vendored
0
cmd/vendor/github.com/boltdb/bolt/doc.go → cmd/vendor/github.com/coreos/bbolt/doc.go
generated
vendored
|
@ -6,25 +6,41 @@ import (
|
||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// txPending holds a list of pgids and corresponding allocation txns
|
||||||
|
// that are pending to be freed.
|
||||||
|
type txPending struct {
|
||||||
|
ids []pgid
|
||||||
|
alloctx []txid // txids allocating the ids
|
||||||
|
lastReleaseBegin txid // beginning txid of last matching releaseRange
|
||||||
|
}
|
||||||
|
|
||||||
// freelist represents a list of all pages that are available for allocation.
|
// freelist represents a list of all pages that are available for allocation.
|
||||||
// It also tracks pages that have been freed but are still in use by open transactions.
|
// It also tracks pages that have been freed but are still in use by open transactions.
|
||||||
type freelist struct {
|
type freelist struct {
|
||||||
ids []pgid // all free and available free page ids.
|
ids []pgid // all free and available free page ids.
|
||||||
pending map[txid][]pgid // mapping of soon-to-be free page ids by tx.
|
allocs map[pgid]txid // mapping of txid that allocated a pgid.
|
||||||
cache map[pgid]bool // fast lookup of all free and pending page ids.
|
pending map[txid]*txPending // mapping of soon-to-be free page ids by tx.
|
||||||
|
cache map[pgid]bool // fast lookup of all free and pending page ids.
|
||||||
}
|
}
|
||||||
|
|
||||||
// newFreelist returns an empty, initialized freelist.
|
// newFreelist returns an empty, initialized freelist.
|
||||||
func newFreelist() *freelist {
|
func newFreelist() *freelist {
|
||||||
return &freelist{
|
return &freelist{
|
||||||
pending: make(map[txid][]pgid),
|
allocs: make(map[pgid]txid),
|
||||||
|
pending: make(map[txid]*txPending),
|
||||||
cache: make(map[pgid]bool),
|
cache: make(map[pgid]bool),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// size returns the size of the page after serialization.
|
// size returns the size of the page after serialization.
|
||||||
func (f *freelist) size() int {
|
func (f *freelist) size() int {
|
||||||
return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count())
|
n := f.count()
|
||||||
|
if n >= 0xFFFF {
|
||||||
|
// The first element will be used to store the count. See freelist.write.
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n)
|
||||||
}
|
}
|
||||||
|
|
||||||
// count returns count of pages on the freelist
|
// count returns count of pages on the freelist
|
||||||
|
@ -40,27 +56,26 @@ func (f *freelist) free_count() int {
|
||||||
// pending_count returns count of pending pages
|
// pending_count returns count of pending pages
|
||||||
func (f *freelist) pending_count() int {
|
func (f *freelist) pending_count() int {
|
||||||
var count int
|
var count int
|
||||||
for _, list := range f.pending {
|
for _, txp := range f.pending {
|
||||||
count += len(list)
|
count += len(txp.ids)
|
||||||
}
|
}
|
||||||
return count
|
return count
|
||||||
}
|
}
|
||||||
|
|
||||||
// all returns a list of all free ids and all pending ids in one sorted list.
|
// copyall copies into dst a list of all free ids and all pending ids in one sorted list.
|
||||||
func (f *freelist) all() []pgid {
|
// f.count returns the minimum length required for dst.
|
||||||
m := make(pgids, 0)
|
func (f *freelist) copyall(dst []pgid) {
|
||||||
|
m := make(pgids, 0, f.pending_count())
|
||||||
for _, list := range f.pending {
|
for _, txp := range f.pending {
|
||||||
m = append(m, list...)
|
m = append(m, txp.ids...)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Sort(m)
|
sort.Sort(m)
|
||||||
return pgids(f.ids).merge(m)
|
mergepgids(dst, f.ids, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
// allocate returns the starting page id of a contiguous list of pages of a given size.
|
// allocate returns the starting page id of a contiguous list of pages of a given size.
|
||||||
// If a contiguous block cannot be found then 0 is returned.
|
// If a contiguous block cannot be found then 0 is returned.
|
||||||
func (f *freelist) allocate(n int) pgid {
|
func (f *freelist) allocate(txid txid, n int) pgid {
|
||||||
if len(f.ids) == 0 {
|
if len(f.ids) == 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
@ -93,7 +108,7 @@ func (f *freelist) allocate(n int) pgid {
|
||||||
for i := pgid(0); i < pgid(n); i++ {
|
for i := pgid(0); i < pgid(n); i++ {
|
||||||
delete(f.cache, initial+i)
|
delete(f.cache, initial+i)
|
||||||
}
|
}
|
||||||
|
f.allocs[initial] = txid
|
||||||
return initial
|
return initial
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -110,28 +125,73 @@ func (f *freelist) free(txid txid, p *page) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Free page and all its overflow pages.
|
// Free page and all its overflow pages.
|
||||||
var ids = f.pending[txid]
|
txp := f.pending[txid]
|
||||||
|
if txp == nil {
|
||||||
|
txp = &txPending{}
|
||||||
|
f.pending[txid] = txp
|
||||||
|
}
|
||||||
|
allocTxid, ok := f.allocs[p.id]
|
||||||
|
if ok {
|
||||||
|
delete(f.allocs, p.id)
|
||||||
|
} else if (p.flags & (freelistPageFlag | metaPageFlag)) != 0 {
|
||||||
|
// Safe to claim txid as allocating since these types are private to txid.
|
||||||
|
allocTxid = txid
|
||||||
|
}
|
||||||
|
|
||||||
for id := p.id; id <= p.id+pgid(p.overflow); id++ {
|
for id := p.id; id <= p.id+pgid(p.overflow); id++ {
|
||||||
// Verify that page is not already free.
|
// Verify that page is not already free.
|
||||||
if f.cache[id] {
|
if f.cache[id] {
|
||||||
panic(fmt.Sprintf("page %d already freed", id))
|
panic(fmt.Sprintf("page %d already freed", id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add to the freelist and cache.
|
// Add to the freelist and cache.
|
||||||
ids = append(ids, id)
|
txp.ids = append(txp.ids, id)
|
||||||
|
txp.alloctx = append(txp.alloctx, allocTxid)
|
||||||
f.cache[id] = true
|
f.cache[id] = true
|
||||||
}
|
}
|
||||||
f.pending[txid] = ids
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// release moves all page ids for a transaction id (or older) to the freelist.
|
// release moves all page ids for a transaction id (or older) to the freelist.
|
||||||
func (f *freelist) release(txid txid) {
|
func (f *freelist) release(txid txid) {
|
||||||
m := make(pgids, 0)
|
m := make(pgids, 0)
|
||||||
for tid, ids := range f.pending {
|
for tid, txp := range f.pending {
|
||||||
if tid <= txid {
|
if tid <= txid {
|
||||||
// Move transaction's pending pages to the available freelist.
|
// Move transaction's pending pages to the available freelist.
|
||||||
// Don't remove from the cache since the page is still free.
|
// Don't remove from the cache since the page is still free.
|
||||||
m = append(m, ids...)
|
m = append(m, txp.ids...)
|
||||||
|
delete(f.pending, tid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Sort(m)
|
||||||
|
f.ids = pgids(f.ids).merge(m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// releaseRange moves pending pages allocated within an extent [begin,end] to the free list.
|
||||||
|
func (f *freelist) releaseRange(begin, end txid) {
|
||||||
|
if begin > end {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var m pgids
|
||||||
|
for tid, txp := range f.pending {
|
||||||
|
if tid < begin || tid > end {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Don't recompute freed pages if ranges haven't updated.
|
||||||
|
if txp.lastReleaseBegin == begin {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := 0; i < len(txp.ids); i++ {
|
||||||
|
if atx := txp.alloctx[i]; atx < begin || atx > end {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m = append(m, txp.ids[i])
|
||||||
|
txp.ids[i] = txp.ids[len(txp.ids)-1]
|
||||||
|
txp.ids = txp.ids[:len(txp.ids)-1]
|
||||||
|
txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1]
|
||||||
|
txp.alloctx = txp.alloctx[:len(txp.alloctx)-1]
|
||||||
|
i--
|
||||||
|
}
|
||||||
|
txp.lastReleaseBegin = begin
|
||||||
|
if len(txp.ids) == 0 {
|
||||||
delete(f.pending, tid)
|
delete(f.pending, tid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -142,12 +202,29 @@ func (f *freelist) release(txid txid) {
|
||||||
// rollback removes the pages from a given pending tx.
|
// rollback removes the pages from a given pending tx.
|
||||||
func (f *freelist) rollback(txid txid) {
|
func (f *freelist) rollback(txid txid) {
|
||||||
// Remove page ids from cache.
|
// Remove page ids from cache.
|
||||||
for _, id := range f.pending[txid] {
|
txp := f.pending[txid]
|
||||||
delete(f.cache, id)
|
if txp == nil {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
var m pgids
|
||||||
// Remove pages from pending list.
|
for i, pgid := range txp.ids {
|
||||||
|
delete(f.cache, pgid)
|
||||||
|
tx := txp.alloctx[i]
|
||||||
|
if tx == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if tx != txid {
|
||||||
|
// Pending free aborted; restore page back to alloc list.
|
||||||
|
f.allocs[pgid] = tx
|
||||||
|
} else {
|
||||||
|
// Freed page was allocated by this txn; OK to throw away.
|
||||||
|
m = append(m, pgid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Remove pages from pending list and mark as free if allocated by txid.
|
||||||
delete(f.pending, txid)
|
delete(f.pending, txid)
|
||||||
|
sort.Sort(m)
|
||||||
|
f.ids = pgids(f.ids).merge(m)
|
||||||
}
|
}
|
||||||
|
|
||||||
// freed returns whether a given page is in the free list.
|
// freed returns whether a given page is in the free list.
|
||||||
|
@ -181,27 +258,33 @@ func (f *freelist) read(p *page) {
|
||||||
f.reindex()
|
f.reindex()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// read initializes the freelist from a given list of ids.
|
||||||
|
func (f *freelist) readIDs(ids []pgid) {
|
||||||
|
f.ids = ids
|
||||||
|
f.reindex()
|
||||||
|
}
|
||||||
|
|
||||||
// write writes the page ids onto a freelist page. All free and pending ids are
|
// write writes the page ids onto a freelist page. All free and pending ids are
|
||||||
// saved to disk since in the event of a program crash, all pending ids will
|
// saved to disk since in the event of a program crash, all pending ids will
|
||||||
// become free.
|
// become free.
|
||||||
func (f *freelist) write(p *page) error {
|
func (f *freelist) write(p *page) error {
|
||||||
// Combine the old free pgids and pgids waiting on an open transaction.
|
// Combine the old free pgids and pgids waiting on an open transaction.
|
||||||
ids := f.all()
|
|
||||||
|
|
||||||
// Update the header flag.
|
// Update the header flag.
|
||||||
p.flags |= freelistPageFlag
|
p.flags |= freelistPageFlag
|
||||||
|
|
||||||
// The page.count can only hold up to 64k elements so if we overflow that
|
// The page.count can only hold up to 64k elements so if we overflow that
|
||||||
// number then we handle it by putting the size in the first element.
|
// number then we handle it by putting the size in the first element.
|
||||||
if len(ids) == 0 {
|
lenids := f.count()
|
||||||
p.count = uint16(len(ids))
|
if lenids == 0 {
|
||||||
} else if len(ids) < 0xFFFF {
|
p.count = uint16(lenids)
|
||||||
p.count = uint16(len(ids))
|
} else if lenids < 0xFFFF {
|
||||||
copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids)
|
p.count = uint16(lenids)
|
||||||
|
f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:])
|
||||||
} else {
|
} else {
|
||||||
p.count = 0xFFFF
|
p.count = 0xFFFF
|
||||||
((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids))
|
((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids)
|
||||||
copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids)
|
f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:])
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -213,8 +296,8 @@ func (f *freelist) reload(p *page) {
|
||||||
|
|
||||||
// Build a cache of only pending pages.
|
// Build a cache of only pending pages.
|
||||||
pcache := make(map[pgid]bool)
|
pcache := make(map[pgid]bool)
|
||||||
for _, pendingIDs := range f.pending {
|
for _, txp := range f.pending {
|
||||||
for _, pendingID := range pendingIDs {
|
for _, pendingID := range txp.ids {
|
||||||
pcache[pendingID] = true
|
pcache[pendingID] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -236,12 +319,12 @@ func (f *freelist) reload(p *page) {
|
||||||
|
|
||||||
// reindex rebuilds the free cache based on available and pending free lists.
|
// reindex rebuilds the free cache based on available and pending free lists.
|
||||||
func (f *freelist) reindex() {
|
func (f *freelist) reindex() {
|
||||||
f.cache = make(map[pgid]bool)
|
f.cache = make(map[pgid]bool, len(f.ids))
|
||||||
for _, id := range f.ids {
|
for _, id := range f.ids {
|
||||||
f.cache[id] = true
|
f.cache[id] = true
|
||||||
}
|
}
|
||||||
for _, pendingIDs := range f.pending {
|
for _, txp := range f.pending {
|
||||||
for _, pendingID := range pendingIDs {
|
for _, pendingID := range txp.ids {
|
||||||
f.cache[pendingID] = true
|
f.cache[pendingID] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
31
cmd/vendor/github.com/boltdb/bolt/page.go → cmd/vendor/github.com/coreos/bbolt/page.go
generated
vendored
31
cmd/vendor/github.com/boltdb/bolt/page.go → cmd/vendor/github.com/coreos/bbolt/page.go
generated
vendored
|
@ -145,12 +145,33 @@ func (a pgids) merge(b pgids) pgids {
|
||||||
// Return the opposite slice if one is nil.
|
// Return the opposite slice if one is nil.
|
||||||
if len(a) == 0 {
|
if len(a) == 0 {
|
||||||
return b
|
return b
|
||||||
} else if len(b) == 0 {
|
}
|
||||||
|
if len(b) == 0 {
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
merged := make(pgids, len(a)+len(b))
|
||||||
|
mergepgids(merged, a, b)
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
// Create a list to hold all elements from both lists.
|
// mergepgids copies the sorted union of a and b into dst.
|
||||||
merged := make(pgids, 0, len(a)+len(b))
|
// If dst is too small, it panics.
|
||||||
|
func mergepgids(dst, a, b pgids) {
|
||||||
|
if len(dst) < len(a)+len(b) {
|
||||||
|
panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b)))
|
||||||
|
}
|
||||||
|
// Copy in the opposite slice if one is nil.
|
||||||
|
if len(a) == 0 {
|
||||||
|
copy(dst, b)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(b) == 0 {
|
||||||
|
copy(dst, a)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merged will hold all elements from both lists.
|
||||||
|
merged := dst[:0]
|
||||||
|
|
||||||
// Assign lead to the slice with a lower starting value, follow to the higher value.
|
// Assign lead to the slice with a lower starting value, follow to the higher value.
|
||||||
lead, follow := a, b
|
lead, follow := a, b
|
||||||
|
@ -172,7 +193,5 @@ func (a pgids) merge(b pgids) pgids {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Append what's left in follow.
|
// Append what's left in follow.
|
||||||
merged = append(merged, follow...)
|
_ = append(merged, follow...)
|
||||||
|
|
||||||
return merged
|
|
||||||
}
|
}
|
62
cmd/vendor/github.com/boltdb/bolt/tx.go → cmd/vendor/github.com/coreos/bbolt/tx.go
generated
vendored
62
cmd/vendor/github.com/boltdb/bolt/tx.go → cmd/vendor/github.com/coreos/bbolt/tx.go
generated
vendored
|
@ -169,26 +169,9 @@ func (tx *Tx) Commit() error {
|
||||||
// Free the old root bucket.
|
// Free the old root bucket.
|
||||||
tx.meta.root.root = tx.root.root
|
tx.meta.root.root = tx.root.root
|
||||||
|
|
||||||
opgid := tx.meta.pgid
|
if !tx.db.NoFreelistSync {
|
||||||
|
err := tx.commitFreelist()
|
||||||
// Free the freelist and allocate new pages for it. This will overestimate
|
if err != nil {
|
||||||
// the size of the freelist but not underestimate the size (which would be bad).
|
|
||||||
tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
|
|
||||||
p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
|
|
||||||
if err != nil {
|
|
||||||
tx.rollback()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := tx.db.freelist.write(p); err != nil {
|
|
||||||
tx.rollback()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
tx.meta.freelist = p.id
|
|
||||||
|
|
||||||
// If the high water mark has moved up then attempt to grow the database.
|
|
||||||
if tx.meta.pgid > opgid {
|
|
||||||
if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
|
|
||||||
tx.rollback()
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -235,6 +218,33 @@ func (tx *Tx) Commit() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (tx *Tx) commitFreelist() error {
|
||||||
|
opgid := tx.meta.pgid
|
||||||
|
|
||||||
|
// Free the freelist and allocate new pages for it. This will overestimate
|
||||||
|
// the size of the freelist but not underestimate the size (which would be bad).
|
||||||
|
tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist))
|
||||||
|
p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1)
|
||||||
|
if err != nil {
|
||||||
|
tx.rollback()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tx.db.freelist.write(p); err != nil {
|
||||||
|
tx.rollback()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tx.meta.freelist = p.id
|
||||||
|
// If the high water mark has moved up then attempt to grow the database.
|
||||||
|
if tx.meta.pgid > opgid {
|
||||||
|
if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil {
|
||||||
|
tx.rollback()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Rollback closes the transaction and ignores all previous updates. Read-only
|
// Rollback closes the transaction and ignores all previous updates. Read-only
|
||||||
// transactions must be rolled back and not committed.
|
// transactions must be rolled back and not committed.
|
||||||
func (tx *Tx) Rollback() error {
|
func (tx *Tx) Rollback() error {
|
||||||
|
@ -381,7 +391,9 @@ func (tx *Tx) Check() <-chan error {
|
||||||
func (tx *Tx) check(ch chan error) {
|
func (tx *Tx) check(ch chan error) {
|
||||||
// Check if any pages are double freed.
|
// Check if any pages are double freed.
|
||||||
freed := make(map[pgid]bool)
|
freed := make(map[pgid]bool)
|
||||||
for _, id := range tx.db.freelist.all() {
|
all := make([]pgid, tx.db.freelist.count())
|
||||||
|
tx.db.freelist.copyall(all)
|
||||||
|
for _, id := range all {
|
||||||
if freed[id] {
|
if freed[id] {
|
||||||
ch <- fmt.Errorf("page %d: already freed", id)
|
ch <- fmt.Errorf("page %d: already freed", id)
|
||||||
}
|
}
|
||||||
|
@ -392,8 +404,10 @@ func (tx *Tx) check(ch chan error) {
|
||||||
reachable := make(map[pgid]*page)
|
reachable := make(map[pgid]*page)
|
||||||
reachable[0] = tx.page(0) // meta0
|
reachable[0] = tx.page(0) // meta0
|
||||||
reachable[1] = tx.page(1) // meta1
|
reachable[1] = tx.page(1) // meta1
|
||||||
for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
|
if !tx.DB().NoFreelistSync {
|
||||||
reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
|
for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ {
|
||||||
|
reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recursively check buckets.
|
// Recursively check buckets.
|
||||||
|
@ -451,7 +465,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo
|
||||||
|
|
||||||
// allocate returns a contiguous block of memory starting at a given page.
|
// allocate returns a contiguous block of memory starting at a given page.
|
||||||
func (tx *Tx) allocate(count int) (*page, error) {
|
func (tx *Tx) allocate(count int) (*page, error) {
|
||||||
p, err := tx.db.allocate(count)
|
p, err := tx.db.allocate(tx.meta.txid, count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
|
@ -1,5 +1,5 @@
|
||||||
hash: 1c01233ff3c05df62210db3ec7bd51114f9b22401187e8e749a3ee5ea99f8594
|
hash: 4151c7de891aaf3c611392ecd302c88fd7b6ea01f494bdb0900eb2b2009c2072
|
||||||
updated: 2017-06-22T14:26:04.148865868-07:00
|
updated: 2017-07-05T14:33:35.042371004-07:00
|
||||||
imports:
|
imports:
|
||||||
- name: github.com/beorn7/perks
|
- name: github.com/beorn7/perks
|
||||||
version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
|
version: 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
|
||||||
|
@ -7,10 +7,10 @@ imports:
|
||||||
- quantile
|
- quantile
|
||||||
- name: github.com/bgentry/speakeasy
|
- name: github.com/bgentry/speakeasy
|
||||||
version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd
|
version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd
|
||||||
- name: github.com/boltdb/bolt
|
|
||||||
version: 583e8937c61f1af6513608ccc75c97b6abdf4ff9
|
|
||||||
- name: github.com/cockroachdb/cmux
|
- name: github.com/cockroachdb/cmux
|
||||||
version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92
|
version: 112f0506e7743d64a6eb8fedbcff13d9979bbf92
|
||||||
|
- name: github.com/coreos/bbolt
|
||||||
|
version: ad39960eb40bb33c9bda31bed2eaf4fdda15efe6
|
||||||
- name: github.com/coreos/go-semver
|
- name: github.com/coreos/go-semver
|
||||||
version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6
|
version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6
|
||||||
subpackages:
|
subpackages:
|
||||||
|
|
Loading…
Reference in New Issue