From a2a077790bfbf2e0f38fec6a6ffcc4bf3129a5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=87=E6=85=95?= Date: Mon, 10 Jun 2019 10:25:59 +0800 Subject: [PATCH] vendor: update boltdb --- go.mod | 2 +- go.sum | 4 ++-- vendor/go.etcd.io/bbolt/bolt_riscv64.go | 12 +++++++++++ vendor/go.etcd.io/bbolt/db.go | 12 ++++++++++- vendor/go.etcd.io/bbolt/freelist.go | 22 ++++++++++++++++++++ vendor/go.etcd.io/bbolt/tx.go | 27 +++++++++++++++++++++---- 6 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 vendor/go.etcd.io/bbolt/bolt_riscv64.go diff --git a/go.mod b/go.mod index a84c2c121..cb28a0d8b 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 github.com/urfave/cli v1.20.0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 - go.etcd.io/bbolt v1.3.2 + go.etcd.io/bbolt v1.3.3 go.uber.org/atomic v1.3.2 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 diff --git a/go.sum b/go.sum index 312d05041..660f98f0d 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= diff --git a/vendor/go.etcd.io/bbolt/bolt_riscv64.go b/vendor/go.etcd.io/bbolt/bolt_riscv64.go new file mode 100644 index 000000000..07b4b47cd --- /dev/null +++ b/vendor/go.etcd.io/bbolt/bolt_riscv64.go @@ -0,0 +1,12 @@ +// +build riscv64 + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = true diff --git a/vendor/go.etcd.io/bbolt/db.go b/vendor/go.etcd.io/bbolt/db.go index 962248c99..870c8b1cc 100644 --- a/vendor/go.etcd.io/bbolt/db.go +++ b/vendor/go.etcd.io/bbolt/db.go @@ -121,6 +121,7 @@ type DB struct { AllocSize int path string + openFile func(string, int, os.FileMode) (*os.File, error) file *os.File dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte @@ -199,10 +200,15 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { db.readOnly = true } + db.openFile = options.OpenFile + if db.openFile == nil { + db.openFile = os.OpenFile + } + // Open data file and separate sync handler for metadata writes. db.path = path var err error - if db.file, err = os.OpenFile(db.path, flag|os.O_CREATE, mode); err != nil { + if db.file, err = db.openFile(db.path, flag|os.O_CREATE, mode); err != nil { _ = db.close() return nil, err } @@ -1054,6 +1060,10 @@ type Options struct { // set directly on the DB itself when returned from Open(), but this option // is useful in APIs which expose Options but not the underlying DB. NoSync bool + + // OpenFile is used to open files. It defaults to os.OpenFile. This option + // is useful for writing hermetic tests. + OpenFile func(string, int, os.FileMode) (*os.File, error) } // DefaultOptions represent the options used if nil options are passed into Open(). diff --git a/vendor/go.etcd.io/bbolt/freelist.go b/vendor/go.etcd.io/bbolt/freelist.go index 93fd85d50..587b8cc02 100644 --- a/vendor/go.etcd.io/bbolt/freelist.go +++ b/vendor/go.etcd.io/bbolt/freelist.go @@ -349,6 +349,28 @@ func (f *freelist) reload(p *page) { f.readIDs(a) } +// noSyncReload reads the freelist from pgids and filters out pending items. +func (f *freelist) noSyncReload(pgids []pgid) { + // Build a cache of only pending pages. + pcache := make(map[pgid]bool) + for _, txp := range f.pending { + for _, pendingID := range txp.ids { + pcache[pendingID] = true + } + } + + // Check each page in the freelist and build a new available freelist + // with any pages not in the pending lists. + var a []pgid + for _, id := range pgids { + if !pcache[id] { + a = append(a, id) + } + } + + f.readIDs(a) +} + // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { ids := f.getFreePageIDs() diff --git a/vendor/go.etcd.io/bbolt/tx.go b/vendor/go.etcd.io/bbolt/tx.go index f50864142..2df7688c2 100644 --- a/vendor/go.etcd.io/bbolt/tx.go +++ b/vendor/go.etcd.io/bbolt/tx.go @@ -254,17 +254,36 @@ func (tx *Tx) Rollback() error { if tx.db == nil { return ErrTxClosed } - tx.rollback() + tx.nonPhysicalRollback() return nil } +// nonPhysicalRollback is called when user calls Rollback directly, in this case we do not need to reload the free pages from disk. +func (tx *Tx) nonPhysicalRollback() { + if tx.db == nil { + return + } + if tx.writable { + tx.db.freelist.rollback(tx.meta.txid) + } + tx.close() +} + +// rollback needs to reload the free pages from disk in case some system error happens like fsync error. func (tx *Tx) rollback() { if tx.db == nil { return } if tx.writable { tx.db.freelist.rollback(tx.meta.txid) - tx.db.freelist.reload(tx.db.page(tx.db.meta().freelist)) + if !tx.db.hasSyncedFreelist() { + // Reconstruct free page list by scanning the DB to get the whole free page list. + // Note: scaning the whole db is heavy if your db size is large in NoSyncFreeList mode. + tx.db.freelist.noSyncReload(tx.db.freepages()) + } else { + // Read free page list from freelist page. + tx.db.freelist.reload(tx.db.page(tx.db.meta().freelist)) + } } tx.close() } @@ -315,7 +334,7 @@ func (tx *Tx) Copy(w io.Writer) error { // If err == nil then exactly tx.Size() bytes will be written into the writer. func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // Attempt to open reader with WriteFlag - f, err := os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) + f, err := tx.db.openFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) if err != nil { return 0, err } @@ -369,7 +388,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // A reader transaction is maintained during the copy so it is safe to continue // using the database while a copy is in progress. func (tx *Tx) CopyFile(path string, mode os.FileMode) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + f, err := tx.db.openFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err }