Add pure XOR for 1 parity (#138)
WithFastOneParityMatrix will switch the matrix to a simple xor if there is only one parity shard. The PAR1 matrix already has this property so it has little effect there.master
parent
d6d9fba4f9
commit
cf8495259a
|
@ -29,6 +29,7 @@ go get -u github.com/klauspost/reedsolomon
|
|||
|
||||
Numerous updates:
|
||||
|
||||
* Added WithFastOneParityMatrix for faster operation with 1 parity shard.
|
||||
* Much better performance when using a limited number of goroutines.
|
||||
* AVX512 is now using multiple cores.
|
||||
* Stream processing overhaul, big speedups in most cases.
|
||||
|
|
20
options.go
20
options.go
|
@ -10,13 +10,15 @@ import (
|
|||
type Option func(*options)
|
||||
|
||||
type options struct {
|
||||
maxGoroutines int
|
||||
minSplitSize int
|
||||
maxGoroutines int
|
||||
minSplitSize int
|
||||
shardSize int
|
||||
perRound int
|
||||
|
||||
useAVX512, useAVX2, useSSSE3, useSSE2 bool
|
||||
usePAR1Matrix bool
|
||||
useCauchy bool
|
||||
shardSize int
|
||||
perRound int
|
||||
fastOneParity bool
|
||||
|
||||
// stream options
|
||||
concReads bool
|
||||
|
@ -27,6 +29,7 @@ type options struct {
|
|||
var defaultOptions = options{
|
||||
maxGoroutines: 384,
|
||||
minSplitSize: -1,
|
||||
fastOneParity: false,
|
||||
|
||||
// Detect CPU capabilities.
|
||||
useSSSE3: cpuid.CPU.SSSE3(),
|
||||
|
@ -161,3 +164,12 @@ func WithCauchyMatrix() Option {
|
|||
o.usePAR1Matrix = false
|
||||
}
|
||||
}
|
||||
|
||||
// WithFastOneParityMatrix will switch the matrix to a simple xor
|
||||
// if there is only one parity shard.
|
||||
// The PAR1 matrix already has this property so it has little effect there.
|
||||
func WithFastOneParityMatrix() Option {
|
||||
return func(o *options) {
|
||||
o.fastOneParity = true
|
||||
}
|
||||
}
|
||||
|
|
|
@ -206,6 +206,32 @@ func buildMatrixCauchy(dataShards, totalShards int) (matrix, error) {
|
|||
return result, nil
|
||||
}
|
||||
|
||||
// buildXorMatrix can be used to build a matrix with pure XOR
|
||||
// operations if there is only one parity shard.
|
||||
func buildXorMatrix(dataShards, totalShards int) (matrix, error) {
|
||||
if dataShards+1 != totalShards {
|
||||
return nil, errors.New("internal error")
|
||||
}
|
||||
result, err := newMatrix(totalShards, dataShards)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for r, row := range result {
|
||||
// The top portion of the matrix is the identity
|
||||
// matrix.
|
||||
if r < dataShards {
|
||||
result[r][r] = 1
|
||||
} else {
|
||||
// Set all values to 1 (XOR)
|
||||
for c := range row {
|
||||
result[r][c] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// New creates a new encoder and initializes it to
|
||||
// the number of data shards and parity shards that
|
||||
// you want to use. You can reuse this encoder.
|
||||
|
@ -232,6 +258,8 @@ func New(dataShards, parityShards int, opts ...Option) (Encoder, error) {
|
|||
|
||||
var err error
|
||||
switch {
|
||||
case r.o.fastOneParity && parityShards == 1:
|
||||
r.m, err = buildXorMatrix(dataShards, r.Shards)
|
||||
case r.o.useCauchy:
|
||||
r.m, err = buildMatrixCauchy(dataShards, r.Shards)
|
||||
case r.o.usePAR1Matrix:
|
||||
|
|
|
@ -137,6 +137,7 @@ func testOpts() [][]Option {
|
|||
}
|
||||
opts := [][]Option{
|
||||
{WithPAR1Matrix()}, {WithCauchyMatrix()},
|
||||
{WithFastOneParityMatrix()}, {WithPAR1Matrix(), WithFastOneParityMatrix()}, {WithCauchyMatrix(), WithFastOneParityMatrix()},
|
||||
{WithMaxGoroutines(1), WithMinSplitSize(500), withSSSE3(false), withAVX2(false), withAVX512(false)},
|
||||
{WithMaxGoroutines(5000), WithMinSplitSize(50), withSSSE3(false), withAVX2(false), withAVX512(false)},
|
||||
{WithMaxGoroutines(5000), WithMinSplitSize(500000), withSSSE3(false), withAVX2(false), withAVX512(false)},
|
||||
|
@ -179,7 +180,7 @@ func TestEncoding(t *testing.T) {
|
|||
|
||||
// matrix sizes to test.
|
||||
// note that par1 matric will fail on some combinations.
|
||||
var testSizes = [][2]int{{1, 1}, {1, 2}, {3, 3}, {3, 1}, {5, 3}, {8, 4}, {10, 30}, {14, 7}, {41, 17}}
|
||||
var testSizes = [][2]int{{1, 1}, {1, 2}, {3, 3}, {3, 1}, {5, 3}, {8, 4}, {10, 30}, {14, 7}, {41, 17}, {49, 1}}
|
||||
var testDataSizes = []int{10, 100, 1000, 10001, 100003, 1000055}
|
||||
var testDataSizesShort = []int{10, 10001, 100003}
|
||||
|
||||
|
@ -1257,7 +1258,7 @@ func TestStandardMatrices(t *testing.T) {
|
|||
continue
|
||||
}
|
||||
sh := shards[:i+j]
|
||||
r, err := New(i, j, testOptions()...)
|
||||
r, err := New(i, j, testOptions(WithFastOneParityMatrix())...)
|
||||
if err != nil {
|
||||
// We are not supposed to write to t from goroutines.
|
||||
t.Fatal("creating matrix size", i, j, ":", err)
|
||||
|
@ -1320,7 +1321,7 @@ func TestCauchyMatrices(t *testing.T) {
|
|||
continue
|
||||
}
|
||||
sh := shards[:i+j]
|
||||
r, err := New(i, j, WithCauchyMatrix())
|
||||
r, err := New(i, j, testOptions(WithCauchyMatrix(), WithFastOneParityMatrix())...)
|
||||
if err != nil {
|
||||
// We are not supposed to write to t from goroutines.
|
||||
t.Fatal("creating matrix size", i, j, ":", err)
|
||||
|
|
Loading…
Reference in New Issue