The number of data shards must be below 257. Check that and update documentation.
parent
5c2ef3ae72
commit
8ebf356efb
|
@ -28,13 +28,13 @@ This section assumes you know the basics of Reed-Solomon encoding. A good start
|
||||||
|
|
||||||
This package performs the calculation of the parity sets. The usage is therefore relatively simple.
|
This package performs the calculation of the parity sets. The usage is therefore relatively simple.
|
||||||
|
|
||||||
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 100 data shards, and the number of parity shards to be 2 or above, and below the number of data shards.
|
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 257 data shards (the maximum supported number), and the number of parity shards to be 2 or above, and below the number of data shards.
|
||||||
|
|
||||||
To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated):
|
To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated):
|
||||||
```Go
|
```Go
|
||||||
enc, err := reedsolomon.New(10, 3)
|
enc, err := reedsolomon.New(10, 3)
|
||||||
```
|
```
|
||||||
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters.
|
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters, or if you specify more than 256 data shards.
|
||||||
|
|
||||||
The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13.
|
The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13.
|
||||||
```Go
|
```Go
|
||||||
|
|
|
@ -45,7 +45,7 @@ import (
|
||||||
"github.com/klauspost/reedsolomon"
|
"github.com/klauspost/reedsolomon"
|
||||||
)
|
)
|
||||||
|
|
||||||
var dataShards = flag.Int("data", 4, "Number of shards to split the data into")
|
var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.")
|
||||||
var parShards = flag.Int("par", 2, "Number of parity shards")
|
var parShards = flag.Int("par", 2, "Number of parity shards")
|
||||||
var outDir = flag.String("out", "", "Alternative output directory")
|
var outDir = flag.String("out", "", "Alternative output directory")
|
||||||
|
|
||||||
|
@ -67,6 +67,10 @@ func main() {
|
||||||
flag.Usage()
|
flag.Usage()
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
if *data > 257 {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
fname := args[0]
|
fname := args[0]
|
||||||
|
|
||||||
// Create encoding matrix.
|
// Create encoding matrix.
|
||||||
|
|
|
@ -85,12 +85,14 @@ type reedSolomon struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrInvShardNum will be returned by New, if you attempt to create
|
// ErrInvShardNum will be returned by New, if you attempt to create
|
||||||
// an Encoder where either data or parity shards is zero or less.
|
// an Encoder where either data or parity shards is zero or less,
|
||||||
|
// or the number of data shards is higher than 256.
|
||||||
var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards")
|
var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards")
|
||||||
|
|
||||||
// New creates a new encoder and initializes it to
|
// New creates a new encoder and initializes it to
|
||||||
// the number of data shards and parity shards that
|
// the number of data shards and parity shards that
|
||||||
// you want to use. You can reuse this encoder.
|
// you want to use. You can reuse this encoder.
|
||||||
|
// Note that the maximum number of data shards is 256.
|
||||||
func New(dataShards, parityShards int) (Encoder, error) {
|
func New(dataShards, parityShards int) (Encoder, error) {
|
||||||
r := reedSolomon{
|
r := reedSolomon{
|
||||||
DataShards: dataShards,
|
DataShards: dataShards,
|
||||||
|
@ -102,6 +104,10 @@ func New(dataShards, parityShards int) (Encoder, error) {
|
||||||
return nil, ErrInvShardNum
|
return nil, ErrInvShardNum
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if dataShards > 256 {
|
||||||
|
return nil, ErrInvShardNum
|
||||||
|
}
|
||||||
|
|
||||||
// Start with a Vandermonde matrix. This matrix would work,
|
// Start with a Vandermonde matrix. This matrix would work,
|
||||||
// in theory, but doesn't have the property that the data
|
// in theory, but doesn't have the property that the data
|
||||||
// shards are unchanged after encoding.
|
// shards are unchanged after encoding.
|
||||||
|
|
|
@ -416,3 +416,80 @@ func ExampleEncoder_slicing() {
|
||||||
// splitB ok
|
// splitB ok
|
||||||
// merge ok
|
// merge ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEncoderReconstruct(t *testing.T) {
|
||||||
|
// Create some sample data
|
||||||
|
var data = make([]byte, 250000)
|
||||||
|
fillRandom(data)
|
||||||
|
|
||||||
|
// Create 5 data slices of 50000 elements each
|
||||||
|
enc, _ := New(5, 3)
|
||||||
|
shards, _ := enc.Split(data)
|
||||||
|
err := enc.Encode(shards)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that it verifies
|
||||||
|
ok, err := enc.Verify(shards)
|
||||||
|
if !ok || err != nil {
|
||||||
|
t.Fatal("not ok:", ok, "err:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete a shard
|
||||||
|
shards[0] = nil
|
||||||
|
|
||||||
|
// Should reconstruct
|
||||||
|
err = enc.Reconstruct(shards)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that it verifies
|
||||||
|
ok, err = enc.Verify(shards)
|
||||||
|
if !ok || err != nil {
|
||||||
|
t.Fatal("not ok:", ok, "err:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete a shard
|
||||||
|
shards[0] = nil
|
||||||
|
shards[1][0], shards[1][500] = 75, 75
|
||||||
|
|
||||||
|
// Should reconstruct
|
||||||
|
err = enc.Reconstruct(shards)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that it verifies
|
||||||
|
ok, err = enc.Verify(shards)
|
||||||
|
if ok || err != nil {
|
||||||
|
t.Fatal("error or ok:", ok, "err:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMatrices(t *testing.T) {
|
||||||
|
_, err := New(10, 500)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("creating matrix size", 10, 500, ":", err)
|
||||||
|
}
|
||||||
|
_, err = New(256, 256)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("creating matrix size", 256, 256, ":", err)
|
||||||
|
}
|
||||||
|
_, err = New(257, 10)
|
||||||
|
if err != ErrInvShardNum {
|
||||||
|
t.Fatal("Expected ErrInvShardNum, but got", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAllMatrices(t *testing.T) {
|
||||||
|
t.Skip("Skipping slow matrix check")
|
||||||
|
for i := 1; i < 257; i++ {
|
||||||
|
_, err := New(i, i)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("creating matrix size", i, i, ":", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue