The number of data shards must be below 257. Check that and update documentation.
parent
5c2ef3ae72
commit
8ebf356efb
|
@ -28,13 +28,13 @@ This section assumes you know the basics of Reed-Solomon encoding. A good start
|
|||
|
||||
This package performs the calculation of the parity sets. The usage is therefore relatively simple.
|
||||
|
||||
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 100 data shards, and the number of parity shards to be 2 or above, and below the number of data shards.
|
||||
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 257 data shards (the maximum supported number), and the number of parity shards to be 2 or above, and below the number of data shards.
|
||||
|
||||
To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated):
|
||||
```Go
|
||||
enc, err := reedsolomon.New(10, 3)
|
||||
```
|
||||
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters.
|
||||
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters, or if you specify more than 256 data shards.
|
||||
|
||||
The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13.
|
||||
```Go
|
||||
|
|
|
@ -45,7 +45,7 @@ import (
|
|||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
var dataShards = flag.Int("data", 4, "Number of shards to split the data into")
|
||||
var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.")
|
||||
var parShards = flag.Int("par", 2, "Number of parity shards")
|
||||
var outDir = flag.String("out", "", "Alternative output directory")
|
||||
|
||||
|
@ -67,6 +67,10 @@ func main() {
|
|||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
if *data > 257 {
|
||||
fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
fname := args[0]
|
||||
|
||||
// Create encoding matrix.
|
||||
|
|
|
@ -85,12 +85,14 @@ type reedSolomon struct {
|
|||
}
|
||||
|
||||
// ErrInvShardNum will be returned by New, if you attempt to create
|
||||
// an Encoder where either data or parity shards is zero or less.
|
||||
// an Encoder where either data or parity shards is zero or less,
|
||||
// or the number of data shards is higher than 256.
|
||||
var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards")
|
||||
|
||||
// New creates a new encoder and initializes it to
|
||||
// the number of data shards and parity shards that
|
||||
// you want to use. You can reuse this encoder.
|
||||
// Note that the maximum number of data shards is 256.
|
||||
func New(dataShards, parityShards int) (Encoder, error) {
|
||||
r := reedSolomon{
|
||||
DataShards: dataShards,
|
||||
|
@ -102,6 +104,10 @@ func New(dataShards, parityShards int) (Encoder, error) {
|
|||
return nil, ErrInvShardNum
|
||||
}
|
||||
|
||||
if dataShards > 256 {
|
||||
return nil, ErrInvShardNum
|
||||
}
|
||||
|
||||
// Start with a Vandermonde matrix. This matrix would work,
|
||||
// in theory, but doesn't have the property that the data
|
||||
// shards are unchanged after encoding.
|
||||
|
|
|
@ -416,3 +416,80 @@ func ExampleEncoder_slicing() {
|
|||
// splitB ok
|
||||
// merge ok
|
||||
}
|
||||
|
||||
func TestEncoderReconstruct(t *testing.T) {
|
||||
// Create some sample data
|
||||
var data = make([]byte, 250000)
|
||||
fillRandom(data)
|
||||
|
||||
// Create 5 data slices of 50000 elements each
|
||||
enc, _ := New(5, 3)
|
||||
shards, _ := enc.Split(data)
|
||||
err := enc.Encode(shards)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check that it verifies
|
||||
ok, err := enc.Verify(shards)
|
||||
if !ok || err != nil {
|
||||
t.Fatal("not ok:", ok, "err:", err)
|
||||
}
|
||||
|
||||
// Delete a shard
|
||||
shards[0] = nil
|
||||
|
||||
// Should reconstruct
|
||||
err = enc.Reconstruct(shards)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check that it verifies
|
||||
ok, err = enc.Verify(shards)
|
||||
if !ok || err != nil {
|
||||
t.Fatal("not ok:", ok, "err:", err)
|
||||
}
|
||||
|
||||
// Delete a shard
|
||||
shards[0] = nil
|
||||
shards[1][0], shards[1][500] = 75, 75
|
||||
|
||||
// Should reconstruct
|
||||
err = enc.Reconstruct(shards)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check that it verifies
|
||||
ok, err = enc.Verify(shards)
|
||||
if ok || err != nil {
|
||||
t.Fatal("error or ok:", ok, "err:", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatrices(t *testing.T) {
|
||||
_, err := New(10, 500)
|
||||
if err != nil {
|
||||
t.Fatal("creating matrix size", 10, 500, ":", err)
|
||||
}
|
||||
_, err = New(256, 256)
|
||||
if err != nil {
|
||||
t.Fatal("creating matrix size", 256, 256, ":", err)
|
||||
}
|
||||
_, err = New(257, 10)
|
||||
if err != ErrInvShardNum {
|
||||
t.Fatal("Expected ErrInvShardNum, but got", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestAllMatrices(t *testing.T) {
|
||||
t.Skip("Skipping slow matrix check")
|
||||
for i := 1; i < 257; i++ {
|
||||
_, err := New(i, i)
|
||||
if err != nil {
|
||||
t.Fatal("creating matrix size", i, i, ":", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue