The number of data shards must be below 257. Check that and update documentation.

master
klauspost 2015-06-23 13:39:57 +02:00
parent 5c2ef3ae72
commit 8ebf356efb
4 changed files with 91 additions and 4 deletions

View File

@ -28,13 +28,13 @@ This section assumes you know the basics of Reed-Solomon encoding. A good start
This package performs the calculation of the parity sets. The usage is therefore relatively simple.
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 100 data shards, and the number of parity shards to be 2 or above, and below the number of data shards.
First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 257 data shards (the maximum supported number), and the number of parity shards to be 2 or above, and below the number of data shards.
To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated):
```Go
enc, err := reedsolomon.New(10, 3)
```
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters.
This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters, or if you specify more than 256 data shards.
The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13.
```Go

View File

@ -45,7 +45,7 @@ import (
"github.com/klauspost/reedsolomon"
)
var dataShards = flag.Int("data", 4, "Number of shards to split the data into")
var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.")
var parShards = flag.Int("par", 2, "Number of parity shards")
var outDir = flag.String("out", "", "Alternative output directory")
@ -67,6 +67,10 @@ func main() {
flag.Usage()
os.Exit(1)
}
if *data > 257 {
fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
os.Exit(1)
}
fname := args[0]
// Create encoding matrix.

View File

@ -85,12 +85,14 @@ type reedSolomon struct {
}
// ErrInvShardNum will be returned by New, if you attempt to create
// an Encoder where either data or parity shards is zero or less.
// an Encoder where either data or parity shards is zero or less,
// or the number of data shards is higher than 256.
var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards")
// New creates a new encoder and initializes it to
// the number of data shards and parity shards that
// you want to use. You can reuse this encoder.
// Note that the maximum number of data shards is 256.
func New(dataShards, parityShards int) (Encoder, error) {
r := reedSolomon{
DataShards: dataShards,
@ -102,6 +104,10 @@ func New(dataShards, parityShards int) (Encoder, error) {
return nil, ErrInvShardNum
}
if dataShards > 256 {
return nil, ErrInvShardNum
}
// Start with a Vandermonde matrix. This matrix would work,
// in theory, but doesn't have the property that the data
// shards are unchanged after encoding.

View File

@ -416,3 +416,80 @@ func ExampleEncoder_slicing() {
// splitB ok
// merge ok
}
func TestEncoderReconstruct(t *testing.T) {
// Create some sample data
var data = make([]byte, 250000)
fillRandom(data)
// Create 5 data slices of 50000 elements each
enc, _ := New(5, 3)
shards, _ := enc.Split(data)
err := enc.Encode(shards)
if err != nil {
t.Fatal(err)
}
// Check that it verifies
ok, err := enc.Verify(shards)
if !ok || err != nil {
t.Fatal("not ok:", ok, "err:", err)
}
// Delete a shard
shards[0] = nil
// Should reconstruct
err = enc.Reconstruct(shards)
if err != nil {
t.Fatal(err)
}
// Check that it verifies
ok, err = enc.Verify(shards)
if !ok || err != nil {
t.Fatal("not ok:", ok, "err:", err)
}
// Delete a shard
shards[0] = nil
shards[1][0], shards[1][500] = 75, 75
// Should reconstruct
err = enc.Reconstruct(shards)
if err != nil {
t.Fatal(err)
}
// Check that it verifies
ok, err = enc.Verify(shards)
if ok || err != nil {
t.Fatal("error or ok:", ok, "err:", err)
}
}
func TestMatrices(t *testing.T) {
_, err := New(10, 500)
if err != nil {
t.Fatal("creating matrix size", 10, 500, ":", err)
}
_, err = New(256, 256)
if err != nil {
t.Fatal("creating matrix size", 256, 256, ":", err)
}
_, err = New(257, 10)
if err != ErrInvShardNum {
t.Fatal("Expected ErrInvShardNum, but got", err)
}
}
func TestAllMatrices(t *testing.T) {
t.Skip("Skipping slow matrix check")
for i := 1; i < 257; i++ {
_, err := New(i, i)
if err != nil {
t.Fatal("creating matrix size", i, i, ":", err)
}
}
}