From 8ebf356efb7dc73edf6a593b42a6842fd5e36179 Mon Sep 17 00:00:00 2001 From: klauspost Date: Tue, 23 Jun 2015 13:39:57 +0200 Subject: [PATCH] The number of data shards must be below 257. Check that and update documentation. --- README.md | 4 +- examples/simple-encoder.go | 6 ++- reedsolomon.go | 8 +++- reedsolomon_test.go | 77 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3cc2d4c..d12f66e 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,13 @@ This section assumes you know the basics of Reed-Solomon encoding. A good start This package performs the calculation of the parity sets. The usage is therefore relatively simple. -First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 100 data shards, and the number of parity shards to be 2 or above, and below the number of data shards. +First of all, you need to choose your distribution of data and parity shards. A 'good' distribution is very subjective, and will depend a lot on your usage scenario. A good starting point is above 5 and below 257 data shards (the maximum supported number), and the number of parity shards to be 2 or above, and below the number of data shards. To create an encoder with 10 data shards (where your data goes) and 3 parity shards (calculated): ```Go enc, err := reedsolomon.New(10, 3) ``` -This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters. +This encoder will work for all parity sets with this distribution of data and parity shards. The error will only be set if you specify 0 or negative values in any of the parameters, or if you specify more than 256 data shards. The you send and receive data is a simple slice of byte slices; `[][]byte`. In the example above, the top slice must have a length of 13. ```Go diff --git a/examples/simple-encoder.go b/examples/simple-encoder.go index 424ef13..abe94da 100644 --- a/examples/simple-encoder.go +++ b/examples/simple-encoder.go @@ -45,7 +45,7 @@ import ( "github.com/klauspost/reedsolomon" ) -var dataShards = flag.Int("data", 4, "Number of shards to split the data into") +var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.") var parShards = flag.Int("par", 2, "Number of parity shards") var outDir = flag.String("out", "", "Alternative output directory") @@ -67,6 +67,10 @@ func main() { flag.Usage() os.Exit(1) } + if *data > 257 { + fmt.Fprintf(os.Stderr, "Error: Too many data shards\n") + os.Exit(1) + } fname := args[0] // Create encoding matrix. diff --git a/reedsolomon.go b/reedsolomon.go index 76f5281..16dff58 100644 --- a/reedsolomon.go +++ b/reedsolomon.go @@ -85,12 +85,14 @@ type reedSolomon struct { } // ErrInvShardNum will be returned by New, if you attempt to create -// an Encoder where either data or parity shards is zero or less. +// an Encoder where either data or parity shards is zero or less, +// or the number of data shards is higher than 256. var ErrInvShardNum = errors.New("cannot create Encoder with zero or less data/parity shards") // New creates a new encoder and initializes it to // the number of data shards and parity shards that // you want to use. You can reuse this encoder. +// Note that the maximum number of data shards is 256. func New(dataShards, parityShards int) (Encoder, error) { r := reedSolomon{ DataShards: dataShards, @@ -102,6 +104,10 @@ func New(dataShards, parityShards int) (Encoder, error) { return nil, ErrInvShardNum } + if dataShards > 256 { + return nil, ErrInvShardNum + } + // Start with a Vandermonde matrix. This matrix would work, // in theory, but doesn't have the property that the data // shards are unchanged after encoding. diff --git a/reedsolomon_test.go b/reedsolomon_test.go index d236fd1..7e4cece 100644 --- a/reedsolomon_test.go +++ b/reedsolomon_test.go @@ -416,3 +416,80 @@ func ExampleEncoder_slicing() { // splitB ok // merge ok } + +func TestEncoderReconstruct(t *testing.T) { + // Create some sample data + var data = make([]byte, 250000) + fillRandom(data) + + // Create 5 data slices of 50000 elements each + enc, _ := New(5, 3) + shards, _ := enc.Split(data) + err := enc.Encode(shards) + if err != nil { + t.Fatal(err) + } + + // Check that it verifies + ok, err := enc.Verify(shards) + if !ok || err != nil { + t.Fatal("not ok:", ok, "err:", err) + } + + // Delete a shard + shards[0] = nil + + // Should reconstruct + err = enc.Reconstruct(shards) + if err != nil { + t.Fatal(err) + } + + // Check that it verifies + ok, err = enc.Verify(shards) + if !ok || err != nil { + t.Fatal("not ok:", ok, "err:", err) + } + + // Delete a shard + shards[0] = nil + shards[1][0], shards[1][500] = 75, 75 + + // Should reconstruct + err = enc.Reconstruct(shards) + if err != nil { + t.Fatal(err) + } + + // Check that it verifies + ok, err = enc.Verify(shards) + if ok || err != nil { + t.Fatal("error or ok:", ok, "err:", err) + } +} + +func TestMatrices(t *testing.T) { + _, err := New(10, 500) + if err != nil { + t.Fatal("creating matrix size", 10, 500, ":", err) + } + _, err = New(256, 256) + if err != nil { + t.Fatal("creating matrix size", 256, 256, ":", err) + } + _, err = New(257, 10) + if err != ErrInvShardNum { + t.Fatal("Expected ErrInvShardNum, but got", err) + } + +} + +func TestAllMatrices(t *testing.T) { + t.Skip("Skipping slow matrix check") + for i := 1; i < 257; i++ { + _, err := New(i, i) + if err != nil { + t.Fatal("creating matrix size", i, i, ":", err) + } + } +}