/** * Unit tests for ReedSolomon * * Copyright 2015, Klaus Post * Copyright 2015, Backblaze, Inc. All rights reserved. */ package reedsolomon import ( "fmt" "math/rand" "testing" ) func TestEncoding(t *testing.T) { perShard := 50000 r, err := New(10, 3) if err != nil { t.Fatal(err) } shards := make([][]byte, 13) for s := range shards { shards[s] = make([]byte, perShard) } rand.Seed(0) for s := 0; s < 13; s++ { fillRandom(shards[s]) } err = r.Encode(shards) if err != nil { t.Fatal(err) } ok, err := r.Verify(shards) if err != nil { t.Fatal(err) } if !ok { t.Fatal("Verification failed") } } func TestReconstruct(t *testing.T) { perShard := 50000 r, err := New(10, 3) if err != nil { t.Fatal(err) } shards := make([][]byte, 13) for s := range shards { shards[s] = make([]byte, perShard) } rand.Seed(0) for s := 0; s < 13; s++ { fillRandom(shards[s]) } err = r.Encode(shards) if err != nil { t.Fatal(err) } shards[0] = nil shards[7] = nil shards[11] = nil err = r.Reconstruct(shards) if err != nil { t.Fatal(err) } ok, err := r.Verify(shards) if err != nil { t.Fatal(err) } if !ok { t.Fatal("Verification failed") } } func TestVerify(t *testing.T) { perShard := 33333 r, err := New(10, 4) if err != nil { t.Fatal(err) } shards := make([][]byte, 14) for s := range shards { shards[s] = make([]byte, perShard) } rand.Seed(0) for s := 0; s < 10; s++ { fillRandom(shards[s]) } err = r.Encode(shards) if err != nil { t.Fatal(err) } ok, err := r.Verify(shards) if err != nil { t.Fatal(err) } if !ok { t.Fatal("Verification failed") } // Put in random data. Verification should fail fillRandom(shards[10]) ok, err = r.Verify(shards) if err != nil { t.Fatal(err) } if ok { t.Fatal("Verification did not fail") } // Re-encode err = r.Encode(shards) if err != nil { t.Fatal(err) } // Fill a data segment with random data fillRandom(shards[0]) ok, err = r.Verify(shards) if err != nil { t.Fatal(err) } if ok { t.Fatal("Verification did not fail") } } func TestOneEncode(t *testing.T) { codec, err := New(5, 5) if err != nil { t.Fatal(err) } shards := make([][]byte, 10) shards[0] = []byte{0, 1} shards[1] = []byte{4, 5} shards[2] = []byte{2, 3} shards[3] = []byte{6, 7} shards[4] = []byte{8, 9} shards[5] = []byte{0, 0} shards[6] = []byte{0, 0} shards[7] = []byte{0, 0} shards[8] = []byte{0, 0} shards[9] = []byte{0, 0} codec.Encode(shards) if shards[5][0] != 12 || shards[5][1] != 13 { t.Fatal("shard 5 mismatch") } if shards[6][0] != 10 || shards[6][1] != 11 { t.Fatal("shard 6 mismatch") } if shards[7][0] != 14 || shards[7][1] != 15 { t.Fatal("shard 7 mismatch") } if shards[8][0] != 90 || shards[8][1] != 91 { t.Fatal("shard 8 mismatch") } if shards[9][0] != 94 || shards[9][1] != 95 { t.Fatal("shard 9 mismatch") } ok, err := codec.Verify(shards) if err != nil { t.Fatal(err) } if !ok { t.Fatal("did not verify") } shards[8][0]++ ok, err = codec.Verify(shards) if err != nil { t.Fatal(err) } if ok { t.Fatal("verify did not fail as expected") } } func fillRandom(b []byte) { for i := range b { b[i] = byte(rand.Int() & 0xff) } } func benchmarkEncode(b *testing.B, dataShards, parityShards, shardSize int) { r, err := New(dataShards, parityShards) if err != nil { b.Fatal(err) } shards := make([][]byte, dataShards+parityShards) for s := range shards { shards[s] = make([]byte, shardSize) } rand.Seed(0) for s := 0; s < dataShards; s++ { fillRandom(shards[s]) } b.SetBytes(int64(shardSize * dataShards)) b.ResetTimer() for i := 0; i < b.N; i++ { err = r.Encode(shards) if err != nil { b.Fatal(err) } } } func BenchmarkEncode10x2x10000(b *testing.B) { benchmarkEncode(b, 10, 2, 10000) } func BenchmarkEncode100x20x10000(b *testing.B) { benchmarkEncode(b, 100, 20, 10000) } func BenchmarkEncode17x3x1M(b *testing.B) { benchmarkEncode(b, 17, 3, 1024*1024) } // Benchmark 10 data shards and 4 parity shards with 16MB each. func BenchmarkEncode10x4x16M(b *testing.B) { benchmarkEncode(b, 10, 4, 16*1024*1024) } // Benchmark 5 data shards and 2 parity shards with 1MB each. func BenchmarkEncode5x2x1M(b *testing.B) { benchmarkEncode(b, 5, 2, 1024*1024) } // Benchmark 1 data shards and 2 parity shards with 1MB each. func BenchmarkEncode10x2x1M(b *testing.B) { benchmarkEncode(b, 10, 2, 1024*1024) } // Benchmark 10 data shards and 4 parity shards with 1MB each. func BenchmarkEncode10x4x1M(b *testing.B) { benchmarkEncode(b, 10, 4, 1024*1024) } // Benchmark 50 data shards and 20 parity shards with 1MB each. func BenchmarkEncode50x20x1M(b *testing.B) { benchmarkEncode(b, 50, 20, 1024*1024) } // Benchmark 17 data shards and 3 parity shards with 16MB each. func BenchmarkEncode17x3x16M(b *testing.B) { benchmarkEncode(b, 17, 3, 16*1024*1024) } func benchmarkVerify(b *testing.B, dataShards, parityShards, shardSize int) { r, err := New(dataShards, parityShards) if err != nil { b.Fatal(err) } shards := make([][]byte, parityShards+dataShards) for s := range shards { shards[s] = make([]byte, shardSize) } rand.Seed(0) for s := 0; s < dataShards; s++ { fillRandom(shards[s]) } err = r.Encode(shards) if err != nil { b.Fatal(err) } b.SetBytes(int64(shardSize * dataShards)) b.ResetTimer() for i := 0; i < b.N; i++ { _, err = r.Verify(shards) if err != nil { b.Fatal(err) } } } // Benchmark 10 data slices with 2 parity slices holding 10000 bytes each func BenchmarkVerify10x2x10000(b *testing.B) { benchmarkVerify(b, 10, 2, 10000) } // Benchmark 50 data slices with 5 parity slices holding 100000 bytes each func BenchmarkVerify50x5x50000(b *testing.B) { benchmarkVerify(b, 50, 5, 100000) } // Benchmark 10 data slices with 2 parity slices holding 1MB bytes each func BenchmarkVerify10x2x1M(b *testing.B) { benchmarkVerify(b, 10, 2, 1024*1024) } // Benchmark 5 data slices with 2 parity slices holding 1MB bytes each func BenchmarkVerify5x2x1M(b *testing.B) { benchmarkVerify(b, 5, 2, 1024*1024) } // Benchmark 10 data slices with 4 parity slices holding 1MB bytes each func BenchmarkVerify10x4x1M(b *testing.B) { benchmarkVerify(b, 10, 4, 1024*1024) } // Benchmark 5 data slices with 2 parity slices holding 1MB bytes each func BenchmarkVerify50x20x1M(b *testing.B) { benchmarkVerify(b, 50, 20, 1024*1024) } // Benchmark 10 data slices with 4 parity slices holding 16MB bytes each func BenchmarkVerify10x4x16M(b *testing.B) { benchmarkVerify(b, 10, 4, 16*1024*1024) } // Simple example of how to use all functions of the Encoder. // Note that all error checks have been removed to keep it short. func ExampleEncoder() { // Create some sample data var data = make([]byte, 250000) fillRandom(data) // Create an encoder with 17 data and 3 parity slices. enc, _ := New(17, 3) // Split the data into shards shards, _ := enc.Split(data) // Encode the parity set _ = enc.Encode(shards) // Verify the parity set ok, _ := enc.Verify(shards) if ok { fmt.Println("ok") } // Delete two shards shards[10], shards[11] = nil, nil // Reconstruct the shards _ = enc.Reconstruct(shards) // Verify the data set ok, _ = enc.Verify(shards) if ok { fmt.Println("ok") } // Output: ok // ok }