360 lines
7.1 KiB
Go
360 lines
7.1 KiB
Go
/**
|
|
* Unit tests for ReedSolomon
|
|
*
|
|
* Copyright 2015, Klaus Post
|
|
* Copyright 2015, Backblaze, Inc. All rights reserved.
|
|
*/
|
|
|
|
package reedsolomon
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"testing"
|
|
)
|
|
|
|
func TestEncoding(t *testing.T) {
|
|
perShard := 50000
|
|
r, err := New(10, 3)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shards := make([][]byte, 13)
|
|
for s := range shards {
|
|
shards[s] = make([]byte, perShard)
|
|
}
|
|
|
|
rand.Seed(0)
|
|
for s := 0; s < 13; s++ {
|
|
fillRandom(shards[s])
|
|
}
|
|
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ok, err := r.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("Verification failed")
|
|
}
|
|
}
|
|
|
|
func TestReconstruct(t *testing.T) {
|
|
perShard := 50000
|
|
r, err := New(10, 3)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shards := make([][]byte, 13)
|
|
for s := range shards {
|
|
shards[s] = make([]byte, perShard)
|
|
}
|
|
|
|
rand.Seed(0)
|
|
for s := 0; s < 13; s++ {
|
|
fillRandom(shards[s])
|
|
}
|
|
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
shards[0] = nil
|
|
shards[7] = nil
|
|
shards[11] = nil
|
|
|
|
err = r.Reconstruct(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ok, err := r.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("Verification failed")
|
|
}
|
|
}
|
|
|
|
func TestVerify(t *testing.T) {
|
|
perShard := 33333
|
|
r, err := New(10, 4)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shards := make([][]byte, 14)
|
|
for s := range shards {
|
|
shards[s] = make([]byte, perShard)
|
|
}
|
|
|
|
rand.Seed(0)
|
|
for s := 0; s < 10; s++ {
|
|
fillRandom(shards[s])
|
|
}
|
|
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ok, err := r.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("Verification failed")
|
|
}
|
|
// Put in random data. Verification should fail
|
|
fillRandom(shards[10])
|
|
ok, err = r.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ok {
|
|
t.Fatal("Verification did not fail")
|
|
}
|
|
// Re-encode
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Fill a data segment with random data
|
|
fillRandom(shards[0])
|
|
ok, err = r.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ok {
|
|
t.Fatal("Verification did not fail")
|
|
}
|
|
}
|
|
|
|
func TestOneEncode(t *testing.T) {
|
|
codec, err := New(5, 5)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
shards := make([][]byte, 10)
|
|
shards[0] = []byte{0, 1}
|
|
shards[1] = []byte{4, 5}
|
|
shards[2] = []byte{2, 3}
|
|
shards[3] = []byte{6, 7}
|
|
shards[4] = []byte{8, 9}
|
|
shards[5] = []byte{0, 0}
|
|
shards[6] = []byte{0, 0}
|
|
shards[7] = []byte{0, 0}
|
|
shards[8] = []byte{0, 0}
|
|
shards[9] = []byte{0, 0}
|
|
codec.Encode(shards)
|
|
if shards[5][0] != 12 || shards[5][1] != 13 {
|
|
t.Fatal("shard 5 mismatch")
|
|
}
|
|
if shards[6][0] != 10 || shards[6][1] != 11 {
|
|
t.Fatal("shard 6 mismatch")
|
|
}
|
|
if shards[7][0] != 14 || shards[7][1] != 15 {
|
|
t.Fatal("shard 7 mismatch")
|
|
}
|
|
if shards[8][0] != 90 || shards[8][1] != 91 {
|
|
t.Fatal("shard 8 mismatch")
|
|
}
|
|
if shards[9][0] != 94 || shards[9][1] != 95 {
|
|
t.Fatal("shard 9 mismatch")
|
|
}
|
|
|
|
ok, err := codec.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("did not verify")
|
|
}
|
|
shards[8][0]++
|
|
ok, err = codec.Verify(shards)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ok {
|
|
t.Fatal("verify did not fail as expected")
|
|
}
|
|
|
|
}
|
|
|
|
func fillRandom(b []byte) {
|
|
for i := range b {
|
|
b[i] = byte(rand.Int() & 0xff)
|
|
}
|
|
}
|
|
|
|
func benchmarkEncode(b *testing.B, dataShards, parityShards, shardSize int) {
|
|
r, err := New(dataShards, parityShards)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
shards := make([][]byte, dataShards+parityShards)
|
|
for s := range shards {
|
|
shards[s] = make([]byte, shardSize)
|
|
}
|
|
|
|
rand.Seed(0)
|
|
for s := 0; s < dataShards; s++ {
|
|
fillRandom(shards[s])
|
|
}
|
|
|
|
b.SetBytes(int64(shardSize * dataShards))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkEncode10x2x10000(b *testing.B) {
|
|
benchmarkEncode(b, 10, 2, 10000)
|
|
}
|
|
|
|
func BenchmarkEncode100x20x10000(b *testing.B) {
|
|
benchmarkEncode(b, 100, 20, 10000)
|
|
}
|
|
|
|
func BenchmarkEncode17x3x1M(b *testing.B) {
|
|
benchmarkEncode(b, 17, 3, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 10 data shards and 4 parity shards with 16MB each.
|
|
func BenchmarkEncode10x4x16M(b *testing.B) {
|
|
benchmarkEncode(b, 10, 4, 16*1024*1024)
|
|
}
|
|
|
|
// Benchmark 5 data shards and 2 parity shards with 1MB each.
|
|
func BenchmarkEncode5x2x1M(b *testing.B) {
|
|
benchmarkEncode(b, 5, 2, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 1 data shards and 2 parity shards with 1MB each.
|
|
func BenchmarkEncode10x2x1M(b *testing.B) {
|
|
benchmarkEncode(b, 10, 2, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 10 data shards and 4 parity shards with 1MB each.
|
|
func BenchmarkEncode10x4x1M(b *testing.B) {
|
|
benchmarkEncode(b, 10, 4, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 50 data shards and 20 parity shards with 1MB each.
|
|
func BenchmarkEncode50x20x1M(b *testing.B) {
|
|
benchmarkEncode(b, 50, 20, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 17 data shards and 3 parity shards with 16MB each.
|
|
func BenchmarkEncode17x3x16M(b *testing.B) {
|
|
benchmarkEncode(b, 17, 3, 16*1024*1024)
|
|
}
|
|
|
|
func benchmarkVerify(b *testing.B, dataShards, parityShards, shardSize int) {
|
|
r, err := New(dataShards, parityShards)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
shards := make([][]byte, parityShards+dataShards)
|
|
for s := range shards {
|
|
shards[s] = make([]byte, shardSize)
|
|
}
|
|
|
|
rand.Seed(0)
|
|
for s := 0; s < dataShards; s++ {
|
|
fillRandom(shards[s])
|
|
}
|
|
err = r.Encode(shards)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
b.SetBytes(int64(shardSize * dataShards))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
_, err = r.Verify(shards)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Benchmark 10 data slices with 2 parity slices holding 10000 bytes each
|
|
func BenchmarkVerify10x2x10000(b *testing.B) {
|
|
benchmarkVerify(b, 10, 2, 10000)
|
|
}
|
|
|
|
// Benchmark 50 data slices with 5 parity slices holding 100000 bytes each
|
|
func BenchmarkVerify50x5x50000(b *testing.B) {
|
|
benchmarkVerify(b, 50, 5, 100000)
|
|
}
|
|
|
|
// Benchmark 10 data slices with 2 parity slices holding 1MB bytes each
|
|
func BenchmarkVerify10x2x1M(b *testing.B) {
|
|
benchmarkVerify(b, 10, 2, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 5 data slices with 2 parity slices holding 1MB bytes each
|
|
func BenchmarkVerify5x2x1M(b *testing.B) {
|
|
benchmarkVerify(b, 5, 2, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 10 data slices with 4 parity slices holding 1MB bytes each
|
|
func BenchmarkVerify10x4x1M(b *testing.B) {
|
|
benchmarkVerify(b, 10, 4, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 5 data slices with 2 parity slices holding 1MB bytes each
|
|
func BenchmarkVerify50x20x1M(b *testing.B) {
|
|
benchmarkVerify(b, 50, 20, 1024*1024)
|
|
}
|
|
|
|
// Benchmark 10 data slices with 4 parity slices holding 16MB bytes each
|
|
func BenchmarkVerify10x4x16M(b *testing.B) {
|
|
benchmarkVerify(b, 10, 4, 16*1024*1024)
|
|
}
|
|
|
|
// Simple example of how to use all functions of the Encoder.
|
|
// Note that all error checks have been removed to keep it short.
|
|
func ExampleEncoder() {
|
|
// Create some sample data
|
|
var data = make([]byte, 250000)
|
|
fillRandom(data)
|
|
|
|
// Create an encoder with 17 data and 3 parity slices.
|
|
enc, _ := New(17, 3)
|
|
|
|
// Split the data into shards
|
|
shards, _ := enc.Split(data)
|
|
|
|
// Encode the parity set
|
|
_ = enc.Encode(shards)
|
|
|
|
// Verify the parity set
|
|
ok, _ := enc.Verify(shards)
|
|
if ok {
|
|
fmt.Println("ok")
|
|
}
|
|
|
|
// Delete two shards
|
|
shards[10], shards[11] = nil, nil
|
|
|
|
// Reconstruct the shards
|
|
_ = enc.Reconstruct(shards)
|
|
|
|
// Verify the data set
|
|
ok, _ = enc.Verify(shards)
|
|
if ok {
|
|
fmt.Println("ok")
|
|
}
|
|
// Output: ok
|
|
// ok
|
|
}
|