vitalif
/
jerasure
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Ran through all of the examples to make them current. I'll have to 

do another sanity-checking pass, and fix multby_2 in the reed_sol code,
but then we're done.
master
Jim Plank 2014-01-24 16:50:41 -05:00
parent 9cca12d46f
commit 9124ad1382
28 changed files with 1092 additions and 1159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Examples/Makefile.am
View File

@ -17,7 +17,6 @@ bin_PROGRAMS = jerasure_01 \
reed_sol_04 \ reed_sol_04 \
reed_sol_test_gf \ reed_sol_test_gf \
reed_sol_time_gf \ reed_sol_time_gf \
reed_sol_hard_time_gf \
cauchy_01 \ cauchy_01 \
cauchy_02 \ cauchy_02 \
cauchy_03 \ cauchy_03 \
@ -68,9 +67,6 @@ reed_sol_test_gf_LDADD = ../src/libJerasure.la
reed_sol_time_gf_SOURCES = reed_sol_time_gf.c reed_sol_time_gf_SOURCES = reed_sol_time_gf.c
reed_sol_time_gf_LDADD = ../src/libJerasure.la reed_sol_time_gf_LDADD = ../src/libJerasure.la
reed_sol_hard_time_gf_SOURCES = reed_sol_hard_time_gf.c
reed_sol_hard_time_gf_LDADD = ../src/libJerasure.la
cauchy_01_SOURCES = cauchy_01.c cauchy_01_SOURCES = cauchy_01.c
cauchy_01_LDADD = ../src/libJerasure.la cauchy_01_LDADD = ../src/libJerasure.la

14
Examples/Makefile.in
View File

@ -41,7 +41,7 @@ bin_PROGRAMS = jerasure_01$(EXEEXT) jerasure_02$(EXEEXT) \
jerasure_06$(EXEEXT) jerasure_07$(EXEEXT) jerasure_08$(EXEEXT) \ jerasure_06$(EXEEXT) jerasure_07$(EXEEXT) jerasure_08$(EXEEXT) \
reed_sol_01$(EXEEXT) reed_sol_02$(EXEEXT) reed_sol_03$(EXEEXT) \ reed_sol_01$(EXEEXT) reed_sol_02$(EXEEXT) reed_sol_03$(EXEEXT) \
reed_sol_04$(EXEEXT) reed_sol_test_gf$(EXEEXT) \ reed_sol_04$(EXEEXT) reed_sol_test_gf$(EXEEXT) \
reed_sol_time_gf$(EXEEXT) reed_sol_hard_time_gf$(EXEEXT) \ reed_sol_time_gf$(EXEEXT) \
cauchy_01$(EXEEXT) cauchy_02$(EXEEXT) cauchy_03$(EXEEXT) \ cauchy_01$(EXEEXT) cauchy_02$(EXEEXT) cauchy_03$(EXEEXT) \
cauchy_04$(EXEEXT) liberation_01$(EXEEXT) encoder$(EXEEXT) \ cauchy_04$(EXEEXT) liberation_01$(EXEEXT) encoder$(EXEEXT) \
decoder$(EXEEXT) decoder$(EXEEXT)
@ -120,9 +120,6 @@ reed_sol_03_DEPENDENCIES = ../src/libJerasure.la
am_reed_sol_04_OBJECTS = reed_sol_04.$(OBJEXT) am_reed_sol_04_OBJECTS = reed_sol_04.$(OBJEXT)
reed_sol_04_OBJECTS = $(am_reed_sol_04_OBJECTS) reed_sol_04_OBJECTS = $(am_reed_sol_04_OBJECTS)
reed_sol_04_DEPENDENCIES = ../src/libJerasure.la reed_sol_04_DEPENDENCIES = ../src/libJerasure.la
am_reed_sol_hard_time_gf_OBJECTS = reed_sol_hard_time_gf.$(OBJEXT)
reed_sol_hard_time_gf_OBJECTS = $(am_reed_sol_hard_time_gf_OBJECTS)
reed_sol_hard_time_gf_DEPENDENCIES = ../src/libJerasure.la
am_reed_sol_test_gf_OBJECTS = reed_sol_test_gf.$(OBJEXT) am_reed_sol_test_gf_OBJECTS = reed_sol_test_gf.$(OBJEXT)
reed_sol_test_gf_OBJECTS = $(am_reed_sol_test_gf_OBJECTS) reed_sol_test_gf_OBJECTS = $(am_reed_sol_test_gf_OBJECTS)
reed_sol_test_gf_DEPENDENCIES = ../src/libJerasure.la reed_sol_test_gf_DEPENDENCIES = ../src/libJerasure.la
@ -150,7 +147,7 @@ SOURCES = $(cauchy_01_SOURCES) $(cauchy_02_SOURCES) \
$(jerasure_08_SOURCES) $(liberation_01_SOURCES) \ $(jerasure_08_SOURCES) $(liberation_01_SOURCES) \
$(reed_sol_01_SOURCES) $(reed_sol_02_SOURCES) \ $(reed_sol_01_SOURCES) $(reed_sol_02_SOURCES) \
$(reed_sol_03_SOURCES) $(reed_sol_04_SOURCES) \ $(reed_sol_03_SOURCES) $(reed_sol_04_SOURCES) \
$(reed_sol_hard_time_gf_SOURCES) $(reed_sol_test_gf_SOURCES) \ $(reed_sol_test_gf_SOURCES) \
$(reed_sol_time_gf_SOURCES) $(reed_sol_time_gf_SOURCES)
DIST_SOURCES = $(cauchy_01_SOURCES) $(cauchy_02_SOURCES) \ DIST_SOURCES = $(cauchy_01_SOURCES) $(cauchy_02_SOURCES) \
$(cauchy_03_SOURCES) $(cauchy_04_SOURCES) $(decoder_SOURCES) \ $(cauchy_03_SOURCES) $(cauchy_04_SOURCES) $(decoder_SOURCES) \
@ -161,7 +158,7 @@ DIST_SOURCES = $(cauchy_01_SOURCES) $(cauchy_02_SOURCES) \
$(jerasure_08_SOURCES) $(liberation_01_SOURCES) \ $(jerasure_08_SOURCES) $(liberation_01_SOURCES) \
$(reed_sol_01_SOURCES) $(reed_sol_02_SOURCES) \ $(reed_sol_01_SOURCES) $(reed_sol_02_SOURCES) \
$(reed_sol_03_SOURCES) $(reed_sol_04_SOURCES) \ $(reed_sol_03_SOURCES) $(reed_sol_04_SOURCES) \
$(reed_sol_hard_time_gf_SOURCES) $(reed_sol_test_gf_SOURCES) \ $(reed_sol_test_gf_SOURCES) \
$(reed_sol_time_gf_SOURCES) $(reed_sol_time_gf_SOURCES)
ETAGS = etags ETAGS = etags
CTAGS = ctags CTAGS = ctags
@ -306,8 +303,6 @@ reed_sol_test_gf_SOURCES = reed_sol_test_gf.c
reed_sol_test_gf_LDADD = ../src/libJerasure.la reed_sol_test_gf_LDADD = ../src/libJerasure.la
reed_sol_time_gf_SOURCES = reed_sol_time_gf.c reed_sol_time_gf_SOURCES = reed_sol_time_gf.c
reed_sol_time_gf_LDADD = ../src/libJerasure.la reed_sol_time_gf_LDADD = ../src/libJerasure.la
reed_sol_hard_time_gf_SOURCES = reed_sol_hard_time_gf.c
reed_sol_hard_time_gf_LDADD = ../src/libJerasure.la
cauchy_01_SOURCES = cauchy_01.c cauchy_01_SOURCES = cauchy_01.c
cauchy_01_LDADD = ../src/libJerasure.la cauchy_01_LDADD = ../src/libJerasure.la
cauchy_02_SOURCES = cauchy_02.c cauchy_02_SOURCES = cauchy_02.c
@ -456,9 +451,6 @@ reed_sol_03$(EXEEXT): $(reed_sol_03_OBJECTS) $(reed_sol_03_DEPENDENCIES) $(EXTRA
reed_sol_04$(EXEEXT): $(reed_sol_04_OBJECTS) $(reed_sol_04_DEPENDENCIES) $(EXTRA_reed_sol_04_DEPENDENCIES) reed_sol_04$(EXEEXT): $(reed_sol_04_OBJECTS) $(reed_sol_04_DEPENDENCIES) $(EXTRA_reed_sol_04_DEPENDENCIES)
@rm -f reed_sol_04$(EXEEXT) @rm -f reed_sol_04$(EXEEXT)
$(LINK) $(reed_sol_04_OBJECTS) $(reed_sol_04_LDADD) $(LIBS) $(LINK) $(reed_sol_04_OBJECTS) $(reed_sol_04_LDADD) $(LIBS)
reed_sol_hard_time_gf$(EXEEXT): $(reed_sol_hard_time_gf_OBJECTS) $(reed_sol_hard_time_gf_DEPENDENCIES) $(EXTRA_reed_sol_hard_time_gf_DEPENDENCIES)
@rm -f reed_sol_hard_time_gf$(EXEEXT)
$(LINK) $(reed_sol_hard_time_gf_OBJECTS) $(reed_sol_hard_time_gf_LDADD) $(LIBS)
reed_sol_test_gf$(EXEEXT): $(reed_sol_test_gf_OBJECTS) $(reed_sol_test_gf_DEPENDENCIES) $(EXTRA_reed_sol_test_gf_DEPENDENCIES) reed_sol_test_gf$(EXEEXT): $(reed_sol_test_gf_OBJECTS) $(reed_sol_test_gf_DEPENDENCIES) $(EXTRA_reed_sol_test_gf_DEPENDENCIES)
@rm -f reed_sol_test_gf$(EXEEXT) @rm -f reed_sol_test_gf$(EXEEXT)
$(LINK) $(reed_sol_test_gf_OBJECTS) $(reed_sol_test_gf_LDADD) $(LIBS) $(LINK) $(reed_sol_test_gf_OBJECTS) $(reed_sol_test_gf_LDADD) $(LIBS)

45
Examples/cauchy_01.c
View File

@ -1,5 +1,5 @@
/* * /* *
* Copyright (c) 2013, James S. Plank and Kevin Greenan * Copyright (c) 2014, James S. Plank and Kevin Greenan
* All rights reserved. * All rights reserved.
* *
* Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure * Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
@ -39,6 +39,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdint.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include "jerasure.h" #include "jerasure.h"
@ -48,36 +49,56 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: cauchy_01 n w - Prints the number of ones in the bitmatrix representation of n in GF(2^w).\n"); fprintf(stderr, "usage: cauchy_01 n w - Converts the value n to a bitmatrix using GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " It also prints out the bit-matrix and confirms that the number of ones is correct.\n"); fprintf(stderr, " It prints the bitmatrix, and reports on the numberof ones.\n");
fprintf(stderr, " Use 0x to input n in hexadecimal.\n");
fprintf(stderr, " W must be <= 32.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, "This demonstrates: cauchy_n_ones()\n"); fprintf(stderr, "This demonstrates: cauchy_n_ones()\n");
fprintf(stderr, " jerasure_matrix_to_bitmatrix()\n"); fprintf(stderr, " jerasure_matrix_to_bitmatrix()\n");
fprintf(stderr, " jerasure_print_bitmatrix()\n"); fprintf(stderr, " jerasure_print_bitmatrix()\n");
if (s != NULL) fprintf(stderr, "%s\n", s); if (s != NULL) fprintf(stderr, "\n%s\n", s);
exit(1); exit(1);
} }
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
int n, i, no, w; uint32_t n;
int i, no, w;
int *bitmatrix; int *bitmatrix;
if (argc != 3) usage(NULL); if (argc != 3) usage(NULL);
if (sscanf(argv[1], "%d", &n) == 0 || n <= 0) usage("Bad n"); if (sscanf(argv[1], "0x%x", &n) == 0) {
if (sscanf(argv[1], "%d", &n) == 0) usage("Bad n");
}
if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (w < 30 && n >= (1 << w)) usage("n is too big"); if (w == 31) {
if (n & 0x80000000L) usage("Bad n/w combination (n not between 0 and 2^w-1)\n");
} else if (w < 31) {
if (n >= (1 << w)) usage("Bad n/w combination (n not between 0 and 2^w-1)\n");
}
bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n); bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n);
printf("<HTML><title>cauchy_01 %u %d</title>\n", w, n);
printf("<HTML><h3>cauchy_01 %u %d</h3>\n", w, n);
printf("<pre>\n");
if (w == 32) {
printf("Converted the value 0x%x to the following bitmatrix:\n\n", n);
} else {
printf("Converted the value %d (0x%x) to the following bitmatrix:\n\n", n, n);
}
jerasure_print_bitmatrix(bitmatrix, w, w, w);
printf("\n");
no = 0; no = 0;
for (i = 0; i < w*w; i++) no += bitmatrix[i]; for (i = 0; i < w*w; i++) no += bitmatrix[i];
if (no != cauchy_n_ones(n, w)) {
fprintf(stderr, "Jerasure error: # ones in the bitmatrix (%d) doesn't match cauchy_n_ones() (%d).\n",
no, cauchy_n_ones(n, w));
exit(1);
}
printf("# Ones: %d\n", cauchy_n_ones(n, w)); printf("# Ones: %d\n", cauchy_n_ones(n, w));
printf("\n");
printf("Bitmatrix has %d ones\n", no);
printf("\n");
jerasure_print_bitmatrix(bitmatrix, w, w, w);
return 0; return 0;
} }

245
Examples/cauchy_02.c
View File

@ -37,15 +37,15 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
/* /*
revised by S. Simmerman revised by S. Simmerman 2/25/08
2/25/08 Re-revised by JSP to use GF-Complete - 1/2014
*/ */
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "cauchy.h" #include "cauchy.h"
@ -53,79 +53,76 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: cauchy_02 k m w - Scheduled CRS coding example with the original matrix in GF(2^w).\n"); fprintf(stderr, "usage: cauchy_02 k m w seed - CRS coding example using Bloemer's original matrix.\n");
fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. It sets up a Cauchy distribution matrix using the original\n");
fprintf(stderr, " Cauchy Distribution matrix construction algorithm, then it encodes\n");
fprintf(stderr, " k devices of w*%d bytes using smart bit-matrix scheduling.\n", sizeof(long));
fprintf(stderr, " It decodes using bit-matrix scheduling as well.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, "k+m must be <= 2^w\n");
fprintf(stderr, "This sets up a generator matrix (G^T) in GF(2^w) whose last m rows are\n");
fprintf(stderr, "created from a Cauchy matrix, using the original definition from [Bloemer95].\n");
fprintf(stderr, "It converts this matrix to a bitmatrix, and then it encodes w packets from\n");
fprintf(stderr, "each of k disks (simulated) onto w packets on each of m disks. Packets are \n");
fprintf(stderr, "simply longs. Then, it deletes m random disks, and decodes. \n");
fprintf(stderr, "\n");
fprintf(stderr, "The encoding and decoding are done twice, first, with jerasure_bitmatrix_encode()\n");
fprintf(stderr, "and jerasure_bitmatrix_decode(), and second using 'smart' scheduling with\n");
fprintf(stderr, "jerasure_schedule_encode() and jerasure_schedule_decode_lazy().\n");
fprintf(stderr, "\n");
fprintf(stderr, "This demonstrates: cauchy_original_coding_matrix()\n"); fprintf(stderr, "This demonstrates: cauchy_original_coding_matrix()\n");
fprintf(stderr, " cauchy_xy_coding_matrix()\n"); fprintf(stderr, " jerasure_bitmatrix_encode()\n");
fprintf(stderr, " jerasure_bitmatrix_decode()\n");
fprintf(stderr, " cauchy_n_ones()\n"); fprintf(stderr, " cauchy_n_ones()\n");
fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n"); fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n");
fprintf(stderr, " jerasure_schedule_encode()\n"); fprintf(stderr, " jerasure_schedule_encode()\n");
fprintf(stderr, " jerasure_schedule_decode_lazy()\n"); fprintf(stderr, " jerasure_schedule_decode_lazy()\n");
fprintf(stderr, " jerasure_print_matrix()\n"); fprintf(stderr, " jerasure_print_matrix()\n");
fprintf(stderr, " jerasure_print_bitmatrix()\n");
fprintf(stderr, " jerasure_get_stats()\n"); fprintf(stderr, " jerasure_get_stats()\n");
if (s != NULL) fprintf(stderr, "%s\n", s); if (s != NULL) fprintf(stderr, "%s\n", s);
exit(1); exit(1);
} }
static void print_data_and_coding(int k, int m, int w, int psize, static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12; for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
printf("</tr>\n");
printf("%-*sCoding\n", sp, "Data"); printf("<td align=right><pre>");
for(i = 0; i < n; i++) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
for (j = 0; j < w; j++) { printf("</pre></td>\n");
if(i < k) { for (i = 0; i < ndevices; i++) {
printf("<td><pre>");
if(j==0) printf("D%-2d p%-2d:", i,j); up = (unsigned char *) ptrs[i];
else printf(" p%-2d:", j); for (j = 0; j < size/packetsize; j++) {
for(x = 0; x < psize; x +=4) { for (x = 0; x < packetsize; x++) {
memcpy(&l, data[i]+j*psize+x, sizeof(long)); if (x > 0 && x%4 == 0) printf(" ");
printf(" %08lx", l); printf("%02x", up[j*packetsize+x]);
} }
printf(" "); printf("\n");
} }
else printf("%*s", sp, ""); printf("</td>\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</tr></table></center>\n");
else printf(" p%-2d:", j);
for(x = 0; x < psize; x +=4) {
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
long l;
int k, w, i, j, m; int k, w, i, j, m;
int *matrix, *bitmatrix, *m2, *x, *y; int *matrix, *bitmatrix, **schedule;
char **data, **coding, **ptrs; char **data, **coding, **ptrs, **dcopy, **ccopy;
int **smart;
int no; int no;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double mstats[3], sstats[3];
uint32_t seed;
if (argc != 4) usage(NULL); if (argc != 5) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
matrix = cauchy_original_coding_matrix(k, m, w); matrix = cauchy_original_coding_matrix(k, m, w);
@ -133,42 +130,90 @@ int main(int argc, char **argv)
usage("couldn't make coding matrix"); usage("couldn't make coding matrix");
} }
/* Print out header information to the output file. */
printf("<HTML>\n");
printf("<TITLE>Jerasure Example Output: cauchy_02 %d %d %d %d</TITLE>\n", k, m, w, seed);
printf("<h2>Jerasure Example Output: cauchy_02 %d %d %d %d</h3>\n", k, m, w, seed);
printf("<hr>\n");
printf("Parameters:\n");
printf("<UL><LI> Number of data disks <i>(k)</i>: %d\n", k);
printf("<LI> Number of coding disks <i>(m)</i>: %d\n", m);
printf("<LI> Word size of the Galois Field: <i>(w)</i>: %d\n", w);
printf("<LI> Seed for the random number generator: %d\n", seed);
printf("<LI> Number of bytes stored per disk: %ld\n", sizeof(long)*w);
printf("<LI> Number of packets stored per disk: %d\n", w);
printf("<LI> Number of bytes per packet: %ld\n", sizeof(long));
printf("</UL>\n");
/* Print out the matrix and the bitmatrix */
printf("<hr>\n");
printf("Here is the matrix, which was created with <b>cauchy_original_coding_matrix()</b>.\n");
printf("This is not the best matrix to use, but we include it to show an example\n");
printf("of <b>cauchy_original_coding_matrix()</b>. For the best matrix and encoding/decoding\n");
printf("methodology, see <b>cauchy_04.</b><p><pre>\n");
jerasure_print_matrix(matrix, m, k, w);
printf("</pre>\n");
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
no = 0; no = 0;
for (i = 0; i < k*m; i++) { for (i = 0; i < k*m; i++) {
no += cauchy_n_ones(matrix[i], w); no += cauchy_n_ones(matrix[i], w);
} }
printf("Matrix has %d ones\n\n", no);
jerasure_print_matrix(matrix, m, k, w);
printf("\n", no);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); printf("The bitmatrix, which has %d one%s:<p><pre>\n", no, (no == 1) ? "" : "s");
jerasure_print_bitmatrix(bitmatrix, m*w, k*w, w);
printf("</pre>\n");
printf("<hr>\n");
MOA_Seed(seed);
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
dcopy = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { dcopy[i] = talloc(char, sizeof(long)*w);
l = lrand48(); MOA_Fill_Random_Region(data[i], sizeof(long)*w);
memcpy(data[i]+j*sizeof(long), &l, sizeof(long)); memcpy(dcopy[i], data[i], sizeof(long)*w);
}
printf("Here are the packets on the data disks:<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
coding = talloc(char *, m);
ccopy = talloc(char *, m);
for (i = 0; i < m; i++) {
coding[i] = talloc(char, sizeof(long)*w);
ccopy[i] = talloc(char, sizeof(long)*w);
}
jerasure_bitmatrix_encode(k, m, w, bitmatrix, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(mstats);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
jerasure_schedule_encode(k, m, w, schedule, data, ccopy, w*sizeof(long), sizeof(long));
jerasure_get_stats(sstats);
printf("<p>Encoding with jerasure_bitmatrix_encode() - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
printf("Encoding with jerasure_schedule_encode() - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
for (i = 0; i < m; i++) {
if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("Problem: the two encodings don't match on disk C%x\n", i);
exit(0);
} }
} }
coding = talloc(char *, m); printf("Here are the packets on the coding disks.<br>\n");
for (i = 0; i < m; i++) { print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
coding[i] = talloc(char, sizeof(long)*w); printf("<hr>\n");
}
jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats);
printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(31, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -176,29 +221,47 @@ int main(int argc, char **argv)
} }
} }
erasures[i] = -1; erasures[i] = -1;
printf("Erasures on the following devices:");
for (i = 0; erasures[i] != -1; i++) {
printf(" %c%x", ((erasures[i] < k) ? 'D' : 'C'), (erasures[i] < k ? erasures[i] : erasures[i]-k));
}
printf("<br>\nHere is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
printf("Erased %d random devices:\n\n", m); jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data, coding, w*sizeof(long), sizeof(long));
print_data_and_coding(k, m, w, sizeof(long), data, coding); jerasure_get_stats(mstats);
jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(stats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("<p>Decoded with jerasure_bitmatrix_decode - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
x = talloc(int, m); printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
y = talloc(int, k); }
if (x == NULL || y == NULL) { perror("malloc"); exit(1); } for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
for (i = 0; i < m; i++) x[i] = i; printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
for (i = 0; i < k; i++) y[i] = m+i;
m2 = cauchy_xy_coding_matrix(k, m, w, x, y);
if (memcmp(matrix, m2, sizeof(int)*k*m) != 0) {
printf("Error -- the matrices made by original and xy don't match\n");
exit(1);
} else {
printf("Generated the identical matrix using cauchy_xy_coding_matrix()\n");
} }
for (i = 0; erasures[i] != -1; i++) {
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
}
return 0; jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(sstats);
printf("jerasure_schedule_decode_lazy - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
printf("Here is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
} }

263
Examples/cauchy_03.c
View File

@ -37,15 +37,15 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
/* /*
revised by S. Simmerman revised by S. Simmerman 2/25/08
2/25/08 Re-revised by JSP to use GF-Complete - 1/2014
*/ */
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "cauchy.h" #include "cauchy.h"
@ -53,128 +53,185 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: cauchy_03 k m w - Scheduled CRS coding example with improved matrix in GF(2^w).\n"); fprintf(stderr, "usage: cauchy_03 k m w seed - CRS coding example improving the matrix.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. It sets up a Cauchy distribution matrix using the original\n"); fprintf(stderr, "k+m must be <= 2^w\n");
fprintf(stderr, " Cauchy Distribution matrix construction algorithm, then improves it\n"); fprintf(stderr, "This sets up a generator matrix (G^T) in GF(2^w) whose last m rows are\n");
fprintf(stderr, " with cauchy_improve_coding_matrix(). Then it does the same encoding and\n"); fprintf(stderr, "created from a Cauchy matrix, using the original definition from [Bloemer95].\n");
fprintf(stderr, " decoding as cauchy_02.\n"); fprintf(stderr, "This is done with cauchy_xy_coding_matrix().\n");
fprintf(stderr, " \n"); fprintf(stderr, "\n");
fprintf(stderr, "This demonstrates: cauchy_original_coding_matrix()\n"); fprintf(stderr, "It then it improves the matrix, which yields a different bitmatrix that is\n");
fprintf(stderr, "MDS like the original bitmatrix, but it will yield a bitmatrix with fewer ones.\n");
fprintf(stderr, "Then, it encodes w packets from each of k disks (simulated) onto w packets on\n");
fprintf(stderr, "on each of m disks. Packets are longs. Then, it deletes m random disks, and decodes.\n");
fprintf(stderr, "\n");
fprintf(stderr, "The encoding and decoding are done twice, first, with jerasure_bitmatrix_encode()\n");
fprintf(stderr, "and jerasure_bitmatrix_decode(), and second using 'smart' scheduling with\n");
fprintf(stderr, "jerasure_schedule_encode() and jerasure_schedule_decode_lazy().\n");
fprintf(stderr, "\n");
fprintf(stderr, "This demonstrates: cauchy_xy_coding_matrix()\n");
fprintf(stderr, " cauchy_improve_coding_matrix()\n"); fprintf(stderr, " cauchy_improve_coding_matrix()\n");
fprintf(stderr, " jerasure_bitmatrix_encode()\n");
fprintf(stderr, " jerasure_bitmatrix_decode()\n");
fprintf(stderr, " cauchy_n_ones()\n"); fprintf(stderr, " cauchy_n_ones()\n");
fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n"); fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n");
fprintf(stderr, " jerasure_schedule_encode()\n"); fprintf(stderr, " jerasure_schedule_encode()\n");
fprintf(stderr, " jerasure_schedule_decode_lazy()\n"); fprintf(stderr, " jerasure_schedule_decode_lazy()\n");
fprintf(stderr, " jerasure_print_matrix()\n"); fprintf(stderr, " jerasure_print_matrix()\n");
fprintf(stderr, " jerasure_print_bitmatrix()\n");
fprintf(stderr, " jerasure_get_stats()\n"); fprintf(stderr, " jerasure_get_stats()\n");
if (s != NULL) fprintf(stderr, "%s\n", s); if (s != NULL) fprintf(stderr, "%s\n", s);
exit(1); exit(1);
} }
static void print_data_and_coding(int k, int m, int w, int psize, static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12; for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
printf("</tr>\n");
printf("%-*sCoding\n", sp, "Data"); printf("<td align=right><pre>");
for(i = 0; i < n; i++) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
for (j = 0; j < w; j++) { printf("</pre></td>\n");
if(i < k) { for (i = 0; i < ndevices; i++) {
printf("<td><pre>");
if(j==0) printf("D%-2d p%-2d:", i,j); up = (unsigned char *) ptrs[i];
else printf(" p%-2d:", j); for (j = 0; j < size/packetsize; j++) {
for(x = 0; x < psize; x +=4) { for (x = 0; x < packetsize; x++) {
memcpy(&l, data[i]+j*psize+x, sizeof(long)); if (x > 0 && x%4 == 0) printf(" ");
printf(" %08lx", l); printf("%02x", up[j*packetsize+x]);
} }
printf(" "); printf("\n");
} }
else printf("%*s", sp, ""); printf("</td>\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</tr></table></center>\n");
else printf(" p%-2d:", j);
for(x = 0; x < psize; x +=4) {
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
long l;
int k, w, i, j, m; int k, w, i, j, m;
int *matrix, *bitmatrix, *m2, *x, *y; int *matrix, *bitmatrix, **schedule;
char **data, **coding, **ptrs; char **data, **coding, **ptrs, **dcopy, **ccopy;
int **smart;
int no; int no;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double mstats[3], sstats[3];
uint32_t seed;
int *X, *Y;
if (argc != 4) usage(NULL); if (argc != 5) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
matrix = cauchy_original_coding_matrix(k, m, w); X = talloc(int, m);
Y = talloc(int, k);
for (i = 0; i < m; i++) X[i] = i;
for (i = 0; i < k; i++) Y[i] = m+i;
matrix = cauchy_xy_coding_matrix(k, m, w, X, Y);
if (matrix == NULL) { if (matrix == NULL) {
usage("couldn't make coding matrix"); usage("couldn't make coding matrix");
} }
no = 0; /* Print out header information to the output file. */
for (i = 0; i < k*m; i++) { printf("<HTML>\n");
no += cauchy_n_ones(matrix[i], w); printf("<TITLE>Jerasure Example Output: cauchy_03 %d %d %d %d</TITLE>\n", k, m, w, seed);
} printf("<h2>Jerasure Example Output: cauchy_03 %d %d %d %d</h3>\n", k, m, w, seed);
printf("The Original Matrix has %d ones\n", no);
cauchy_improve_coding_matrix(k, m, w, matrix); printf("<hr>\n");
no = 0; printf("Parameters:\n");
for (i = 0; i < k*m; i++) { printf("<UL><LI> Number of data disks <i>(k)</i>: %d\n", k);
no += cauchy_n_ones(matrix[i], w); printf("<LI> Number of coding disks <i>(m)</i>: %d\n", m);
} printf("<LI> Word size of the Galois Field: <i>(w)</i>: %d\n", w);
printf("The Improved Matrix has %d ones\n\n", no); printf("<LI> Seed for the random number generator: %d\n", seed);
printf("<LI> Number of bytes stored per disk: %ld\n", sizeof(long)*w);
printf("<LI> Number of packets stored per disk: %d\n", w);
printf("<LI> Number of bytes per packet: %ld\n", sizeof(long));
printf("</UL>\n");
/* Print out the matrix and the bitmatrix */
printf("<hr>\n");
printf("Here is the matrix, which was created with <b>cauchy_xy_coding_matrix()</b>.\n");
printf("<pre>\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n", no); printf("</pre>\n");
cauchy_improve_coding_matrix(k, m, w, matrix);
printf("<hr>\n");
printf("Here is the matrix improved with <b>cauchy_improve_coding_matrix()</b>.\n");
printf("<pre>\n");
jerasure_print_matrix(matrix, m, k, w);
printf("</pre>\n");
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); no = 0;
for (i = 0; i < k*m; i++) {
no += cauchy_n_ones(matrix[i], w);
}
printf("The bitmatrix, which has %d one%s:<p><pre>\n", no, (no == 1) ? "" : "s");
jerasure_print_bitmatrix(bitmatrix, m*w, k*w, w);
printf("</pre>\n");
printf("<hr>\n");
MOA_Seed(seed);
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
dcopy = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { dcopy[i] = talloc(char, sizeof(long)*w);
l = lrand48(); MOA_Fill_Random_Region(data[i], sizeof(long)*w);
memcpy(data[i]+j*sizeof(long), &l, sizeof(long)); memcpy(dcopy[i], data[i], sizeof(long)*w);
}
printf("Here are the packets on the data disks:<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
coding = talloc(char *, m);
ccopy = talloc(char *, m);
for (i = 0; i < m; i++) {
coding[i] = talloc(char, sizeof(long)*w);
ccopy[i] = talloc(char, sizeof(long)*w);
}
jerasure_bitmatrix_encode(k, m, w, bitmatrix, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(mstats);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
jerasure_schedule_encode(k, m, w, schedule, data, ccopy, w*sizeof(long), sizeof(long));
jerasure_get_stats(sstats);
printf("<p>Encoding with jerasure_bitmatrix_encode() - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
printf("Encoding with jerasure_schedule_encode() - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
for (i = 0; i < m; i++) {
if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("Problem: the two encodings don't match on disk C%x\n", i);
exit(0);
} }
} }
coding = talloc(char *, m); printf("Here are the packets on the coding disks.<br>\n");
for (i = 0; i < m; i++) { print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
coding[i] = talloc(char, sizeof(long)*w); printf("<hr>\n");
}
jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats);
printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(31, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -182,15 +239,47 @@ int main(int argc, char **argv)
} }
} }
erasures[i] = -1; erasures[i] = -1;
printf("Erasures on the following devices:");
for (i = 0; erasures[i] != -1; i++) {
printf(" %c%x", ((erasures[i] < k) ? 'D' : 'C'), (erasures[i] < k ? erasures[i] : erasures[i]-k));
}
printf("<br>\nHere is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(mstats);
printf("<p>Decoded with jerasure_bitmatrix_decode - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; erasures[i] != -1; i++) {
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
}
printf("Erased %d random devices:\n\n", m);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1); jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(stats); jerasure_get_stats(sstats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("jerasure_schedule_decode_lazy - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
return 0; for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
printf("Here is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
} }

236
Examples/cauchy_04.c
View File

@ -37,15 +37,15 @@
* POSSIBILITY OF SUCH DAMAGE. * POSSIBILITY OF SUCH DAMAGE.
*/ */
/* /*
revised by S. Simmerman revised by S. Simmerman 2/25/08
2/25/08 Re-revised by JSP to use GF-Complete - 1/2014
*/ */
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "cauchy.h" #include "cauchy.h"
@ -53,78 +53,77 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: cauchy_04 k m w - Scheduled CRS coding example with a good matrix in GF(2^w).\n"); fprintf(stderr, "usage: cauchy_04 k m w seed - CRS coding example improving the matrix.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. It sets up a Cauchy distribution matrix using \n"); fprintf(stderr, "k+m must be <= 2^w\n");
fprintf(stderr, " cauchy_good_coding_matrix(), then it encodes\n"); fprintf(stderr, "This sets up a generator matrix (G^T) in GF(2^w) whose last m rows are\n");
fprintf(stderr, " k devices of w*%d bytes using smart bit-matrix scheduling.\n", sizeof(long)); fprintf(stderr, "a 'good' matrix, created with cauchy_good_general_coding_matrix().\n");
fprintf(stderr, " It decodes using bit-matrix scheduling as well.\n"); fprintf(stderr, "It converts this matrix to a bitmatrix.\n");
fprintf(stderr, " \n"); fprintf(stderr, "\n");
fprintf(stderr, "This demonstrates: cauchy_original_coding_matrix()\n"); fprintf(stderr, "Then, it encodes w packets from each of k disks (simulated) onto w packets on\n");
fprintf(stderr, "on each of m disks. Packets are longs. Then, it deletes m random disks, and decodes.\n");
fprintf(stderr, "\n");
fprintf(stderr, "The encoding and decoding are done twice, first, with jerasure_bitmatrix_encode()\n");
fprintf(stderr, "and jerasure_bitmatrix_decode(), and second using 'smart' scheduling with\n");
fprintf(stderr, "jerasure_schedule_encode() and jerasure_schedule_decode_lazy().\n");
fprintf(stderr, "\n");
fprintf(stderr, "This demonstrates: cauchy_good_general_coding_matrix()\n");
fprintf(stderr, " jerasure_bitmatrix_encode()\n");
fprintf(stderr, " jerasure_bitmatrix_decode()\n");
fprintf(stderr, " cauchy_n_ones()\n"); fprintf(stderr, " cauchy_n_ones()\n");
fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n"); fprintf(stderr, " jerasure_smart_bitmatrix_to_schedule()\n");
fprintf(stderr, " jerasure_schedule_encode()\n"); fprintf(stderr, " jerasure_schedule_encode()\n");
fprintf(stderr, " jerasure_schedule_decode_lazy()\n"); fprintf(stderr, " jerasure_schedule_decode_lazy()\n");
fprintf(stderr, " jerasure_print_matrix()\n"); fprintf(stderr, " jerasure_print_matrix()\n");
fprintf(stderr, " jerasure_print_bitmatrix()\n");
fprintf(stderr, " jerasure_get_stats()\n"); fprintf(stderr, " jerasure_get_stats()\n");
if (s != NULL) fprintf(stderr, "%s\n", s); if (s != NULL) fprintf(stderr, "%s\n", s);
exit(1); exit(1);
} }
static void print_data_and_coding(int k, int m, int w, int psize, static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12; for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
printf("</tr>\n");
printf("%-*sCoding\n", sp, "Data"); printf("<td align=right><pre>");
for(i = 0; i < n; i++) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
for (j = 0; j < w; j++) { printf("</pre></td>\n");
if(i < k) { for (i = 0; i < ndevices; i++) {
printf("<td><pre>");
if(j==0) printf("D%-2d p%-2d:", i,j); up = (unsigned char *) ptrs[i];
else printf(" p%-2d:", j); for (j = 0; j < size/packetsize; j++) {
for(x = 0; x < psize; x +=4) { for (x = 0; x < packetsize; x++) {
memcpy(&l, data[i]+j*psize+x, sizeof(long)); if (x > 0 && x%4 == 0) printf(" ");
printf(" %08lx", l); printf("%02x", up[j*packetsize+x]);
} }
printf(" "); printf("\n");
} }
else printf("%*s", sp, ""); printf("</td>\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</tr></table></center>\n");
else printf(" p%-2d:", j);
for(x = 0; x < psize; x +=4) {
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
long l;
int k, w, i, j, m; int k, w, i, j, m;
int *matrix, *bitmatrix; int *matrix, *bitmatrix, **schedule;
char **data, **coding, **ptrs; char **data, **coding, **ptrs, **dcopy, **ccopy;
int **smart;
int no; int no;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double mstats[3], sstats[3];
uint32_t seed;
if (argc != 4) usage(NULL); if (argc != 5) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
matrix = cauchy_good_general_coding_matrix(k, m, w); matrix = cauchy_good_general_coding_matrix(k, m, w);
@ -132,42 +131,89 @@ int main(int argc, char **argv)
usage("couldn't make coding matrix"); usage("couldn't make coding matrix");
} }
/* Print out header information to the output file. */
printf("<HTML>\n");
printf("<TITLE>Jerasure Example Output: cauchy_04 %d %d %d %d</TITLE>\n", k, m, w, seed);
printf("<h2>Jerasure Example Output: cauchy_04 %d %d %d %d</h3>\n", k, m, w, seed);
printf("<hr>\n");
printf("Parameters:\n");
printf("<UL><LI> Number of data disks <i>(k)</i>: %d\n", k);
printf("<LI> Number of coding disks <i>(m)</i>: %d\n", m);
printf("<LI> Word size of the Galois Field: <i>(w)</i>: %d\n", w);
printf("<LI> Seed for the random number generator: %d\n", seed);
printf("<LI> Number of bytes stored per disk: %ld\n", sizeof(long)*w);
printf("<LI> Number of packets stored per disk: %d\n", w);
printf("<LI> Number of bytes per packet: %ld\n", sizeof(long));
printf("</UL>\n");
/* Print out the matrix and the bitmatrix */
printf("<hr>\n");
printf("Here is the matrix, which was created with <b>cauchy_xy_coding_matrix()</b>.\n");
printf("<pre>\n");
jerasure_print_matrix(matrix, m, k, w);
printf("</pre>\n");
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
no = 0; no = 0;
for (i = 0; i < k*m; i++) { for (i = 0; i < k*m; i++) {
no += cauchy_n_ones(matrix[i], w); no += cauchy_n_ones(matrix[i], w);
} }
printf("Matrix has %d ones\n\n", no);
jerasure_print_matrix(matrix, m, k, w);
printf("\n");
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); printf("The bitmatrix, which has %d one%s:<p><pre>\n", no, (no == 1) ? "" : "s");
jerasure_print_bitmatrix(bitmatrix, m*w, k*w, w);
printf("</pre>\n");
printf("<hr>\n");
MOA_Seed(seed);
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
dcopy = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { dcopy[i] = talloc(char, sizeof(long)*w);
l = lrand48(); MOA_Fill_Random_Region(data[i], sizeof(long)*w);
memcpy(data[i]+j*sizeof(long), &l, sizeof(long)); memcpy(dcopy[i], data[i], sizeof(long)*w);
}
printf("Here are the packets on the data disks:<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
coding = talloc(char *, m);
ccopy = talloc(char *, m);
for (i = 0; i < m; i++) {
coding[i] = talloc(char, sizeof(long)*w);
ccopy[i] = talloc(char, sizeof(long)*w);
}
jerasure_bitmatrix_encode(k, m, w, bitmatrix, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(mstats);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
jerasure_schedule_encode(k, m, w, schedule, data, ccopy, w*sizeof(long), sizeof(long));
jerasure_get_stats(sstats);
printf("<p>Encoding with jerasure_bitmatrix_encode() - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
printf("Encoding with jerasure_schedule_encode() - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
for (i = 0; i < m; i++) {
if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("Problem: the two encodings don't match on disk C%x\n", i);
exit(0);
} }
} }
coding = talloc(char *, m); printf("Here are the packets on the coding disks.<br>\n");
for (i = 0; i < m; i++) { print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
coding[i] = talloc(char, sizeof(long)*w); printf("<hr>\n");
}
jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats);
printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(31, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -175,15 +221,47 @@ int main(int argc, char **argv)
} }
} }
erasures[i] = -1; erasures[i] = -1;
printf("Erasures on the following devices:");
for (i = 0; erasures[i] != -1; i++) {
printf(" %c%x", ((erasures[i] < k) ? 'D' : 'C'), (erasures[i] < k ? erasures[i] : erasures[i]-k));
}
printf("<br>\nHere is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(mstats);
printf("<p>Decoded with jerasure_bitmatrix_decode - Bytes XOR'd: %.0lf.<br>\n", mstats[0]);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; erasures[i] != -1; i++) {
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
}
printf("Erased %d random pieces of data/coding:\n\n", m);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1); jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(stats); jerasure_get_stats(sstats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("jerasure_schedule_decode_lazy - Bytes XOR'd: %.0lf.<br>\n", sstats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)*w) != 0) {
return 0; printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)*w) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
printf("Here is the state of the system:\n<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
} }

12
Examples/decoder.c
View File

@ -90,7 +90,7 @@ int main (int argc, char **argv) {
/* Parameters */ /* Parameters */
int k, m, w, packetsize, buffersize; int k, m, w, packetsize, buffersize;
enum Coding_Technique tech; int tech;
char *c_tech; char *c_tech;
int i, j; // loop control variables int i, j; // loop control variables
@ -148,7 +148,7 @@ int main (int argc, char **argv) {
} else { } else {
extension = strdup(""); extension = strdup("");
} }
fname = (char *)malloc(sizeof(char*)*(100+strlen(argv[1])+10)); fname = (char *)malloc(sizeof(char*)*(100+strlen(argv[1])+20));
/* Read in parameters from metadata file */ /* Read in parameters from metadata file */
sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1); sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1);
@ -158,7 +158,7 @@ int main (int argc, char **argv) {
fprintf(stderr, "Error: no metadata file %s\n", fname); fprintf(stderr, "Error: no metadata file %s\n", fname);
exit(1); exit(1);
} }
temp = (char *)malloc(sizeof(char)*(strlen(argv[1])+10)); temp = (char *)malloc(sizeof(char)*(strlen(argv[1])+20));
fscanf(fp, "%s", temp); fscanf(fp, "%s", temp);
if (fscanf(fp, "%d", &origsize) != 1) { if (fscanf(fp, "%d", &origsize) != 1) {
@ -169,7 +169,7 @@ int main (int argc, char **argv) {
fprintf(stderr, "Parameters are not correct\n"); fprintf(stderr, "Parameters are not correct\n");
exit(0); exit(0);
} }
c_tech = (char *)malloc(sizeof(char)*(strlen(argv[1])+10)); c_tech = (char *)malloc(sizeof(char)*(strlen(argv[1])+20));
fscanf(fp, "%s", c_tech); fscanf(fp, "%s", c_tech);
fscanf(fp, "%d", &tech); fscanf(fp, "%d", &tech);
method = tech; method = tech;
@ -374,8 +374,8 @@ int main (int argc, char **argv) {
tsec /= 1000000.0; tsec /= 1000000.0;
tsec += t2.tv_sec; tsec += t2.tv_sec;
tsec -= t1.tv_sec; tsec -= t1.tv_sec;
printf("Decoding (MB/sec): %0.10f\n", (origsize/1024/1024)/totalsec); printf("Decoding (MB/sec): %0.10f\n", (((double) origsize)/1024.0/1024.0)/totalsec);
printf("De_Total (MB/sec): %0.10f\n\n", (origsize/1024/1024)/tsec); printf("De_Total (MB/sec): %0.10f\n\n", (((double) origsize)/1024.0/1024.0)/tsec);
} }
void ctrl_bs_handler(int dummy) { void ctrl_bs_handler(int dummy) {

49
Examples/encoder.c
View File

@ -56,9 +56,9 @@ is the file name with "_k#" or "_m#" and then the extension.
#include <stdlib.h> #include <stdlib.h>
#include <errno.h> #include <errno.h>
#include <signal.h> #include <signal.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
#include "galois.h"
#include "cauchy.h" #include "cauchy.h"
#include "liberation.h" #include "liberation.h"
@ -82,9 +82,7 @@ int jfread(void *ptr, int size, int nmembers, FILE *stream)
int *li, i; int *li, i;
if (stream != NULL) return fread(ptr, size, nmembers, stream); if (stream != NULL) return fread(ptr, size, nmembers, stream);
nd = size/sizeof(int); MOA_Fill_Random_Region(ptr, size);
li = (int *) ptr;
for (i = 0; i < nd; i++) li[i] = mrand48();
return size; return size;
} }
@ -143,8 +141,11 @@ int main (int argc, char **argv) {
/* Error check Arguments*/ /* Error check Arguments*/
if (argc != 8) { if (argc != 8) {
fprintf(stderr, "usage: inputfile k m coding_technique w (packetsize) (buffersize)\n"); fprintf(stderr, "usage: inputfile k m coding_technique w packetsize buffersize\n");
fprintf(stderr, "\nChoose one of the following coding techniques: \nreed_sol_van, \nreed_sol_r6_op, \ncauchy_orig, \ncauchy_good, \nliberation, \nblaum_roth, \nliber8tion"); fprintf(stderr, "\nChoose one of the following coding techniques: \nreed_sol_van, \nreed_sol_r6_op, \ncauchy_orig, \ncauchy_good, \nliberation, \nblaum_roth, \nliber8tion");
fprintf(stderr, "\n\nPacketsize is ignored for the reed_sol's");
fprintf(stderr, "\nBuffersize of 0 means the buffersize is chosen automatically.\n");
fprintf(stderr, "\nIf you just want to test speed, use an inputfile of \"-number\" where number is the size of the fake file you want to test.\n\n");
exit(0); exit(0);
} }
/* Conversion of parameters and error checking */ /* Conversion of parameters and error checking */
@ -182,16 +183,16 @@ int main (int argc, char **argv) {
/* Determine proper buffersize by finding the closest valid buffersize to the input value */ /* Determine proper buffersize by finding the closest valid buffersize to the input value */
if (buffersize != 0) { if (buffersize != 0) {
if (packetsize != 0 && buffersize%(sizeof(int)*w*k*packetsize) != 0) { if (packetsize != 0 && buffersize%(sizeof(long)*w*k*packetsize) != 0) {
up = buffersize; up = buffersize;
down = buffersize; down = buffersize;
while (up%(sizeof(int)*w*k*packetsize) != 0 && (down%(sizeof(int)*w*k*packetsize) != 0)) { while (up%(sizeof(long)*w*k*packetsize) != 0 && (down%(sizeof(long)*w*k*packetsize) != 0)) {
up++; up++;
if (down == 0) { if (down == 0) {
down--; down--;
} }
} }
if (up%(sizeof(int)*w*k*packetsize) == 0) { if (up%(sizeof(long)*w*k*packetsize) == 0) {
buffersize = up; buffersize = up;
} }
else { else {
@ -200,14 +201,14 @@ int main (int argc, char **argv) {
} }
} }
} }
else if (packetsize == 0 && buffersize%(sizeof(int)*w*k) != 0) { else if (packetsize == 0 && buffersize%(sizeof(long)*w*k) != 0) {
up = buffersize; up = buffersize;
down = buffersize; down = buffersize;
while (up%(sizeof(int)*w*k) != 0 && down%(sizeof(int)*w*k) != 0) { while (up%(sizeof(long)*w*k) != 0 && down%(sizeof(long)*w*k) != 0) {
up++; up++;
down--; down--;
} }
if (up%(sizeof(int)*w*k) == 0) { if (up%(sizeof(long)*w*k) == 0) {
buffersize = up; buffersize = up;
} }
else { else {
@ -266,8 +267,8 @@ int main (int argc, char **argv) {
fprintf(stderr, "Must include packetsize.\n"); fprintf(stderr, "Must include packetsize.\n");
exit(0); exit(0);
} }
if ((packetsize%(sizeof(int))) != 0) { if ((packetsize%(sizeof(long))) != 0) {
fprintf(stderr, "packetsize must be a multiple of sizeof(int)\n"); fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n");
exit(0); exit(0);
} }
tech = Liberation; tech = Liberation;
@ -285,8 +286,8 @@ int main (int argc, char **argv) {
fprintf(stderr, "Must include packetsize.\n"); fprintf(stderr, "Must include packetsize.\n");
exit(0); exit(0);
} }
if ((packetsize%(sizeof(int))) != 0) { if ((packetsize%(sizeof(long))) != 0) {
fprintf(stderr, "packetsize must be a multiple of sizeof(int)\n"); fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n");
exit(0); exit(0);
} }
tech = Blaum_Roth; tech = Blaum_Roth;
@ -347,21 +348,21 @@ int main (int argc, char **argv) {
exit(1); exit(1);
} }
fp = NULL; fp = NULL;
srand48(time(0)); MOA_Seed(time(0));
} }
newsize = size; newsize = size;
/* Find new size by determining next closest multiple */ /* Find new size by determining next closest multiple */
if (packetsize != 0) { if (packetsize != 0) {
if (size%(k*w*packetsize*sizeof(int)) != 0) { if (size%(k*w*packetsize*sizeof(long)) != 0) {
while (newsize%(k*w*packetsize*sizeof(int)) != 0) while (newsize%(k*w*packetsize*sizeof(long)) != 0)
newsize++; newsize++;
} }
} }
else { else {
if (size%(k*w*sizeof(int)) != 0) { if (size%(k*w*sizeof(long)) != 0) {
while (newsize%(k*w*sizeof(int)) != 0) while (newsize%(k*w*sizeof(long)) != 0)
newsize++; newsize++;
} }
} }
@ -466,6 +467,7 @@ int main (int argc, char **argv) {
tsec += t4.tv_sec; tsec += t4.tv_sec;
tsec -= t3.tv_sec; tsec -= t3.tv_sec;
totalsec += tsec; totalsec += tsec;
/* Read in data until finished */ /* Read in data until finished */
@ -490,13 +492,12 @@ int main (int argc, char **argv) {
} }
} }
/* Set pointers to point to file data */ /* Set pointers to point to file data */
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = block+(i*blocksize); data[i] = block+(i*blocksize);
} }
gettimeofday(&t3, &tz); gettimeofday(&t3, &tz);
/* Encode according to coding method */ /* Encode according to coding method */
switch(tech) { switch(tech) {
case No_Coding: case No_Coding:
@ -596,8 +597,8 @@ int main (int argc, char **argv) {
tsec /= 1000000.0; tsec /= 1000000.0;
tsec += t2.tv_sec; tsec += t2.tv_sec;
tsec -= t1.tv_sec; tsec -= t1.tv_sec;
printf("Encoding (MB/sec): %0.10f\n", (size/1024/1024)/totalsec); printf("Encoding (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/totalsec);
printf("En_Total (MB/sec): %0.10f\n", (size/1024/1024)/tsec); printf("En_Total (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/tsec);
} }
/* is_prime returns 1 if number if prime, 0 if not prime */ /* is_prime returns 1 if number if prime, 0 if not prime */

8
Examples/jerasure_01.c
View File

@ -73,6 +73,14 @@ int main(int argc, char **argv)
n = galois_single_multiply(n, 2, w); n = galois_single_multiply(n, 2, w);
} }
printf("<HTML><TITLE>jerasure_01");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_01");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
jerasure_print_matrix(matrix, r, c, w); jerasure_print_matrix(matrix, r, c, w);
return 0; return 0;
} }

9
Examples/jerasure_02.c
View File

@ -73,6 +73,15 @@ int main(int argc, char **argv)
} }
bitmatrix = jerasure_matrix_to_bitmatrix(c, r, w, matrix); bitmatrix = jerasure_matrix_to_bitmatrix(c, r, w, matrix);
printf("<HTML><TITLE>jerasure_02");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_02");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
jerasure_print_bitmatrix(bitmatrix, r*w, c*w, w); jerasure_print_bitmatrix(bitmatrix, r*w, c*w, w);
return 0; return 0;
} }

10
Examples/jerasure_03.c
View File

@ -50,7 +50,7 @@ usage(char *s)
{ {
fprintf(stderr, "usage: jerasure_03 k w - Creates a kxk Cauchy matrix in GF(2^w). \n\n"); fprintf(stderr, "usage: jerasure_03 k w - Creates a kxk Cauchy matrix in GF(2^w). \n\n");
fprintf(stderr, " k must be < 2^w. Element i,j is 1/(i+(2^w-j-1)). (If that is\n"); fprintf(stderr, " k must be < 2^w. Element i,j is 1/(i+(2^w-j-1)). (If that is\n");
fprintf(stderr, " 1/0, then it sets it to zero). \n"); fprintf(stderr, " If that is 1/0, then it sets it to zero). \n");
fprintf(stderr, " It then tests whether that matrix is invertible.\n"); fprintf(stderr, " It then tests whether that matrix is invertible.\n");
fprintf(stderr, " If it is invertible, then it prints out the inverse.\n"); fprintf(stderr, " If it is invertible, then it prints out the inverse.\n");
fprintf(stderr, " Finally, it prints the product of the matrix and its inverse.\n"); fprintf(stderr, " Finally, it prints the product of the matrix and its inverse.\n");
@ -87,6 +87,14 @@ int main(int argc, char **argv)
} }
} }
printf("<HTML><TITLE>jerasure_03");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_03");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
printf("The Cauchy Matrix:\n"); printf("The Cauchy Matrix:\n");
jerasure_print_matrix(matrix, k, k, w); jerasure_print_matrix(matrix, k, k, w);
memcpy(matrix_copy, matrix, sizeof(int)*k*k); memcpy(matrix_copy, matrix, sizeof(int)*k*k);

8
Examples/jerasure_04.c
View File

@ -85,6 +85,14 @@ int main(int argc, char **argv)
} }
bitmatrix = jerasure_matrix_to_bitmatrix(k, k, w, matrix); bitmatrix = jerasure_matrix_to_bitmatrix(k, k, w, matrix);
printf("<HTML><TITLE>jerasure_04");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_04");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
printf("The Cauchy Bit-Matrix:\n"); printf("The Cauchy Bit-Matrix:\n");
jerasure_print_bitmatrix(bitmatrix, k*w, k*w, w); jerasure_print_bitmatrix(bitmatrix, k*w, k*w, w);
memcpy(bitmatrix_copy, bitmatrix, sizeof(int)*k*w*k*w); memcpy(bitmatrix_copy, bitmatrix, sizeof(int)*k*w*k*w);

34
Examples/jerasure_05.c
View File

@ -43,20 +43,22 @@
revised by S. Simmerman revised by S. Simmerman
2/25/08 2/25/08
*/ */
#include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) #define talloc(type, num) (type *) malloc(sizeof(type)*(num))
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: jerasure_05 k m w size - Does a simple Reed-Solomon coding example in GF(2^w).\n"); fprintf(stderr, "usage: jerasure_05 k m w size seed - Does a simple Reed-Solomon coding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. w can be 8, 16 or 32.\n"); fprintf(stderr, " k+m must be <= 2^w. w can be 8, 16 or 32.\n");
fprintf(stderr, " It sets up a Cauchy distribution matrix and encodes\n"); fprintf(stderr, " It sets up a Cauchy generator matrix and encodes\n");
fprintf(stderr, " k devices of size bytes with it. Then it decodes.\n", sizeof(long)); fprintf(stderr, " k devices of size bytes with it. Then it decodes.\n");
fprintf(stderr, " After that, it decodes device 0 by using jerasure_make_decoding_matrix()\n"); fprintf(stderr, " After that, it decodes device 0 by using jerasure_make_decoding_matrix()\n");
fprintf(stderr, " and jerasure_matrix_dotprod().\n"); fprintf(stderr, " and jerasure_matrix_dotprod().\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
@ -116,15 +118,16 @@ int main(int argc, char **argv)
char **data, **coding; char **data, **coding;
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
uint32_t seed;
if (argc != 5) usage(NULL); if (argc != 6) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
usage("Bad w");
if (w < 32 && k + m > (1 << w)) usage("k + m must be <= 2 ^ w"); if (w < 32 && k + m > (1 << w)) usage("k + m must be <= 2 ^ w");
if (sscanf(argv[4], "%d", &size) == 0 || size % sizeof(long) != 0) if (sscanf(argv[4], "%d", &size) == 0 || size % sizeof(long) != 0)
usage("size must be multiple of sizeof(long)"); usage("size must be multiple of sizeof(long)");
if (sscanf(argv[5], "%d", &seed) == 0) usage("Bad seed");
matrix = talloc(int, m*k); matrix = talloc(int, m*k);
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
@ -133,18 +136,23 @@ int main(int argc, char **argv)
} }
} }
printf("The Coding Matrix (the last m rows of the Distribution Matrix):\n\n"); printf("<HTML><TITLE>jerasure_05");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_05");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
printf("The Coding Matrix (the last m rows of the Generator Matrix G^T):\n\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n"); printf("\n");
srand48(0); MOA_Seed(seed);
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, size); data[i] = talloc(char, size);
for(j = 0; j < size; j+=sizeof(long)) { MOA_Fill_Random_Region(data[i], size);
l = lrand48();
memcpy(data[i] + j, &l, sizeof(long));
}
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -162,7 +170,7 @@ int main(int argc, char **argv)
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0; l = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = (MOA_Random_W(w, 1))%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;

165
Examples/jerasure_06.c
View File

@ -44,18 +44,20 @@
*/ */
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) #define talloc(type, num) (type *) malloc(sizeof(type)*(num))
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: jerasure_06 k m w packetsize\n"); fprintf(stderr, "usage: jerasure_06 k m w packetsize seed\n");
fprintf(stderr, "Does a simple Cauchy Reed-Solomon coding example in GF(2^w).\n"); fprintf(stderr, "Does a simple Cauchy Reed-Solomon coding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be < 2^w. Packetsize must be a multiple of sizeof(long)\n"); fprintf(stderr, " k+m must be < 2^w. Packetsize must be a multiple of sizeof(long)\n");
fprintf(stderr, " It sets up a Cauchy distribution matrix and encodes k devices of w*packetsize bytes.\n"); fprintf(stderr, " It sets up a Cauchy generator matrix and encodes k devices of w*packetsize bytes.\n");
fprintf(stderr, " After that, it decodes device 0 by using jerasure_make_decoding_bitmatrix()\n"); fprintf(stderr, " After that, it decodes device 0 by using jerasure_make_decoding_bitmatrix()\n");
fprintf(stderr, " and jerasure_bitmatrix_dotprod().\n"); fprintf(stderr, " and jerasure_bitmatrix_dotprod().\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
@ -68,43 +70,31 @@ usage(char *s)
exit(1); exit(1);
} }
static void print_data_and_coding(int k, int m, int w, int psize, static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12;
printf("%-*sCoding\n", sp, "Data"); for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
for(i = 0; i < n; i++) { printf("</tr>\n");
for (j = 0; j < w; j++) { printf("<td align=right><pre>");
if(i < k) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
printf("</pre></td>\n");
if(j==0) printf("D%-2d p%-2d:", i,j); for (i = 0; i < ndevices; i++) {
else printf(" p%-2d:", j); printf("<td><pre>");
for(x = 0; x < psize; x +=4) { up = (unsigned char *) ptrs[i];
memcpy(&l, data[i]+j*psize+x, sizeof(long)); for (j = 0; j < size/packetsize; j++) {
printf(" %08lx", l); for (x = 0; x < packetsize; x++) {
} if (x > 0 && x%4 == 0) printf(" ");
printf(" "); printf("%02x", up[j*packetsize+x]);
} }
else printf("%*s", sp, ""); printf("\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</td>\n");
else printf(" p%-2d:", j); }
for(x = 0; x < psize; x +=4) { printf("</tr></table></center>\n");
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
@ -115,15 +105,18 @@ int main(int argc, char **argv)
char **data, **coding; char **data, **coding;
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
uint32_t seed;
if (argc != 5) usage(NULL); if (argc != 6) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
if (sscanf(argv[4], "%d", &psize) == 0 || psize <= 0) usage("Bad packetsize"); if (sscanf(argv[4], "%d", &psize) == 0 || psize <= 0) usage("Bad packetsize");
if(psize%sizeof(long) != 0) usage("Packetsize must be multiple of sizeof(long)"); if(psize%sizeof(long) != 0) usage("Packetsize must be multiple of sizeof(long)");
if (sscanf(argv[5], "%d", &seed) == 0) usage("Bad seed");
MOA_Seed(seed);
matrix = talloc(int, m*k); matrix = talloc(int, m*k);
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
for (j = 0; j < k; j++) { for (j = 0; j < k; j++) {
@ -132,21 +125,22 @@ int main(int argc, char **argv)
} }
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
printf("Last (m * w) rows of the Binary Distribution Matrix:\n\n"); printf("<HTML><TITLE>jerasure_06");
jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w); for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("\n"); printf("</TITLE>\n");
printf("<h3>jerasure_06");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<hr>\n");
printf("Last (m * w) rows of the Generator Matrix: (G^T):\n<pre>\n");
jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w);
printf("</pre><hr>\n");
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, psize*w); data[i] = talloc(char, psize*w);
for (j = 0; j < w; j++) { MOA_Fill_Random_Region(data[i], psize*w);
for(x = 0; x < psize; x += 4) {
l = lrand48();
memcpy(data[i]+j*psize+x, &l, sizeof(long));
}
}
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -156,14 +150,18 @@ int main(int argc, char **argv)
jerasure_bitmatrix_encode(k, m, w, bitmatrix, data, coding, w*psize, psize); jerasure_bitmatrix_encode(k, m, w, bitmatrix, data, coding, w*psize, psize);
printf("Encoding Complete:\n\n"); printf("Encoding Complete - Here is the state of the system\n\n");
print_data_and_coding(k, m, w, psize, data, coding); printf("<p>\n");
print_array(data, k, psize*w, psize, "D");
printf("<p>\n");
print_array(coding, m, psize*w, psize, "C");
printf("<hr>\n");
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(w, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], psize*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], psize*w);
@ -172,42 +170,63 @@ int main(int argc, char **argv)
} }
erasures[i] = -1; erasures[i] = -1;
printf("Erased %d random devices:\n\n", m); printf("Erased %d random devices:", m);
print_data_and_coding(k, m, w, psize, data, coding); for (i = 0; erasures[i] != -1; i++) {
printf(" %c%x", ((erasures[i] < k) ? 'D' : 'C'), (erasures[i] < k ? erasures[i] : erasures[i]-k));
}
printf(". Here is the state of the system:\n");
printf("<p>\n");
print_array(data, k, psize*w, psize, "D");
printf("<p>\n");
print_array(coding, m, psize*w, psize, "C");
printf("<hr>\n");
i = jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data, coding, i = jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data, coding,
w*psize, psize); w*psize, psize);
printf("State of the system after decoding:\n\n"); printf("Here is the state of the system after decoding:\n\n");
print_data_and_coding(k, m, w, psize, data, coding); printf("<p>\n");
print_array(data, k, psize*w, psize, "D");
printf("<p>\n");
print_array(coding, m, psize*w, psize, "C");
printf("<hr>\n");
decoding_matrix = talloc(int, k*k*w*w); decoding_matrix = talloc(int, k*k*w*w);
dm_ids = talloc(int, k); dm_ids = talloc(int, k);
for (i = 0; i < m; i++) erased[i] = 1; x = (m < k) ? m : k;
for (i = 0; i < x; i++) erased[i] = 1;
for (; i < k+m; i++) erased[i] = 0; for (; i < k+m; i++) erased[i] = 0;
jerasure_make_decoding_bitmatrix(k, m, w, bitmatrix, erased, decoding_matrix, dm_ids); jerasure_make_decoding_bitmatrix(k, m, w, bitmatrix, erased, decoding_matrix, dm_ids);
printf("Suppose we erase the first %d devices. Here is the decoding matrix:\n\n", m); printf("Suppose we erase the first %d devices. Here is the decoding matrix:\n<pre>\n", x);
jerasure_print_bitmatrix(decoding_matrix, k*w, k*w, w); jerasure_print_bitmatrix(decoding_matrix, k*w, k*w, w);
printf("\n"); printf("</pre>\n");
printf("And dm_ids:\n\n"); printf("And dm_ids:\n<pre>\n");
jerasure_print_matrix(dm_ids, 1, k, w); jerasure_print_matrix(dm_ids, 1, k, w);
printf("</pre><hr>\n");
//memcpy(&l, data[0], sizeof(long)); for (i = 0; i < x; i++) bzero(data[i], w*psize);
//printf("\nThe value of device #%d, word 0 is: %lx\n", 0, l);
bzero(data[0], w*psize);
jerasure_bitmatrix_dotprod(k, w, decoding_matrix, dm_ids, 0, data, coding, w*psize, psize);
printf("\nAfter calling jerasure_matrix_dotprod, we calculate the value of device #0, packet 0 to be:\n"); printf("Here is the state of the system after the erasures:\n\n");
printf("\nD0 p0 :"); printf("<p>\n");
for(i = 0; i < psize; i +=sizeof(long)) { print_array(data, k, psize*w, psize, "D");
memcpy(&l, data[0]+i, sizeof(long)); printf("<p>\n");
printf(" %08lx", l); print_array(coding, m, psize*w, psize, "C");
} printf("<hr>\n");
printf("\n\n");
//memcpy(&l, data[0], sizeof(long));
for (i = 0; i < x; i++) {
jerasure_bitmatrix_dotprod(k, w, decoding_matrix+i*(k*w*w), dm_ids, i, data, coding, w*psize, psize);
}
printf("Here is the state of the system after calling <b>jerasure_bitmatrix_dotprod()</b> %d time%s with the decoding matrix:\n\n", x, (x == 1) ? "" : "s");
printf("<p>\n");
print_array(data, k, psize*w, psize, "D");
printf("<p>\n");
print_array(coding, m, psize*w, psize, "C");
printf("<hr>\n");
return 0; return 0;
} }

135
Examples/jerasure_07.c
View File

@ -46,16 +46,18 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <stdint.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) #define talloc(type, num) (type *) malloc(sizeof(type)*(num))
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: jerasure_07 k m w - Scheduled Cauchy Reed-Solomon coding example in GF(2^w).\n"); fprintf(stderr, "usage: jerasure_07 k m w seed - Scheduled Cauchy Reed-Solomon coding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. It sets up a Cauchy distribution matrix and encodes\n"); fprintf(stderr, " k+m must be <= 2^w. It sets up a Cauchy generator matrix and encodes\n");
fprintf(stderr, " k sets of w*%d bytes. It uses bit-matrix scheduling, both smart and dumb.\n", sizeof(long)); fprintf(stderr, " k sets of w*%ld bytes. It uses bit-matrix scheduling, both smart and dumb.\n", sizeof(long));
fprintf(stderr, " It decodes using bit-matrix scheduling, then shows an example of\n"); fprintf(stderr, " It decodes using bit-matrix scheduling, then shows an example of\n");
fprintf(stderr, " using jerasure_do_scheduled_operations().\n"); fprintf(stderr, " using jerasure_do_scheduled_operations().\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
@ -69,43 +71,31 @@ usage(char *s)
exit(1); exit(1);
} }
static void print_data_and_coding(int k, int m, int w, int psize, static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12;
printf("%-*sCoding\n", sp, "Data"); for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
for(i = 0; i < n; i++) { printf("</tr>\n");
for (j = 0; j < w; j++) { printf("<td align=right><pre>");
if(i < k) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
printf("</pre></td>\n");
if(j==0) printf("D%-2d p%-2d:", i,j); for (i = 0; i < ndevices; i++) {
else printf(" p%-2d:", j); printf("<td><pre>");
for(x = 0; x < psize; x +=4) { up = (unsigned char *) ptrs[i];
memcpy(&l, data[i]+j*psize+x, sizeof(long)); for (j = 0; j < size/packetsize; j++) {
printf(" %08lx", l); for (x = 0; x < packetsize; x++) {
} if (x > 0 && x%4 == 0) printf(" ");
printf(" "); printf("%02x", up[j*packetsize+x]);
} }
else printf("%*s", sp, ""); printf("\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</td>\n");
else printf(" p%-2d:", j); }
for(x = 0; x < psize; x +=4) { printf("</tr></table></center>\n");
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
@ -117,11 +107,13 @@ int main(int argc, char **argv)
int **smart, **dumb; int **smart, **dumb;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double stats[3];
uint32_t seed;
if (argc != 4) usage(NULL); if (argc != 5) usage("Wrong number of arguments");
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad m"); if (sscanf(argv[3], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
matrix = talloc(int, m*k); matrix = talloc(int, m*k);
@ -132,21 +124,26 @@ int main(int argc, char **argv)
} }
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
printf("Last m rows of the Binary Distribution Matrix:\n\n"); printf("<HTML><TITLE>jerasure_07");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_07");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<hr>\n");
printf("Last m*w rows of the generator matrix (G^T):\n<pre>\n");
jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w); jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w);
printf("\n"); printf("</pre><hr>\n");
dumb = jerasure_dumb_bitmatrix_to_schedule(k, m, w, bitmatrix); dumb = jerasure_dumb_bitmatrix_to_schedule(k, m, w, bitmatrix);
smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
srand48(0); MOA_Seed(seed);
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { MOA_Fill_Random_Region(data[i], sizeof(long)*w);
l = lrand48();
memcpy(data[i]+j*sizeof(long), &l, sizeof(long));
}
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -156,19 +153,27 @@ int main(int argc, char **argv)
jerasure_schedule_encode(k, m, w, dumb, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_encode(k, m, w, dumb, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("Dumb Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]); printf("Dumb Encoding Complete: - %.0lf XOR'd bytes. State of the system:\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]); printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(w, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -178,23 +183,41 @@ int main(int argc, char **argv)
erasures[i] = -1; erasures[i] = -1;
printf("Erased %d random devices:\n\n", m); printf("Erased %d random devices:\n\n", m);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1); jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
ptrs = talloc(char *, (k+m)); ptrs = talloc(char *, (k+m));
for (i = 0; i < k; i++) ptrs[i] = data[i]; for (i = 0; i < k; i++) ptrs[i] = data[i];
for (i = 0; i < m; i++) ptrs[k+i] = coding[i]; for (i = 0; i < m; i++) ptrs[k+i] = coding[i];
for (j = 0; j < m; j++) bzero(coding[j], sizeof(long)*w); for (j = 0; j < m; j++) bzero(coding[j], sizeof(long)*w);
printf("State of the system after erasing the coding devices:\n");
printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_do_scheduled_operations(ptrs, smart, sizeof(long)); jerasure_do_scheduled_operations(ptrs, smart, sizeof(long));
printf("State of the system after deleting the coding devices and\n"); printf("And using <b>jerasure_do_scheduled_operations()</b>: %.0lf XOR'd bytes\n\n", stats[0]);
printf("using jerasure_do_scheduled_operations(): %.0lf XOR'd bytes\n\n", stats[0]); printf("<p>\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
return 0; return 0;
} }

140
Examples/jerasure_08.c
View File

@ -38,24 +38,21 @@
*/ */
/*
revised by S. Simmerman
2/25/08
*/
#include <stdio.h> #include <stdio.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#define talloc(type, num) (type *) malloc(sizeof(type)*(num)) #define talloc(type, num) (type *) malloc(sizeof(type)*(num))
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: jerasure_08 k w - Example schedule cache usage with RAID-6\n"); fprintf(stderr, "usage: jerasure_08 k w seed - Example schedule cache usage with RAID-6\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " m=2. k+m must be <= 2^w. It sets up a RAID-6 distribution matrix and encodes\n"); fprintf(stderr, " m=2. k+m must be <= 2^w. It sets up a RAID-6 generator matrix and encodes\n");
fprintf(stderr, " k sets of w*%d bytes. It creates a schedule cache for decoding.\n", sizeof(long)); fprintf(stderr, " k sets of w*%ld bytes. It creates a schedule cache for decoding.\n", sizeof(long));
fprintf(stderr, " It demonstrates using the schedule cache for both encoding and decoding.\n"); fprintf(stderr, " It demonstrates using the schedule cache for both encoding and decoding.\n");
fprintf(stderr, " Then it demonstrates using jerasure_do_parity() to re-encode the first.\n"); fprintf(stderr, " Then it demonstrates using jerasure_do_parity() to re-encode the first.\n");
fprintf(stderr, " coding device\n"); fprintf(stderr, " coding device\n");
@ -72,44 +69,31 @@ usage(char *s)
exit(1); exit(1);
} }
static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
static void print_data_and_coding(int k, int m, int w, int psize,
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12;
printf("%-*sCoding\n", sp, "Data"); for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
for(i = 0; i < n; i++) { printf("</tr>\n");
for (j = 0; j < w; j++) { printf("<td align=right><pre>");
if(i < k) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
printf("</pre></td>\n");
if(j==0) printf("D%-2d p%-2d:", i,j); for (i = 0; i < ndevices; i++) {
else printf(" p%-2d:", j); printf("<td><pre>");
for(x = 0; x < psize; x +=4) { up = (unsigned char *) ptrs[i];
memcpy(&l, data[i]+j*psize+x, sizeof(long)); for (j = 0; j < size/packetsize; j++) {
printf(" %08lx", l); for (x = 0; x < packetsize; x++) {
} if (x > 0 && x%4 == 0) printf(" ");
printf(" "); printf("%02x", up[j*packetsize+x]);
} }
else printf("%*s", sp, ""); printf("\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</td>\n");
else printf(" p%-2d:", j); }
for(x = 0; x < psize; x +=4) { printf("</tr></table></center>\n");
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
@ -121,13 +105,17 @@ int main(int argc, char **argv)
int **smart, ***cache; int **smart, ***cache;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double stats[3];
uint32_t seed;
if (argc != 3) usage(NULL); if (argc != 4) usage("Wrong number of arguments");
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad m"); if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad m");
if (sscanf(argv[3], "%d", &seed) == 0) usage("Bad seed");
m = 2; m = 2;
if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big"); if (w < 30 && (k+m) > (1 << w)) usage("k + m is too big");
MOA_Seed(seed);
matrix = talloc(int, m*k); matrix = talloc(int, m*k);
for (j = 0; j < k; j++) matrix[j] = 1; for (j = 0; j < k; j++) matrix[j] = 1;
i = 1; i = 1;
@ -140,14 +128,10 @@ int main(int argc, char **argv)
smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
cache = jerasure_generate_schedule_cache(k, m, w, bitmatrix, 1); cache = jerasure_generate_schedule_cache(k, m, w, bitmatrix, 1);
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { MOA_Fill_Random_Region(data[i], sizeof(long)*w);
l = lrand48();
memcpy(data[i]+j*sizeof(long), &l, sizeof(long));
}
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -157,8 +141,21 @@ int main(int argc, char **argv)
jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_encode(k, m, w, smart, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<HTML><TITLE>jerasure_08");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>jerasure_08");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<hr>\n");
printf("Encoding Complete: - %.0lf XOR'd bytes. Here is the state of the system:\n<p>\n", stats[0]);
printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erasures[0] = k; erasures[0] = k;
@ -169,12 +166,16 @@ int main(int argc, char **argv)
jerasure_schedule_decode_cache(k, m, w, cache, erasures, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_decode_cache(k, m, w, cache, erasures, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("Encoding Using the Schedule Cache: - %.0lf XOR'd bytes\n\n", stats[0]); printf("Encoding Using the Schedule Cache: - %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(w, 1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -184,24 +185,43 @@ int main(int argc, char **argv)
erasures[i] = -1; erasures[i] = -1;
printf("Erased %d random devices:\n\n", m); printf("Erased %d random devices:\n\n", m);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_schedule_decode_cache(k, m, w, cache, erasures, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_decode_cache(k, m, w, cache, erasures, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
bzero(coding[0], sizeof(long)*w); bzero(coding[0], sizeof(long)*w);
printf("Erased the first coding device:\n\n", m);
printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_do_parity(k, data, coding[0], sizeof(long)*w); jerasure_do_parity(k, data, coding[0], sizeof(long)*w);
printf("State of the system after deleting coding device 0 and using\n"); printf("State of the system after using\n");
printf("jerasure_do_parity to re-encode it:\n\n"); printf("<b>jerasure_do_parity()</b> to re-encode it:\n\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_free_schedule(smart); jerasure_free_schedule(smart);
jerasure_free_schedule_cache(k, m, cache); jerasure_free_schedule_cache(k, m, cache);
printf("Smart schedule and cache freed\n\n"); printf("Smart schedule and cache freed.\n\n");
return 0; return 0;
} }

109
Examples/liberation_01.c
View File

@ -46,6 +46,8 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <stdlib.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "liberation.h" #include "liberation.h"
@ -53,10 +55,10 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: liberation_01 k w - Liberation RAID-6 coding/decoding example in GF(2^w).\n"); fprintf(stderr, "usage: liberation_01 k w seed - Liberation RAID-6 coding/decoding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be prime and k <= w. It sets up a Liberation bit-matrix\n"); fprintf(stderr, " w must be prime and k <= w. It sets up a Liberation bit-matrix\n");
fprintf(stderr, " then it encodes k devices of w*%d bytes using dumb bit-matrix scheduling.\n", sizeof(long)); fprintf(stderr, " then it encodes k devices of w*%ld bytes using dumb bit-matrix scheduling.\n", sizeof(long));
fprintf(stderr, " It decodes using smart bit-matrix scheduling.\n"); fprintf(stderr, " It decodes using smart bit-matrix scheduling.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, "This demonstrates: liberation_coding_bitmatrix()\n"); fprintf(stderr, "This demonstrates: liberation_coding_bitmatrix()\n");
@ -70,44 +72,31 @@ usage(char *s)
exit(1); exit(1);
} }
static print_array(char **ptrs, int ndevices, int size, int packetsize, char *label)
static void print_data_and_coding(int k, int m, int w, int psize,
char **data, char **coding)
{ {
int i, j, x, n, sp; int i, j, x;
long l; unsigned char *up;
if(k > m) n = k; printf("<center><table border=3 cellpadding=3><tr><td></td>\n");
else n = m;
sp = psize * 2 + (psize/4) + 12;
printf("%-*sCoding\n", sp, "Data"); for (i = 0; i < ndevices; i++) printf("<td align=center>%s%x</td>\n", label, i);
for(i = 0; i < n; i++) { printf("</tr>\n");
for (j = 0; j < w; j++) { printf("<td align=right><pre>");
if(i < k) { for (j = 0; j < size/packetsize; j++) printf("Packet %d\n", j);
printf("</pre></td>\n");
if(j==0) printf("D%-2d p%-2d:", i,j); for (i = 0; i < ndevices; i++) {
else printf(" p%-2d:", j); printf("<td><pre>");
for(x = 0; x < psize; x +=4) { up = (unsigned char *) ptrs[i];
memcpy(&l, data[i]+j*psize+x, sizeof(long)); for (j = 0; j < size/packetsize; j++) {
printf(" %08lx", l); for (x = 0; x < packetsize; x++) {
} if (x > 0 && x%4 == 0) printf(" ");
printf(" "); printf("%02x", up[j*packetsize+x]);
} }
else printf("%*s", sp, ""); printf("\n");
if(i < m) { }
if(j==0) printf("C%-2d p%-2d:", i,j); printf("</td>\n");
else printf(" p%-2d:", j); }
for(x = 0; x < psize; x +=4) { printf("</tr></table></center>\n");
memcpy(&l, coding[i]+j*psize+x, sizeof(long));
printf(" %08lx", l);
}
}
printf("\n");
}
}
printf("\n");
} }
int main(int argc, char **argv) int main(int argc, char **argv)
@ -119,32 +108,40 @@ int main(int argc, char **argv)
int **dumb; int **dumb;
int *erasures, *erased; int *erasures, *erased;
double stats[3]; double stats[3];
uint32_t seed;
if (argc != 3) usage(NULL); if (argc != 4) usage("Wrong number of arguments");
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (sscanf(argv[3], "%u", &seed) == 0) usage("Bad seed");
m = 2; m = 2;
if (w < k) usage("k is too big"); if (w < k) usage("k is too big");
for (i = 2; i*i <= w; i++) if (w%i == 0) usage("w isn't prime");
bitmatrix = liberation_coding_bitmatrix(k, w); bitmatrix = liberation_coding_bitmatrix(k, w);
if (bitmatrix == NULL) { if (bitmatrix == NULL) {
usage("couldn't make coding matrix"); usage("couldn't make coding matrix");
} }
printf("Coding Bit-Matrix:\n\n"); printf("<HTML><TITLE>liberation_01");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>liberation_01");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<hr>\n");
printf("Coding Bit-Matrix:\n<pre>\n");
jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w); jerasure_print_bitmatrix(bitmatrix, w*m, w*k, w);
printf("\n"); printf("</pre><hr>\n");
dumb = jerasure_dumb_bitmatrix_to_schedule(k, m, w, bitmatrix); dumb = jerasure_dumb_bitmatrix_to_schedule(k, m, w, bitmatrix);
srand48(0); MOA_Seed(seed);
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)*w); data[i] = talloc(char, sizeof(long)*w);
for (j = 0; j < w; j++) { MOA_Fill_Random_Region(data[i], sizeof(long)*w);
l = lrand48();
memcpy(data[i]+j*sizeof(long), &l, sizeof(long));
}
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -154,14 +151,18 @@ int main(int argc, char **argv)
jerasure_schedule_encode(k, m, w, dumb, data, coding, w*sizeof(long), sizeof(long)); jerasure_schedule_encode(k, m, w, dumb, data, coding, w*sizeof(long), sizeof(long));
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("Smart Encoding Complete: - %.0lf XOR'd bytes\n\n", stats[0]); printf("Smart Encoding Complete: - %.0lf XOR'd bytes. State of the system:\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(30,1)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w); bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], sizeof(long)*w);
@ -171,13 +172,21 @@ int main(int argc, char **argv)
erasures[i] = -1; erasures[i] = -1;
printf("Erased %d random devices:\n\n", m); printf("Erased %d random devices:\n\n", m);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1); jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, w*sizeof(long), sizeof(long), 1);
jerasure_get_stats(stats); jerasure_get_stats(stats);
printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]); printf("State of the system after decoding: %.0lf XOR'd bytes\n\n", stats[0]);
print_data_and_coding(k, m, w, sizeof(long), data, coding); printf("<p>\n");
print_array(data, k, sizeof(long)*w, sizeof(long), "D");
printf("<p>\n");
print_array(coding, m, sizeof(long)*w, sizeof(long), "C");
printf("<hr>\n");
return 0; return 0;
} }

43
Examples/reed_sol_01.c
View File

@ -42,7 +42,9 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
@ -50,11 +52,11 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: reed_sol_01 k m w - Does a simple Reed-Solomon coding example in GF(2^w).\n"); fprintf(stderr, "usage: reed_sol_01 k m w seed - Does a simple Reed-Solomon coding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w. It sets up a classic\n"); fprintf(stderr, "w must be 8, 16 or 32. k+m must be <= 2^w. It sets up a classic\n");
fprintf(stderr, " Vandermonde-based generator matrix and encodes k devices of\n"); fprintf(stderr, "Vandermonde-based generator matrix and encodes k devices of\n");
fprintf(stderr, " %ld bytes each with it. Then it decodes.\n", sizeof(long)); fprintf(stderr, "%ld bytes each with it. Then it decodes.\n", sizeof(long));
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, "This demonstrates: jerasure_matrix_encode()\n"); fprintf(stderr, "This demonstrates: jerasure_matrix_encode()\n");
fprintf(stderr, " jerasure_matrix_decode()\n"); fprintf(stderr, " jerasure_matrix_decode()\n");
@ -107,39 +109,53 @@ int main(int argc, char **argv)
long l; long l;
int k, w, i, j, m; int k, w, i, j, m;
int *matrix; int *matrix;
char **data, **coding; char **data, **coding, **dcopy, **ccopy;
unsigned char uc; unsigned char uc;
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
uint32_t seed;
if (argc != 4) usage(NULL); if (argc != 5) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[4], "%u", &seed) == 0) usage("Bad seed");
if (w <= 16 && k + m > (1 << w)) usage("k + m is too big"); if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
matrix = reed_sol_vandermonde_coding_matrix(k, m, w); matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
printf("Last m rows of the Distribution Matrix:\n\n"); printf("<HTML><TITLE>reed_sol_01 %d %d %d %d</title>\n", k, m, w, seed);
printf("<h3>reed_sol_01 %d %d %d %d</h3>\n", k, m, w, seed);
printf("<pre>\n");
printf("Last m rows of the generator Matrix (G^T):\n\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n"); printf("\n");
srand48(0); MOA_Seed(seed);
data = talloc(char *, k); data = talloc(char *, k);
dcopy = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)); data[i] = talloc(char, sizeof(long));
dcopy[i] = talloc(char, sizeof(long));
for (j = 0; j < sizeof(long); j++) { for (j = 0; j < sizeof(long); j++) {
uc = lrand48()%256; uc = MOA_Random_W(8, 1);
data[i][j] = (char) uc; data[i][j] = (char) uc;
} }
memcpy(dcopy[i], data[i], sizeof(long));
} }
coding = talloc(char *, m); coding = talloc(char *, m);
ccopy = talloc(char *, m);
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
coding[i] = talloc(char, sizeof(long)); coding[i] = talloc(char, sizeof(long));
ccopy[i] = talloc(char, sizeof(long));
} }
jerasure_matrix_encode(k, m, w, matrix, data, coding, sizeof(long)); jerasure_matrix_encode(k, m, w, matrix, data, coding, sizeof(long));
for (i = 0; i < m; i++) {
memcpy(ccopy[i], coding[i], sizeof(long));
}
printf("Encoding Complete:\n\n"); printf("Encoding Complete:\n\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); print_data_and_coding(k, m, w, sizeof(long), data, coding);
@ -149,7 +165,7 @@ int main(int argc, char **argv)
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0; l = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = MOA_Random_W(31, 0)%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
memcpy((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], &l, sizeof(long)); memcpy((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], &l, sizeof(long));
@ -166,5 +182,12 @@ int main(int argc, char **argv)
printf("State of the system after decoding:\n\n"); printf("State of the system after decoding:\n\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); print_data_and_coding(k, m, w, sizeof(long), data, coding);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
return 0; return 0;
} }

8
Examples/reed_sol_02.c
View File

@ -50,7 +50,7 @@ usage(char *s)
fprintf(stderr, "usage: reed_sol_02 k m w - Vandermonde matrices in GF(2^w).\n"); fprintf(stderr, "usage: reed_sol_02 k m w - Vandermonde matrices in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " k+m must be <= 2^w. This simply prints out the \n"); fprintf(stderr, " k+m must be <= 2^w. This simply prints out the \n");
fprintf(stderr, " Vandermonde matrix in GF(2^w), and then the distribution\n"); fprintf(stderr, " Vandermonde matrix in GF(2^w), and then the generator\n");
fprintf(stderr, " matrix that is constructed from it. See [Plank-Ding-05] for\n"); fprintf(stderr, " matrix that is constructed from it. See [Plank-Ding-05] for\n");
fprintf(stderr, " information on how this construction proceeds\n"); fprintf(stderr, " information on how this construction proceeds\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
@ -75,12 +75,16 @@ int main(int argc, char **argv)
if (w <= 30 && k + m > (1 << w)) usage("k + m is too big"); if (w <= 30 && k + m > (1 << w)) usage("k + m is too big");
matrix = reed_sol_extended_vandermonde_matrix(k+m, k, w); matrix = reed_sol_extended_vandermonde_matrix(k+m, k, w);
printf("<HTML><TITLE>reed_sol_02 %d %d %d</title>\n", k, m, w);
printf("<h3>reed_sol_02 %d %d %d</h3>\n", k, m, w);
printf("<pre>\n");
printf("Extended Vandermonde Matrix:\n\n"); printf("Extended Vandermonde Matrix:\n\n");
jerasure_print_matrix(matrix, k+m, k, w); jerasure_print_matrix(matrix, k+m, k, w);
printf("\n"); printf("\n");
matrix = reed_sol_big_vandermonde_distribution_matrix(k+m, k, w); matrix = reed_sol_big_vandermonde_distribution_matrix(k+m, k, w);
printf("Vandermonde Distribution Matrix:\n\n"); printf("Vandermonde Generator Matrix (G^T):\n\n");
jerasure_print_matrix(matrix, k+m, k, w); jerasure_print_matrix(matrix, k+m, k, w);
printf("\n"); printf("\n");

34
Examples/reed_sol_03.c
View File

@ -46,6 +46,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
@ -53,7 +54,7 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: reed_sol_03 k w - Does a simple RAID-6 coding example in GF(2^w).\n"); fprintf(stderr, "usage: reed_sol_03 k w seed - Does a simple RAID-6 coding example in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. k+2 must be <= 2^w. It sets up a classic\n"); fprintf(stderr, " w must be 8, 16 or 32. k+2 must be <= 2^w. It sets up a classic\n");
fprintf(stderr, " RAID-6 coding matrix based on Anvin's optimization and encodes\n"); fprintf(stderr, " RAID-6 coding matrix based on Anvin's optimization and encodes\n");
@ -112,38 +113,52 @@ int main(int argc, char **argv)
unsigned char uc; unsigned char uc;
int k, w, i, j, m; int k, w, i, j, m;
int *matrix; int *matrix;
char **data, **coding; char **data, **coding, **dcopy, **ccopy;
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
uint32_t seed;
if (argc != 3) usage(NULL); if (argc != 4) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); if (sscanf(argv[2], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[3], "%d", &seed) == 0) usage("Bad seed");
m = 2; m = 2;
if (w <= 16 && k + m > (1 << w)) usage("k + m is too big"); if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
MOA_Seed(seed);
matrix = reed_sol_r6_coding_matrix(k, w); matrix = reed_sol_r6_coding_matrix(k, w);
printf("<HTML><TITLE>reed_sol_03 %d %d %d</title>\n", k, w, seed);
printf("<h3>reed_sol_03 %d %d %d</h3>\n", k, w, seed);
printf("<pre>\n");
printf("Last 2 rows of the Generator Matrix:\n\n"); printf("Last 2 rows of the Generator Matrix:\n\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n"); printf("\n");
srand48(0);
data = talloc(char *, k); data = talloc(char *, k);
dcopy = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, sizeof(long)); data[i] = talloc(char, sizeof(long));
dcopy[i] = talloc(char, sizeof(long));
for (j = 0; j < sizeof(long); j++) { for (j = 0; j < sizeof(long); j++) {
uc = lrand48()%256; uc = MOA_Random_W(8, 1) %256;
data[i][j] = (char) uc; data[i][j] = (char) uc;
} }
memcpy(dcopy[i], data[i], sizeof(long));
} }
coding = talloc(char *, m); coding = talloc(char *, m);
ccopy = talloc(char *, m);
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
coding[i] = talloc(char, sizeof(long)); coding[i] = talloc(char, sizeof(long));
ccopy[i] = talloc(char, sizeof(long));
} }
reed_sol_r6_encode(k, w, data, coding, sizeof(long)); reed_sol_r6_encode(k, w, data, coding, sizeof(long));
for (i = 0; i < m; i++) {
memcpy(ccopy[i], coding[i], sizeof(long));
}
printf("Encoding Complete:\n\n"); printf("Encoding Complete:\n\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); print_data_and_coding(k, m, w, sizeof(long), data, coding);
@ -153,7 +168,7 @@ int main(int argc, char **argv)
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0; l = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = lrand48()%(k+m); erasures[i] = ((unsigned int) MOA_Random_W(w, 1))%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
memcpy((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], &l, sizeof(long)); memcpy((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], &l, sizeof(long));
@ -170,5 +185,12 @@ int main(int argc, char **argv)
printf("State of the system after decoding:\n\n"); printf("State of the system after decoding:\n\n");
print_data_and_coding(k, m, w, sizeof(long), data, coding); print_data_and_coding(k, m, w, sizeof(long), data, coding);
for (i = 0; i < k; i++) if (memcmp(data[i], dcopy[i], sizeof(long)) != 0) {
printf("ERROR: D%x after decoding does not match its state before decoding!<br>\n", i);
}
for (i = 0; i < m; i++) if (memcmp(coding[i], ccopy[i], sizeof(long)) != 0) {
printf("ERROR: C%x after decoding does not match its state before decoding!<br>\n", i);
}
return 0; return 0;
} }

16
Examples/reed_sol_04.c
View File

@ -40,7 +40,9 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdint.h>
#include <string.h> #include <string.h>
#include <gf_rand.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
@ -48,7 +50,7 @@
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: reed_sol_04 w - Shows reed_sol_galois_wXX_region_multby_2\n"); fprintf(stderr, "usage: reed_sol_04 w seed - Shows reed_sol_galois_wXX_region_multby_2\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. Sets up an array of 4 random words in\n"); fprintf(stderr, " w must be 8, 16 or 32. Sets up an array of 4 random words in\n");
fprintf(stderr, " GF(2^w) and multiplies them by two. \n"); fprintf(stderr, " GF(2^w) and multiplies them by two. \n");
@ -65,18 +67,24 @@ int main(int argc, char **argv)
unsigned char *x, *y; unsigned char *x, *y;
unsigned short *xs, *ys; unsigned short *xs, *ys;
unsigned int *xi, *yi; unsigned int *xi, *yi;
uint32_t seed;
int *a32, *copy; int *a32, *copy;
int i; int i;
int w; int w;
if (argc != 2) usage(NULL); if (argc != 3) usage(NULL);
if (sscanf(argv[1], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); if (sscanf(argv[1], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[2], "%d", &seed) == 0) usage("Bad seed");
srand48(time(0)); printf("<HTML><TITLE>reed_sol_04 %d %d</title>\n", w, seed);
printf("<h3>reed_sol_04 %d %d</h3>\n", w, seed);
printf("<pre>\n");
MOA_Seed(seed);
a32 = talloc(int, 4); a32 = talloc(int, 4);
copy = talloc(int, 4); copy = talloc(int, 4);
y = (unsigned char *) a32; y = (unsigned char *) a32;
for (i = 0; i < 4*sizeof(int); i++) y[i] = lrand48()%255; for (i = 0; i < 4*sizeof(int); i++) y[i] = MOA_Random_W(8, 1);
memcpy(copy, a32, sizeof(int)*4); memcpy(copy, a32, sizeof(int)*4);
if (w == 8) { if (w == 8) {

361
Examples/reed_sol_hard_time_gf.c
View File

@ -1,361 +0,0 @@
/* *
* Copyright (c) 2013, James S. Plank and Kevin Greenan
* All rights reserved.
*
* Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
* Coding Techniques
*
* Revision 2.0: Galois Field backend now links to GF-Complete
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* - Neither the name of the University of Tennessee nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
* WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <gf_complete.h>
#include "jerasure.h"
#include "reed_sol.h"
#define BUFSIZE 4096
int get_gfp_from_argv(gf_t **gfp, int w, int argc, char **argv, int starting)
{
int mult_type, divide_type, region_type;
int arg1, arg2, subrg_size, degree;
uint64_t prim_poly;
gf_t *base;
char *crt, *x, *y;
mult_type = GF_MULT_DEFAULT;
region_type = GF_REGION_DEFAULT;
divide_type = GF_DIVIDE_DEFAULT;
prim_poly = 0;
base = NULL;
arg1 = 0;
arg2 = 0;
degree = 0;
while (1) {
if (argc > starting) {
if (strcmp(argv[starting], "-m") == 0) {
starting++;
if (mult_type != GF_MULT_DEFAULT) {
if (base != NULL) gf_free(base, 1);
return 0;
}
if (strcmp(argv[starting], "SHIFT") == 0) {
mult_type = GF_MULT_SHIFT;
starting++;
} else if (strcmp(argv[starting], "CARRY_FREE") == 0) {
mult_type = GF_MULT_CARRY_FREE;
starting++;
} else if (strcmp(argv[starting], "GROUP") == 0) {
mult_type = GF_MULT_GROUP;
if (argc < starting + 3) {
return 0;
}
if (sscanf(argv[starting+1], "%d", &arg1) == 0 ||
sscanf(argv[starting+2], "%d", &arg2) == 0) {
return 0;
}
starting += 3;
} else if (strcmp(argv[starting], "BYTWO_p") == 0) {
mult_type = GF_MULT_BYTWO_p;
starting++;
} else if (strcmp(argv[starting], "BYTWO_b") == 0) {
mult_type = GF_MULT_BYTWO_b;
starting++;
} else if (strcmp(argv[starting], "TABLE") == 0) {
mult_type = GF_MULT_TABLE;
starting++;
} else if (strcmp(argv[starting], "LOG") == 0) {
mult_type = GF_MULT_LOG_TABLE;
starting++;
} else if (strcmp(argv[starting], "LOG_ZERO") == 0) {
mult_type = GF_MULT_LOG_ZERO;
starting++;
} else if (strcmp(argv[starting], "LOG_ZERO_EXT") == 0) {
mult_type = GF_MULT_LOG_ZERO_EXT;
starting++;
} else if (strcmp(argv[starting], "SPLIT") == 0) {
mult_type = GF_MULT_SPLIT_TABLE;
if (argc < starting + 3) {
return 0;
}
if (sscanf(argv[starting+1], "%d", &arg1) == 0 ||
sscanf(argv[starting+2], "%d", &arg2) == 0) {
return 0;
}
starting += 3;
} else if (strcmp(argv[starting], "COMPOSITE") == 0) {
mult_type = GF_MULT_COMPOSITE;
if (argc < starting + 2) { return 0; }
if (sscanf(argv[starting+1], "%d", &arg1) == 0) {
return 0;
}
starting += 2;
degree = arg1;
starting = get_gfp_from_argv(&base, w/degree, argc, argv, starting);
if (starting == 0) {
free(base);
return 0;
}
} else {
if (base != NULL) gf_free(base, 1);
return 0;
}
} else if (strcmp(argv[starting], "-r") == 0) {
starting++;
if (strcmp(argv[starting], "DOUBLE") == 0) {
region_type |= GF_REGION_DOUBLE_TABLE;
starting++;
} else if (strcmp(argv[starting], "QUAD") == 0) {
region_type |= GF_REGION_QUAD_TABLE;
starting++;
} else if (strcmp(argv[starting], "LAZY") == 0) {
region_type |= GF_REGION_LAZY;
starting++;
} else if (strcmp(argv[starting], "SSE") == 0) {
region_type |= GF_REGION_SSE;
starting++;
} else if (strcmp(argv[starting], "NOSSE") == 0) {
region_type |= GF_REGION_NOSSE;
starting++;
} else if (strcmp(argv[starting], "CAUCHY") == 0) {
region_type |= GF_REGION_CAUCHY;
starting++;
} else if (strcmp(argv[starting], "ALTMAP") == 0) {
region_type |= GF_REGION_ALTMAP;
starting++;
} else {
if (base != NULL) gf_free(base, 1);
return 0;
}
} else if (strcmp(argv[starting], "-p") == 0) {
starting++;
if (sscanf(argv[starting], "%llx", (long long unsigned int *)(&prim_poly)) == 0) {
if (base != NULL) gf_free(base, 1);
return 0;
}
starting++;
} else if (strcmp(argv[starting], "-d") == 0) {
starting++;
if (divide_type != GF_DIVIDE_DEFAULT) {
if (base != NULL) gf_free(base, 1);
return 0;
} else if (strcmp(argv[starting], "EUCLID") == 0) {
divide_type = GF_DIVIDE_EUCLID;
starting++;
} else if (strcmp(argv[starting], "MATRIX") == 0) {
divide_type = GF_DIVIDE_MATRIX;
starting++;
} else {
return 0;
}
} else if (strcmp(argv[starting], "-") == 0) {
if (mult_type == GF_MULT_COMPOSITE) {
*gfp = galois_init_composite_field(w, region_type, divide_type, degree, base);
} else {
*gfp = galois_init_field(w, mult_type, region_type, divide_type, prim_poly, arg1, arg2);
}
starting++;
break;
} else {
if (base != NULL) gf_free(base, 1);
return 0;
}
}
}
return starting;
}
static void *malloc16(int size) {
void *mem = malloc(size+16+sizeof(void*));
void **ptr = (void**)((long)(mem+16+sizeof(void*)) & ~(15));
ptr[-1] = mem;
return ptr;
}
static void free16(void *ptr) {
free(((void**)ptr)[-1]);
}
#define talloc(type, num) (type *) malloc16(sizeof(type)*(num))
void
timer_start (double *t)
{
struct timeval tv;
gettimeofday (&tv, NULL);
*t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
}
double
timer_split (const double *t)
{
struct timeval tv;
double cur_t;
gettimeofday (&tv, NULL);
cur_t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
return (cur_t - *t);
}
usage(char *s)
{
fprintf(stderr, "usage: reed_sol_hard_test_gf k m w iterations bufsize [additional GF args]- Test and time Reed-Solomon in a particular GF(2^w).\n");
fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n");
fprintf(stderr, " See the README for information on the additional GF args.\n");
fprintf(stderr, " Set up a Vandermonde-based distribution matrix and encodes k devices of\n");
fprintf(stderr, " %d bytes each with it. Then it decodes.\n", BUFSIZE);
fprintf(stderr, " \n");
fprintf(stderr, "This tests: jerasure_matrix_encode()\n");
fprintf(stderr, " jerasure_matrix_decode()\n");
fprintf(stderr, " jerasure_print_matrix()\n");
fprintf(stderr, " galois_init_composite_field()\n");
fprintf(stderr, " galois_init_field()\n");
fprintf(stderr, " galois_change_technique()\n");
fprintf(stderr, " reed_sol_vandermonde_coding_matrix()\n");
if (s != NULL) fprintf(stderr, "%s\n", s);
exit(1);
}
static void fill_buffer(unsigned char *buf, int size)
{
int i;
buf[0] = (char)(lrand48() % 256);
for (i=1; i < size; i++) {
buf[i] = ((buf[i-1] + i) % 256);
}
}
int main(int argc, char **argv)
{
long l;
int k, w, i, j, m, iterations, bufsize;
int *matrix;
char **data, **coding, **old_values;
int *erasures, *erased;
int *decoding_matrix, *dm_ids;
double t = 0, total_time = 0;
gf_t *gf = NULL;
int ret = 0;
if (argc < 6) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[4], "%d", &iterations) == 0) usage("Bad iterations");
if (sscanf(argv[5], "%d", &bufsize) == 0) usage("Bad bufsize");
if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
srand48(time(0));
ret = get_gfp_from_argv(&gf, w, argc, argv, 6);
if (ret == 0 || gf == NULL) {
usage("Invalid arguments given for GF!\n");
}
galois_change_technique(gf, w);
matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
printf("Last m rows of the Distribution Matrix:\n\n");
jerasure_print_matrix(matrix, m, k, w);
printf("\n");
data = talloc(char *, k);
for (i = 0; i < k; i++) {
data[i] = talloc(char, bufsize);
fill_buffer(data[i], bufsize);
}
coding = talloc(char *, m);
old_values = talloc(char *, m);
for (i = 0; i < m; i++) {
coding[i] = talloc(char, bufsize);
old_values[i] = talloc(char, bufsize);
}
for (i = 0; i < iterations; i++) {
timer_start(&t);
jerasure_matrix_encode(k, m, w, matrix, data, coding, bufsize);
total_time += timer_split(&t);
}
fprintf(stderr, "Encode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time);
erasures = talloc(int, (m+1));
erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0;
for (i = 0; i < m; ) {
erasures[i] = ((unsigned int)lrand48())%(k+m);
if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1;
memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
i++;
}
}
erasures[i] = -1;
for (i = 0; i < iterations; i++) {
timer_start(&t);
jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, bufsize);
total_time += timer_split(&t);
}
fprintf(stderr, "Decode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(m*iterations*bufsize/1024/1024) / total_time, total_time);
for (i = 0; i < m; i++) {
if (erasures[i] < k) {
if (memcmp(data[erasures[i]], old_values[i], bufsize)) {
fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
exit(1);
}
} else {
if (memcmp(coding[erasures[i]-k], old_values[i], bufsize)) {
fprintf(stderr, "Decoding failed for %d!\n", erasures[i]);
exit(1);
}
}
}
return 0;
}

38
Examples/reed_sol_test_gf.c
View File

@ -42,6 +42,8 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <gf_complete.h> #include <gf_complete.h>
#include <gf_method.h>
#include <stdint.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
@ -62,7 +64,7 @@ static void free16(void *ptr) {
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: reed_sol_test_gf k m w [additional GF args]- Tests Reed-Solomon in GF(2^w).\n"); fprintf(stderr, "usage: reed_sol_test_gf k m w seed [additional GF args]- Tests Reed-Solomon in GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n"); fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n");
fprintf(stderr, " See the README for information on the additional GF args.\n"); fprintf(stderr, " See the README for information on the additional GF args.\n");
@ -88,17 +90,6 @@ gf_t* get_gf(int w, int argc, char **argv, int starting)
return gf; return gf;
} }
static void fill_buffer(unsigned char *buf, int size)
{
int i;
buf[0] = (char)(lrand48() % 256);
for (i=1; i < size; i++) {
buf[i] = ((buf[i-1] + i) % 256);
}
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
long l; long l;
@ -108,16 +99,18 @@ int main(int argc, char **argv)
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
gf_t *gf = NULL; gf_t *gf = NULL;
uint32_t seed;
if (argc < 4) usage(NULL); if (argc < 6) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (w <= 16 && k + m > (1 << w)) usage("k + m is too big"); if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
srand48(time(0)); MOA_Seed(seed);
gf = get_gf(w, argc, argv, 4); gf = get_gf(w, argc, argv, 5);
if (gf == NULL) { if (gf == NULL) {
usage("Invalid arguments given for GF!\n"); usage("Invalid arguments given for GF!\n");
@ -127,14 +120,22 @@ int main(int argc, char **argv)
matrix = reed_sol_vandermonde_coding_matrix(k, m, w); matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
printf("Last m rows of the Distribution Matrix:\n\n"); printf("<HTML><TITLE>reed_sol_test_gf");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>reed_sol_test_gf");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
printf("Last m rows of the generator matrix (G^T):\n\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n"); printf("\n");
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, BUFSIZE); data[i] = talloc(char, BUFSIZE);
fill_buffer(data[i], BUFSIZE); MOA_Fill_Random_Region(data[i], BUFSIZE);
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -151,7 +152,7 @@ int main(int argc, char **argv)
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0; l = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = ((unsigned int)lrand48())%(k+m); erasures[i] = ((unsigned int)MOA_Random_W(w,1))%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], BUFSIZE); memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], BUFSIZE);
@ -177,5 +178,6 @@ int main(int argc, char **argv)
} }
} }
printf("Encoding and decoding were both successful.\n");
return 0; return 0;
} }

50
Examples/reed_sol_time_gf.c
View File

@ -41,11 +41,12 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <gf_complete.h> #include <gf_complete.h>
#include <gf_rand.h>
#include <gf_method.h>
#include <stdint.h>
#include "jerasure.h" #include "jerasure.h"
#include "reed_sol.h" #include "reed_sol.h"
#define BUFSIZE 4096
static void *malloc16(int size) { static void *malloc16(int size) {
void *mem = malloc(size+16+sizeof(void*)); void *mem = malloc(size+16+sizeof(void*));
void **ptr = (void**)((long)(mem+16+sizeof(void*)) & ~(15)); void **ptr = (void**)((long)(mem+16+sizeof(void*)) & ~(15));
@ -81,12 +82,12 @@ timer_split (const double *t)
usage(char *s) usage(char *s)
{ {
fprintf(stderr, "usage: reed_sol_time_gf k m w iterations bufsize [additional GF args]- Test and time Reed-Solomon in a particular GF(2^w).\n"); fprintf(stderr, "usage: reed_sol_time_gf k m w seed iterations bufsize (additional GF args) - Test and time Reed-Solomon in a particular GF(2^w).\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n"); fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n");
fprintf(stderr, " See the README for information on the additional GF args.\n"); fprintf(stderr, " See the README for information on the additional GF args.\n");
fprintf(stderr, " Set up a Vandermonde-based distribution matrix and encodes k devices of\n"); fprintf(stderr, " Set up a Vandermonde-based distribution matrix and encodes k devices of\n");
fprintf(stderr, " %d bytes each with it. Then it decodes.\n", BUFSIZE); fprintf(stderr, " bufsize bytes each with it. Then it decodes.\n");
fprintf(stderr, " \n"); fprintf(stderr, " \n");
fprintf(stderr, "This tests: jerasure_matrix_encode()\n"); fprintf(stderr, "This tests: jerasure_matrix_encode()\n");
fprintf(stderr, " jerasure_matrix_decode()\n"); fprintf(stderr, " jerasure_matrix_decode()\n");
@ -107,17 +108,6 @@ gf_t* get_gf(int w, int argc, char **argv, int starting)
return gf; return gf;
} }
static void fill_buffer(unsigned char *buf, int size)
{
int i;
buf[0] = (char)(lrand48() % 256);
for (i=1; i < size; i++) {
buf[i] = ((buf[i-1] + i) % 256);
}
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
long l; long l;
@ -126,20 +116,22 @@ int main(int argc, char **argv)
char **data, **coding, **old_values; char **data, **coding, **old_values;
int *erasures, *erased; int *erasures, *erased;
int *decoding_matrix, *dm_ids; int *decoding_matrix, *dm_ids;
uint32_t seed;
double t = 0, total_time = 0; double t = 0, total_time = 0;
gf_t *gf = NULL; gf_t *gf = NULL;
if (argc < 6) usage(NULL); if (argc < 8) usage(NULL);
if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k");
if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m");
if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w");
if (sscanf(argv[4], "%d", &iterations) == 0) usage("Bad iterations"); if (sscanf(argv[4], "%d", &seed) == 0) usage("Bad seed");
if (sscanf(argv[5], "%d", &bufsize) == 0) usage("Bad bufsize"); if (sscanf(argv[5], "%d", &iterations) == 0) usage("Bad iterations");
if (sscanf(argv[6], "%d", &bufsize) == 0) usage("Bad bufsize");
if (w <= 16 && k + m > (1 << w)) usage("k + m is too big"); if (w <= 16 && k + m > (1 << w)) usage("k + m is too big");
srand48(time(0)); MOA_Seed(seed);
gf = get_gf(w, argc, argv, 6); gf = get_gf(w, argc, argv, 7);
if (gf == NULL) { if (gf == NULL) {
usage("Invalid arguments given for GF!\n"); usage("Invalid arguments given for GF!\n");
@ -149,14 +141,22 @@ int main(int argc, char **argv)
matrix = reed_sol_vandermonde_coding_matrix(k, m, w); matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
printf("Last m rows of the Distribution Matrix:\n\n"); printf("<HTML><TITLE>reed_sol_time_gf");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</TITLE>\n");
printf("<h3>reed_sol_time_gf");
for (i = 1; i < argc; i++) printf(" %s", argv[i]);
printf("</h3>\n");
printf("<pre>\n");
printf("Last m rows of the generator matrix (G^T):\n\n");
jerasure_print_matrix(matrix, m, k, w); jerasure_print_matrix(matrix, m, k, w);
printf("\n"); printf("\n");
data = talloc(char *, k); data = talloc(char *, k);
for (i = 0; i < k; i++) { for (i = 0; i < k; i++) {
data[i] = talloc(char, bufsize); data[i] = talloc(char, bufsize);
fill_buffer(data[i], bufsize); MOA_Fill_Random_Region(data[i], bufsize);
} }
coding = talloc(char *, m); coding = talloc(char *, m);
@ -172,14 +172,14 @@ int main(int argc, char **argv)
total_time += timer_split(&t); total_time += timer_split(&t);
} }
fprintf(stderr, "Encode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time); printf("Encode throughput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time);
erasures = talloc(int, (m+1)); erasures = talloc(int, (m+1));
erased = talloc(int, (k+m)); erased = talloc(int, (k+m));
for (i = 0; i < m+k; i++) erased[i] = 0; for (i = 0; i < m+k; i++) erased[i] = 0;
l = 0; l = 0;
for (i = 0; i < m; ) { for (i = 0; i < m; ) {
erasures[i] = ((unsigned int)lrand48())%(k+m); erasures[i] = ((unsigned int)MOA_Random_W(w, 1))%(k+m);
if (erased[erasures[i]] == 0) { if (erased[erasures[i]] == 0) {
erased[erasures[i]] = 1; erased[erasures[i]] = 1;
memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize); memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize);
@ -195,7 +195,7 @@ int main(int argc, char **argv)
total_time += timer_split(&t); total_time += timer_split(&t);
} }
fprintf(stderr, "Decode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(m*iterations*bufsize/1024/1024) / total_time, total_time); printf("Decode throughput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time);
for (i = 0; i < m; i++) { for (i = 0; i < m; i++) {
if (erasures[i] < k) { if (erasures[i] < k) {

30
Examples/test_all_gfs.sh
View File

@ -41,14 +41,15 @@ GF_COMPLETE_DIR=/usr/local/bin
GF_METHODS=${GF_COMPLETE_DIR}/gf_methods GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
k=12 k=12
m=3 m=3
seed=1370
# Test all w=8 # Test all w=8
${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' | ${GF_METHODS} 8 -B -L | awk -F: '{ if ($1 == "w=8") print $2; }' |
while read method; do while read method; do
echo "Testing ${k} ${m} 8 ${method}" echo "Testing ${k} ${m} 8 $seed ${method}"
./reed_sol_test_gf ${k} ${m} 8 ${method} ./reed_sol_test_gf ${k} ${m} 8 $seed ${method} | tail -n 1
if [[ $? != "0" ]]; then if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 8 ${method}" echo "Failed test for ${k} ${m} 8 $seed ${method}"
exit 1 exit 1
fi fi
done done
@ -57,34 +58,35 @@ if [[ $? == "1" ]]; then
exit 1 exit 1
fi fi
# Test all w=16 # Test all w=16
${GF_METHODS} | awk -F: '{ if ($1 == "w=16") print $2; }' | ${GF_METHODS} 16 -B -L | awk -F: '{ if ($1 == "w=16") print $2; }' |
while read method; do while read method; do
echo "Testing ${k} ${m} 16 ${method}" echo "Testing ${k} ${m} 16 $seed ${method}"
./reed_sol_test_gf ${k} ${m} 16 ${method} ./reed_sol_test_gf ${k} ${m} 16 $seed ${method} | tail -n 1
if [[ $? != "0" ]]; then if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 16 ${method}" echo "Failed test for ${k} ${m} 16 $seed ${method}"
FAIL="1"
exit 1 exit 1
fi fi
done done
if [[ $? == "1" ]]; then if [[ $? == "1" ]]; then
exit 1 exit 1
fi fi
# Test all w=32 # Test all w=32
${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' | ${GF_METHODS} 32 -B -L | awk -F: '{ if ($1 == "w=32") print $2; }' |
while read method; do while read method; do
echo "Testing ${k} ${m} 32 ${method}" echo "Testing ${k} ${m} 32 $seed ${method}"
./reed_sol_test_gf ${k} ${m} 32 ${method} ./reed_sol_test_gf ${k} ${m} 32 $seed ${method} | tail -n 1
if [[ $? != "0" ]]; then if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 32 ${method}" echo "Failed test for ${k} ${m} 32 $seed ${method}"
FAIL="1"
exit 1 exit 1
fi fi
done done
if [[ $? == "1" ]]; then if [[ $? == "1" ]]; then
exit 1 exit 1
fi fi

54
Examples/time_all_gfs_argv_init.sh
View File

@ -43,50 +43,22 @@ ITERATIONS=128
BUFSIZE=65536 BUFSIZE=65536
k=12 k=12
m=3 m=3
seed=1370
# Test all w=8 for w in 8 16 32 ; do
${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' | ${GF_METHODS} $w -B -L | awk -F: '{ if ($1 == "w='$w'") print $2; }' |
while read method; do while read method; do
echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}" echo "Testing ${k} ${m} ${w} ${seed} ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method} ./reed_sol_time_gf ${k} ${m} ${w} ${seed} ${ITERATIONS} ${BUFSIZE} ${method} | tail -n 2
if [[ $? != "0" ]]; then if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}" echo "Failed test for ${k} ${m} ${w} ${seed} ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1 exit 1
fi fi
done done
if [[ $? == "1" ]]; then
exit 1
fi
# Test all w=16
${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
while read method; do
echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1
fi
# Test all w=32
${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
while read method; do
echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1
fi
echo "Passed all tests!" echo "Passed all tests!"

92
Examples/time_all_gfs_hard_init.sh
View File

@ -1,92 +0,0 @@
#
#
# Copyright (c) 2013, James S. Plank and Kevin Greenan
# All rights reserved.
#
# Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
# Coding Techniques
#
# Revision 2.0: Galois Field backend now links to GF-Complete
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# - Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# - Neither the name of the University of Tennessee nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
GF_COMPLETE_DIR=/usr/local/bin
GF_METHODS=${GF_COMPLETE_DIR}/gf_methods
ITERATIONS=128
BUFSIZE=65536
k=12
m=3
# Test all w=8
${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' |
while read method; do
echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_hard_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}
if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1
fi
# Test all w=16
${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' |
while read method; do
echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_hard_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}
if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1
fi
# Test all w=32
${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' |
while read method; do
echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
./reed_sol_hard_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}
if [[ $? != "0" ]]; then
echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}"
exit 1
fi
done
if [[ $? == "1" ]]; then
exit 1
fi
echo "Passed all tests!"

39
README
View File

@ -1,5 +1,7 @@
This is revision 2.0 of Jerasure. This is pretty much Jerasure 1.2 without the This is revision 2.0 of Jerasure. This is pretty much Jerasure 1.2 without the
original Galois Field backend. Version 2.0 links directly to GF-Complete. original Galois Field backend. Version 2.0 links directly to GF-Complete, which
is more flexible than the original, and *much* faster, because it leverages SIMD
instructions.
External Documentation: External Documentation:
@ -12,10 +14,9 @@ Custom usgae of GF-Complete is explained in this file (see below).
NOTE: You must have GF-Complete installed in order to use Jerasure 2.0. NOTE: You must have GF-Complete installed in order to use Jerasure 2.0.
There are two directories: There are two directories of source code:
The src directory contains the jerasure code. The src directory contains the jerasure code.
The Examples directory contains the example programs. The Examples directory contains the example programs.
The makefile assumes that Examples is a subdirectory of the home directory. The makefile assumes that Examples is a subdirectory of the home directory.
@ -33,7 +34,7 @@ Installing:
This will install the examples under PREFIX/bin, the library under PREFIX/lib This will install the examples under PREFIX/bin, the library under PREFIX/lib
and the header files under PREFIX/include and the header files under PREFIX/include
See individual source file to determine what the examples do. See individual source files to determine what the examples do.
Inclusion of GF-Complete: Inclusion of GF-Complete:
@ -44,8 +45,8 @@ determine the default field to use, if one is not specified.
If you would like to explore a using a different Galois Field implementation, If you would like to explore a using a different Galois Field implementation,
you can dynamically set the backend GF for a given word-size (w). you can dynamically set the backend GF for a given word-size (w).
The new galois.[ch] exports the following functions to be used by applications The new galois.c and galois.h export the following functions to be used by applications
for dynamically setting the backend GF: for dynamically setting the back-end GF:
1.) galois_change_technique 1.) galois_change_technique
@ -53,21 +54,25 @@ for dynamically setting the backend GF:
void galois_change_technique(gf_t *gf, int w); void galois_change_technique(gf_t *gf, int w);
This is the recommended way for you to change techniques.
This function will take a pointer to a Galois field structure and set it as the This function will take a pointer to a Galois field structure and set it as the
current backend for all operations in GF(2^w). Note that you must specify 'w' current backend for all operations in GF(2^w). Note that you must specify 'w'
here, since the internal GF structure is mostly opaque to Jerasure. Be sure to here, since the internal GF structure is mostly opaque to Jerasure. Be sure to
change the technique with the correct structure and word-size. change the technique with the correct structure and word-size.
There are a few ways to get a pointer to a gf_t structure: via GF-Complete or There are a few ways to get a pointer to a gf_t structure: GF-Complete gives three
using the helper functions provided by Jerasure: galois_init_field and primitives for this -- create_gf_from_argv(), gf_init_easy() and gf_init_hard().
galois_init_composite_field. Please read the documentation on GF-Complete for how these work. By far, the
most powerful and easy is create_gf_from_argv(), which parses an argv-style
string. Otherwise, the most flexible is gf_init_hard().
GF-Complete exposes create_gf_from_argv, gf_init_easy and gf_init_hard. See In galois.c/galois.h, we have defined galois_init_field(), which is pretty much
the GF-Complete documentation for more detail on how to use those functions. identical to gf_init_hard(), except it performs memory allocation with malloc(),
You can definitely create more complicated fields with the GF-Complete and galois_init_composite_field(), which facilitates creating composite fields.
initialization functions, but the two helper functions provided by Jerasure
(galois_init_field and galois_init_composite_field) can be used to create most These are described below, but once again, we recommend using create_gf_from_argv()
of the supported Galois Fields. or gf_init_hard() if you want to change your Galois field.
2.) galois_init_field 2.) galois_init_field
@ -149,10 +154,6 @@ For more information on how to change the backing fields for Jerasure, please re
2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests 2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests
for a backing GF (uses create_gf_from_argv to get gf_t pointer) for a backing GF (uses create_gf_from_argv to get gf_t pointer)
3.) Examples/reed_sol_hard_time_gf.c: Runs more thorough timing and validation tests
for a backing GF (uses galois_init_field and galois_init_composite_field to get gf_t
pointer)
Performance: Performance:
There are two performance-based test scripts: time_all_gfs_argv_init.sh and There are two performance-based test scripts: time_all_gfs_argv_init.sh and