diff --git a/Examples/makefile b/Examples/makefile index 21ea13d..d838ee6 100644 --- a/Examples/makefile +++ b/Examples/makefile @@ -63,6 +63,7 @@ ALL = jerasure_01 \ reed_sol_04 \ reed_sol_test_gf \ reed_sol_time_gf \ + reed_sol_hard_time_gf \ cauchy_01 \ cauchy_02 \ cauchy_03 \ @@ -190,9 +191,13 @@ decoder: decoder.o galois.o jerasure.o liberation.o reed_sol.o cauchy.o $(CC) $(CFLAGS) -o decoder decoder.o liberation.o jerasure.o galois.o reed_sol.o cauchy.o -lgf_complete reed_sol_test_gf.o: galois.h reed_sol.h jerasure.h -reed_sol_test_gf: reed_sol_test_gf.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o - $(CC) $(CFLAGS) -o reed_sol_test_gf reed_sol_test_gf.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a +reed_sol_test_gf: reed_sol_test_gf.o galois.o jerasure.o reed_sol.o + $(CC) $(CFLAGS) -o reed_sol_test_gf reed_sol_test_gf.o reed_sol.o jerasure.o galois.o -lgf_complete reed_sol_time_gf.o: galois.h reed_sol.h jerasure.h -reed_sol_time_gf: reed_sol_time_gf.o galois.o ${LIBDIR}/gf_complete.a jerasure.o reed_sol.o - $(CC) $(CFLAGS) -o reed_sol_time_gf reed_sol_time_gf.o reed_sol.o jerasure.o galois.o ${LIBDIR}/gf_complete.a +reed_sol_time_gf: reed_sol_time_gf.o galois.o jerasure.o reed_sol.o + $(CC) $(CFLAGS) -o reed_sol_time_gf reed_sol_time_gf.o reed_sol.o jerasure.o galois.o -lgf_complete + +reed_sol_hard_time_gf.o: galois.h reed_sol.h jerasure.h +reed_sol_hard_time_gf: reed_sol_hard_time_gf.o galois.o jerasure.o reed_sol.o + $(CC) $(CFLAGS) -o reed_sol_hard_time_gf reed_sol_hard_time_gf.o reed_sol.o jerasure.o galois.o -lgf_complete diff --git a/Examples/reed_sol_hard_time_gf.c b/Examples/reed_sol_hard_time_gf.c new file mode 100644 index 0000000..e3cf6a2 --- /dev/null +++ b/Examples/reed_sol_hard_time_gf.c @@ -0,0 +1,361 @@ +/* * + * Copyright (c) 2013, James S. Plank and Kevin Greenan + * All rights reserved. + * + * Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure + * Coding Techniques + * + * Revision 2.0: Galois Field backend now links to GF-Complete + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * - Neither the name of the University of Tennessee nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + + +#include +#include +#include +#include +#include "jerasure.h" +#include "reed_sol.h" + +#define BUFSIZE 4096 + +int get_gfp_from_argv(gf_t **gfp, int w, int argc, char **argv, int starting) +{ + int mult_type, divide_type, region_type; + int arg1, arg2, subrg_size, degree; + uint64_t prim_poly; + gf_t *base; + char *crt, *x, *y; + + mult_type = GF_MULT_DEFAULT; + region_type = GF_REGION_DEFAULT; + divide_type = GF_DIVIDE_DEFAULT; + prim_poly = 0; + base = NULL; + arg1 = 0; + arg2 = 0; + degree = 0; + while (1) { + if (argc > starting) { + if (strcmp(argv[starting], "-m") == 0) { + starting++; + if (mult_type != GF_MULT_DEFAULT) { + if (base != NULL) gf_free(base, 1); + return 0; + } + if (strcmp(argv[starting], "SHIFT") == 0) { + mult_type = GF_MULT_SHIFT; + starting++; + } else if (strcmp(argv[starting], "CARRY_FREE") == 0) { + mult_type = GF_MULT_CARRY_FREE; + starting++; + } else if (strcmp(argv[starting], "GROUP") == 0) { + mult_type = GF_MULT_GROUP; + if (argc < starting + 3) { + return 0; + } + if (sscanf(argv[starting+1], "%d", &arg1) == 0 || + sscanf(argv[starting+2], "%d", &arg2) == 0) { + return 0; + } + starting += 3; + } else if (strcmp(argv[starting], "BYTWO_p") == 0) { + mult_type = GF_MULT_BYTWO_p; + starting++; + } else if (strcmp(argv[starting], "BYTWO_b") == 0) { + mult_type = GF_MULT_BYTWO_b; + starting++; + } else if (strcmp(argv[starting], "TABLE") == 0) { + mult_type = GF_MULT_TABLE; + starting++; + } else if (strcmp(argv[starting], "LOG") == 0) { + mult_type = GF_MULT_LOG_TABLE; + starting++; + } else if (strcmp(argv[starting], "LOG_ZERO") == 0) { + mult_type = GF_MULT_LOG_ZERO; + starting++; + } else if (strcmp(argv[starting], "LOG_ZERO_EXT") == 0) { + mult_type = GF_MULT_LOG_ZERO_EXT; + starting++; + } else if (strcmp(argv[starting], "SPLIT") == 0) { + mult_type = GF_MULT_SPLIT_TABLE; + if (argc < starting + 3) { + return 0; + } + if (sscanf(argv[starting+1], "%d", &arg1) == 0 || + sscanf(argv[starting+2], "%d", &arg2) == 0) { + return 0; + } + starting += 3; + } else if (strcmp(argv[starting], "COMPOSITE") == 0) { + mult_type = GF_MULT_COMPOSITE; + if (argc < starting + 2) { return 0; } + if (sscanf(argv[starting+1], "%d", &arg1) == 0) { + return 0; + } + starting += 2; + degree = arg1; + starting = get_gfp_from_argv(&base, w/degree, argc, argv, starting); + if (starting == 0) { + free(base); + return 0; + } + } else { + if (base != NULL) gf_free(base, 1); + return 0; + } + } else if (strcmp(argv[starting], "-r") == 0) { + starting++; + if (strcmp(argv[starting], "DOUBLE") == 0) { + region_type |= GF_REGION_DOUBLE_TABLE; + starting++; + } else if (strcmp(argv[starting], "QUAD") == 0) { + region_type |= GF_REGION_QUAD_TABLE; + starting++; + } else if (strcmp(argv[starting], "LAZY") == 0) { + region_type |= GF_REGION_LAZY; + starting++; + } else if (strcmp(argv[starting], "SSE") == 0) { + region_type |= GF_REGION_SSE; + starting++; + } else if (strcmp(argv[starting], "NOSSE") == 0) { + region_type |= GF_REGION_NOSSE; + starting++; + } else if (strcmp(argv[starting], "CAUCHY") == 0) { + region_type |= GF_REGION_CAUCHY; + starting++; + } else if (strcmp(argv[starting], "ALTMAP") == 0) { + region_type |= GF_REGION_ALTMAP; + starting++; + } else { + if (base != NULL) gf_free(base, 1); + return 0; + } + } else if (strcmp(argv[starting], "-p") == 0) { + starting++; + if (sscanf(argv[starting], "%llx", (long long unsigned int *)(&prim_poly)) == 0) { + if (base != NULL) gf_free(base, 1); + return 0; + } + starting++; + } else if (strcmp(argv[starting], "-d") == 0) { + starting++; + if (divide_type != GF_DIVIDE_DEFAULT) { + if (base != NULL) gf_free(base, 1); + return 0; + } else if (strcmp(argv[starting], "EUCLID") == 0) { + divide_type = GF_DIVIDE_EUCLID; + starting++; + } else if (strcmp(argv[starting], "MATRIX") == 0) { + divide_type = GF_DIVIDE_MATRIX; + starting++; + } else { + return 0; + } + } else if (strcmp(argv[starting], "-") == 0) { + if (mult_type == GF_MULT_COMPOSITE) { + *gfp = galois_init_composite_field(w, region_type, divide_type, degree, base); + } else { + *gfp = galois_init_field(w, mult_type, region_type, divide_type, prim_poly, arg1, arg2); + } + starting++; + break; + } else { + if (base != NULL) gf_free(base, 1); + return 0; + } + } + } + + return starting; +} + +static void *malloc16(int size) { + void *mem = malloc(size+16+sizeof(void*)); + void **ptr = (void**)((long)(mem+16+sizeof(void*)) & ~(15)); + ptr[-1] = mem; + return ptr; +} + +static void free16(void *ptr) { + free(((void**)ptr)[-1]); +} + +#define talloc(type, num) (type *) malloc16(sizeof(type)*(num)) + +void +timer_start (double *t) +{ + struct timeval tv; + + gettimeofday (&tv, NULL); + *t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6; +} + +double +timer_split (const double *t) +{ + struct timeval tv; + double cur_t; + + gettimeofday (&tv, NULL); + cur_t = (double)tv.tv_sec + (double)tv.tv_usec * 1e-6; + return (cur_t - *t); +} + +usage(char *s) +{ + fprintf(stderr, "usage: reed_sol_hard_test_gf k m w [additional GF args]- Test and time Reed-Solomon in a particular GF(2^w).\n"); + fprintf(stderr, " \n"); + fprintf(stderr, " w must be 8, 16 or 32. k+m must be <= 2^w.\n"); + fprintf(stderr, " See the README for information on the additional GF args.\n"); + fprintf(stderr, " Set up a Vandermonde-based distribution matrix and encodes k devices of\n"); + fprintf(stderr, " %d bytes each with it. Then it decodes.\n", BUFSIZE); + fprintf(stderr, " \n"); + fprintf(stderr, "This tests: jerasure_matrix_encode()\n"); + fprintf(stderr, " jerasure_matrix_decode()\n"); + fprintf(stderr, " jerasure_print_matrix()\n"); + fprintf(stderr, " galois_init_composite_field()\n"); + fprintf(stderr, " galois_init_field()\n"); + fprintf(stderr, " galois_change_technique()\n"); + fprintf(stderr, " reed_sol_vandermonde_coding_matrix()\n"); + if (s != NULL) fprintf(stderr, "%s\n", s); + exit(1); +} + +static void fill_buffer(unsigned char *buf, int size) +{ + int i; + + buf[0] = (char)(lrand48() % 256); + + for (i=1; i < size; i++) { + buf[i] = ((buf[i-1] + i) % 256); + } +} + +int main(int argc, char **argv) +{ + long l; + int k, w, i, j, m, iterations, bufsize; + int *matrix; + char **data, **coding, **old_values; + int *erasures, *erased; + int *decoding_matrix, *dm_ids; + double t = 0, total_time = 0; + gf_t *gf = NULL; + int ret = 0; + + if (argc < 6) usage(NULL); + if (sscanf(argv[1], "%d", &k) == 0 || k <= 0) usage("Bad k"); + if (sscanf(argv[2], "%d", &m) == 0 || m <= 0) usage("Bad m"); + if (sscanf(argv[3], "%d", &w) == 0 || (w != 8 && w != 16 && w != 32)) usage("Bad w"); + if (sscanf(argv[4], "%d", &iterations) == 0) usage("Bad iterations"); + if (sscanf(argv[5], "%d", &bufsize) == 0) usage("Bad bufsize"); + if (w <= 16 && k + m > (1 << w)) usage("k + m is too big"); + + srand48(time(0)); + + ret = get_gfp_from_argv(&gf, w, argc, argv, 6); + + if (ret == 0 || gf == NULL) { + usage("Invalid arguments given for GF!\n"); + } + + galois_change_technique(gf, w); + + matrix = reed_sol_vandermonde_coding_matrix(k, m, w); + + printf("Last m rows of the Distribution Matrix:\n\n"); + jerasure_print_matrix(matrix, m, k, w); + printf("\n"); + + data = talloc(char *, k); + for (i = 0; i < k; i++) { + data[i] = talloc(char, bufsize); + fill_buffer(data[i], bufsize); + } + + coding = talloc(char *, m); + old_values = talloc(char *, m); + for (i = 0; i < m; i++) { + coding[i] = talloc(char, bufsize); + old_values[i] = talloc(char, bufsize); + } + + for (i = 0; i < iterations; i++) { + timer_start(&t); + jerasure_matrix_encode(k, m, w, matrix, data, coding, bufsize); + total_time += timer_split(&t); + } + + fprintf(stderr, "Encode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(k*iterations*bufsize/1024/1024) / total_time, total_time); + + erasures = talloc(int, (m+1)); + erased = talloc(int, (k+m)); + for (i = 0; i < m+k; i++) erased[i] = 0; + l = 0; + for (i = 0; i < m; ) { + erasures[i] = ((unsigned int)lrand48())%(k+m); + if (erased[erasures[i]] == 0) { + erased[erasures[i]] = 1; + memcpy(old_values[i], (erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize); + bzero((erasures[i] < k) ? data[erasures[i]] : coding[erasures[i]-k], bufsize); + i++; + } + } + erasures[i] = -1; + + for (i = 0; i < iterations; i++) { + timer_start(&t); + jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, bufsize); + total_time += timer_split(&t); + } + + fprintf(stderr, "Decode thput for %d iterations: %.2f MB/s (%.2f sec)\n", iterations, (double)(m*iterations*bufsize/1024/1024) / total_time, total_time); + + for (i = 0; i < m; i++) { + if (erasures[i] < k) { + if (memcmp(data[erasures[i]], old_values[i], bufsize)) { + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]); + exit(1); + } + } else { + if (memcmp(coding[erasures[i]-k], old_values[i], bufsize)) { + fprintf(stderr, "Decoding failed for %d!\n", erasures[i]); + exit(1); + } + } + } + + return 0; +} diff --git a/Examples/time_all_gfs.sh b/Examples/time_all_gfs_argv_init.sh similarity index 100% rename from Examples/time_all_gfs.sh rename to Examples/time_all_gfs_argv_init.sh diff --git a/Examples/time_all_gfs_hard_init.sh b/Examples/time_all_gfs_hard_init.sh new file mode 100755 index 0000000..586eac3 --- /dev/null +++ b/Examples/time_all_gfs_hard_init.sh @@ -0,0 +1,92 @@ +# +# +# Copyright (c) 2013, James S. Plank and Kevin Greenan +# All rights reserved. +# +# Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure +# Coding Techniques +# +# Revision 2.0: Galois Field backend now links to GF-Complete +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# - Neither the name of the University of Tennessee nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +GF_COMPLETE_DIR=/usr/local/bin +GF_METHODS=${GF_COMPLETE_DIR}/gf_methods +ITERATIONS=128 +BUFSIZE=65536 +k=12 +m=3 + +# Test all w=8 +${GF_METHODS} | awk -F: '{ if ($1 == "w=8") print $2; }' | +while read method; do + echo "Testing ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}" + ./reed_sol_hard_time_gf ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method} + if [[ $? != "0" ]]; then + echo "Failed test for ${k} ${m} 8 ${ITERATIONS} ${BUFSIZE} ${method}" + exit 1 + fi +done + +if [[ $? == "1" ]]; then + exit 1 +fi + +# Test all w=16 +${GF_METHODS} | grep -v 'TABLE' | awk -F: '{ if ($1 == "w=16") print $2; }' | +while read method; do + echo "Testing ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}" + ./reed_sol_hard_time_gf ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method} + if [[ $? != "0" ]]; then + echo "Failed test for ${k} ${m} 16 ${ITERATIONS} ${BUFSIZE} ${method}" + exit 1 + fi +done + +if [[ $? == "1" ]]; then + exit 1 +fi + +# Test all w=32 +${GF_METHODS} | awk -F: '{ if ($1 == "w=32") print $2; }' | +while read method; do + echo "Testing ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}" + ./reed_sol_hard_time_gf ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method} + if [[ $? != "0" ]]; then + echo "Failed test for ${k} ${m} 32 ${ITERATIONS} ${BUFSIZE} ${method}" + exit 1 + fi +done + +if [[ $? == "1" ]]; then + exit 1 +fi + +echo "Passed all tests!" diff --git a/README.nd b/README.nd index 8962d35..e1c7628 100644 --- a/README.nd +++ b/README.nd @@ -29,4 +29,112 @@ determine the default field to use, if one is not specified. If you would like to explore a using a different Galois Field implementation, you can dynamically set the backend GF for a given word-size (w). - +The new galois.[ch] exports the following functions to be used by applications +for dynamically setting the backend GF: + +1.) galois_change_technique + + Function signature: + + void galois_change_technique(gf_t *gf, int w); + + This function will take a pointer to a Galois field structure and set it as the + current backend for all operations in GF(2^w). Note that you must specify 'w' + here, since the internal GF structure is mostly opaque to Jerasure. Be sure to + change the technique with the correct structure and word-size. + + There are a few ways to get a pointer to a gf_t structure: via GF-Complete or + using the helper functions provided by Jerasure: galois_init_field and + galois_init_composite_field. + + GF-Complete exposes create_gf_from_argv, gf_init_easy and gf_init_hard. See + the GF-Complete documentation for more detail on how to use those functions. + You can definitely create more complicated fields with the GF-Complete + initialization functions, but the two helper functions provided by Jerasure + (galois_init_field and galois_init_composite_field) can be used to create most + of the supported Galois Fields. + +2.) galois_init_field + + Function signature: + + gf_t* galois_init_field(int w, + int mult_type, + int region_type, + int divide_type, + uint64_t prim_poly, + int arg1, + int arg2); + + This is a helper function that will initialize a Galois field. See the GF-Complete + documentation for more info on what the arguments mean. Here is a brief description + of the arguments: + + mult_type can be any *one* of the following: + + GF_MULT_DEFAULT + GF_MULT_SHIFT + GF_MULT_CARRY_FREE + GF_MULT_GROUP + GF_MULT_BYTWO_p + GF_MULT_BYTWO_b + GF_MULT_TABLE + GF_MULT_LOG_TABLE + GF_MULT_LOG_ZERO + GF_MULT_LOG_ZERO_EXT + GF_MULT_SPLIT_TABLE + + region_type can be a combination of the following (some combinations will not + be valid): + + GF_REGION_DEFAULT + GF_REGION_DOUBLE_TABLE + GF_REGION_QUAD_TABLE + GF_REGION_LAZY + GF_REGION_SSE + GF_REGION_NOSSE + GF_REGION_ALTMAP + GF_REGION_CAUCHY + + divide_type can be one of the following: + + GF_DIVIDE_DEFAULT + GF_DIVIDE_MATRIX + GF_DIVIDE_EUCLID + + prim_poly is the field-defining primitive polynomial + + arg1 and arg2 are special arguments usually used for defining SPLIT and GROUP + operations + +3.) galois_init_composite_field + + Function signature: + + gf_t* galois_init_composite_field(int w, + int region_type, + int divide_type, + int degree, + gf_t* base_gf); + + This is a helper function designed to make creating Composite fields easier. All you + need to do is hand it w, region mult type, divide type, degree and a pointer to a base + field. Note that the base_gf must have degree w/degree in order for this to work. + For example, if we create a GF using: + + galois_init_composite_field(32, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT, 2, base_gf); + + Then base_gf must have w=16. + +For more information on how to change the backing fields for Jerasure, please refer to + + 1.) Examples/reed_sol_test_gf.c: Runs basic tests for Reed-Solomon given args + for a backing GF (uses create_gf_from_argv to get gf_t pointer) + + 2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests + for a backing GF (uses create_gf_from_argv to get gf_t pointer) + + 3.) Examples/reed_sol_hard_time_gf.c: Runs more thorough timing and validation tests + for a backing GF (uses galois_init_field and galois_init_composite_field to get gf_t + pointer) + diff --git a/README.txt b/README.txt index 8962d35..e1c7628 100644 --- a/README.txt +++ b/README.txt @@ -29,4 +29,112 @@ determine the default field to use, if one is not specified. If you would like to explore a using a different Galois Field implementation, you can dynamically set the backend GF for a given word-size (w). - +The new galois.[ch] exports the following functions to be used by applications +for dynamically setting the backend GF: + +1.) galois_change_technique + + Function signature: + + void galois_change_technique(gf_t *gf, int w); + + This function will take a pointer to a Galois field structure and set it as the + current backend for all operations in GF(2^w). Note that you must specify 'w' + here, since the internal GF structure is mostly opaque to Jerasure. Be sure to + change the technique with the correct structure and word-size. + + There are a few ways to get a pointer to a gf_t structure: via GF-Complete or + using the helper functions provided by Jerasure: galois_init_field and + galois_init_composite_field. + + GF-Complete exposes create_gf_from_argv, gf_init_easy and gf_init_hard. See + the GF-Complete documentation for more detail on how to use those functions. + You can definitely create more complicated fields with the GF-Complete + initialization functions, but the two helper functions provided by Jerasure + (galois_init_field and galois_init_composite_field) can be used to create most + of the supported Galois Fields. + +2.) galois_init_field + + Function signature: + + gf_t* galois_init_field(int w, + int mult_type, + int region_type, + int divide_type, + uint64_t prim_poly, + int arg1, + int arg2); + + This is a helper function that will initialize a Galois field. See the GF-Complete + documentation for more info on what the arguments mean. Here is a brief description + of the arguments: + + mult_type can be any *one* of the following: + + GF_MULT_DEFAULT + GF_MULT_SHIFT + GF_MULT_CARRY_FREE + GF_MULT_GROUP + GF_MULT_BYTWO_p + GF_MULT_BYTWO_b + GF_MULT_TABLE + GF_MULT_LOG_TABLE + GF_MULT_LOG_ZERO + GF_MULT_LOG_ZERO_EXT + GF_MULT_SPLIT_TABLE + + region_type can be a combination of the following (some combinations will not + be valid): + + GF_REGION_DEFAULT + GF_REGION_DOUBLE_TABLE + GF_REGION_QUAD_TABLE + GF_REGION_LAZY + GF_REGION_SSE + GF_REGION_NOSSE + GF_REGION_ALTMAP + GF_REGION_CAUCHY + + divide_type can be one of the following: + + GF_DIVIDE_DEFAULT + GF_DIVIDE_MATRIX + GF_DIVIDE_EUCLID + + prim_poly is the field-defining primitive polynomial + + arg1 and arg2 are special arguments usually used for defining SPLIT and GROUP + operations + +3.) galois_init_composite_field + + Function signature: + + gf_t* galois_init_composite_field(int w, + int region_type, + int divide_type, + int degree, + gf_t* base_gf); + + This is a helper function designed to make creating Composite fields easier. All you + need to do is hand it w, region mult type, divide type, degree and a pointer to a base + field. Note that the base_gf must have degree w/degree in order for this to work. + For example, if we create a GF using: + + galois_init_composite_field(32, GF_REGION_DEFAULT, GF_DIVIDE_DEFAULT, 2, base_gf); + + Then base_gf must have w=16. + +For more information on how to change the backing fields for Jerasure, please refer to + + 1.) Examples/reed_sol_test_gf.c: Runs basic tests for Reed-Solomon given args + for a backing GF (uses create_gf_from_argv to get gf_t pointer) + + 2.) Examples/reed_sol_time_gf.c: Runs more thorough timing and validation tests + for a backing GF (uses create_gf_from_argv to get gf_t pointer) + + 3.) Examples/reed_sol_hard_time_gf.c: Runs more thorough timing and validation tests + for a backing GF (uses galois_init_field and galois_init_composite_field to get gf_t + pointer) + diff --git a/galois.c b/galois.c index f2ef8da..b05de8e 100644 --- a/galois.c +++ b/galois.c @@ -44,8 +44,9 @@ #include "galois.h" -#define MAX_GF_INSTANCES 128 +#define MAX_GF_INSTANCES 64 gf_t *gfp_array[MAX_GF_INSTANCES] = { 0 }; +int gfp_is_composite[MAX_GF_INSTANCES] = { 0 }; gf_t *galois_get_field_ptr(int w) { @@ -56,23 +57,29 @@ gf_t *galois_get_field_ptr(int w) return NULL; } -int galois_init_base_field(int w, - int mult_type, - int region_type, - int divide_type, - uint64_t prim_poly, - int arg1, - int arg2) +gf_t* galois_init_field(int w, + int mult_type, + int region_type, + int divide_type, + uint64_t prim_poly, + int arg1, + int arg2) { int scratch_size; void *scratch_memory; - int ret; + gf_t *gfp; if (w <= 0 || w > 32) { fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w); exit(1); } + gfp = (gf_t *) malloc(sizeof(gf_t)); + if (!gfp) { + fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w); + exit(1); + } + scratch_size = gf_scratch_size(w, mult_type, region_type, divide_type, arg1, arg2); if (!scratch_size) { fprintf(stderr, "ERROR -- cannot get scratch size for base field w=%d\n", w); @@ -85,20 +92,7 @@ int galois_init_base_field(int w, exit(1); } - /* - * Properly free up the old field - */ - if (gfp_array[w] != NULL) { - gf_free(gfp_array[w], 0); - } - - gfp_array[w] = (gf_t*)malloc(sizeof(gf_t)); - if (gfp_array[w] == NULL) { - fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w); - exit(1); - } - - if(!gf_init_hard(gfp_array[w], + if(!gf_init_hard(gfp, w, mult_type, region_type, @@ -112,27 +106,33 @@ int galois_init_base_field(int w, fprintf(stderr, "ERROR -- cannot init default Galois field for w=%d\n", w); exit(1); } - return 0; + + gfp_is_composite[w] = 0; + return gfp; } -int galois_init_composite_field(int w, +gf_t* galois_init_composite_field(int w, int region_type, int divide_type, - uint64_t prim_poly, - int arg1, - int arg2, + int degree, gf_t* base_gf) { int scratch_size; void *scratch_memory; - int ret; + gf_t *gfp; if (w <= 0 || w > 32) { fprintf(stderr, "ERROR -- cannot init composite field for w=%d\n", w); exit(1); } + + gfp = (gf_t *) malloc(sizeof(gf_t)); + if (!gfp) { + fprintf(stderr, "ERROR -- cannot allocate memory for Galois field w=%d\n", w); + exit(1); + } - scratch_size = gf_scratch_size(w, GF_MULT_COMPOSITE, region_type, divide_type, arg1, arg2); + scratch_size = gf_scratch_size(w, GF_MULT_COMPOSITE, region_type, divide_type, degree, 0); if (!scratch_size) { fprintf(stderr, "ERROR -- cannot get scratch size for composite field w=%d\n", w); exit(1); @@ -144,34 +144,22 @@ int galois_init_composite_field(int w, exit(1); } - /* - * Properly free up the old field - */ - if (gfp_array[w] != NULL) { - gf_free(gfp_array[w], 1); - } - - gfp_array[w] = (gf_t*)malloc(sizeof(gf_t)); - if (gfp_array[w] == NULL) { - fprintf(stderr, "ERROR -- cannot allocate memory for composite field w=%d\n", w); - exit(1); - } - - if(!gf_init_hard(gfp_array[w], + if(!gf_init_hard(gfp, w, GF_MULT_COMPOSITE, region_type, divide_type, - prim_poly, - arg1, - arg2, + 0, + degree, + 0, base_gf, scratch_memory)) { fprintf(stderr, "ERROR -- cannot init default composite field for w=%d\n", w); exit(1); } - return 0; + gfp_is_composite[w] = 1; + return gfp; } static void galois_init_default_field(int w) @@ -237,6 +225,10 @@ void galois_change_technique(gf_t *gf, int w) exit(1); } + if (gfp_array[w] != NULL) { + gf_free(gfp_array[w], gfp_is_composite[w]); + } + gfp_array[w] = gf; } diff --git a/galois.h b/galois.h index 3479d9f..9e0dc25 100644 --- a/galois.h +++ b/galois.h @@ -80,20 +80,18 @@ void galois_w32_region_multiply(char *region, /* Region to multiply */ Otherwise region is overwritten */ int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */ -int galois_init_base_field(int w, - int mult_type, - int region_type, - int divide_type, - uint64_t prim_poly, - int arg1, - int arg2); +gf_t* galois_init_field(int w, + int mult_type, + int region_type, + int divide_type, + uint64_t prim_poly, + int arg1, + int arg2); -int galois_init_composite_field(int w, +gf_t* galois_init_composite_field(int w, int region_type, int divide_type, - uint64_t prim_poly, - int arg1, - int arg2, + int degree, gf_t* base_gf); gf_t * galois_get_field_ptr(int w);