static int
do_rehydrate(struct rain_encoding_s *enc, uint8_t **data,
uint8_t **coding, int *erasures)
{
/* Creating coding matrix or bitmatrix */
int *bit_matrix=NULL, *matrix=NULL;
if (enc->algo == JALG_liberation)
bit_matrix = liber8tion_coding_bitmatrix(enc->k);
else if (enc->algo == JALG_crs) {
matrix = cauchy_good_general_coding_matrix(enc->k, enc->m, enc->w);
bit_matrix = jerasure_matrix_to_bitmatrix(enc->k, enc->m, enc->w, matrix);
}
/* Choose proper decoding method */
jerasure_schedule_decode_lazy(enc->k, enc->m, enc->w,
bit_matrix, erasures, (char**)data, (char**)coding,
enc->block_size, enc->packet_size, 1);
/* Freeing previously allocated memory */
if (bit_matrix)
free(bit_matrix);
if (matrix)
free(matrix);
return 1;
}
int repair_decode_MSR_product_matrix(char **input, size_t input_size, char *output, size_t output_size, int to_device_ID, int* helpers, struct coding_info *info)
{
int i,counter;
int d = info->req.d;
int k = info->req.k;
int w = info->req.w;
int subpacket_size = input_size;
char** coding_ptrs = malloc(sizeof(void*)*(d-k+1));
int *repair_matrix = malloc(sizeof(int)*d*d);
int *repair_matrix_inv = malloc(sizeof(int)*d*d);
int *combination_matrix = malloc(sizeof(int)*(d-k+1)*d);
for(i=0,counter=0;i<d&&helpers[i]>=0;i++,counter++)
memcpy(repair_matrix+d*i,info->matrix+d*helpers[i],sizeof(int)*d);
if(counter<d){
printf("Insufficient number of helpers.\n");
return(-1);
}
jerasure_invert_matrix(repair_matrix,repair_matrix_inv,d,w);
memset(combination_matrix,0,sizeof(int)*(d-k+1)*d);
for(i=0;i<k-1;i++){
*(combination_matrix+(d*i)+i) = to_device_ID;
*(combination_matrix+(d*i)+i+k-1) = 1;
}
for(i=k-1;i<d-k+1;i++)
*(combination_matrix+(d*i)+i+k-1) = 1;
int *coding_matrix = jerasure_matrix_multiply(combination_matrix,repair_matrix_inv,d-k+1,d,d,d,w);
for(i=0; i<d-k+1 ;i++)
coding_ptrs[i] = output + i*subpacket_size;
int *coding_bitmatrix = jerasure_matrix_to_bitmatrix(d,d-k+1,w,coding_matrix);
jerasure_bitmatrix_encode(d,d-k+1,w,coding_bitmatrix,(char**)input,coding_ptrs,subpacket_size,ALIGNMENT);
free(coding_ptrs);
free(coding_matrix);
free(repair_matrix);
free(repair_matrix_inv);
free(combination_matrix);
free(coding_bitmatrix);
return(1);
}
int main(int argc, char **argv)
{
int n;
int i, no, w;
int *bitmatrix;
if (argc != 3) usage(NULL);
if (sscanf(argv[1], "0x%x", &n) == 0) {
if (sscanf(argv[1], "%d", &n) == 0) usage("Bad n");
}
if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (w == 31) {
if (n & 0x80000000L) usage("Bad n/w combination (n not between 0 and 2^w-1)\n");
} else if (w < 31) {
if (n >= (1 << w)) usage("Bad n/w combination (n not between 0 and 2^w-1)\n");
}
bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n);
printf("<HTML><title>cauchy_01 %u %d</title>\n", w, n);
printf("<HTML><h3>cauchy_01 %u %d</h3>\n", w, n);
printf("<pre>\n");
if (w == 32) {
printf("Converted the value 0x%x to the following bitmatrix:\n\n", n);
} else {
printf("Converted the value %d (0x%x) to the following bitmatrix:\n\n", n, n);
}
jerasure_print_bitmatrix(bitmatrix, w, w, w);
printf("\n");
no = 0;
for (i = 0; i < w*w; i++) no += bitmatrix[i];
if (no != cauchy_n_ones(n, w)) {
fprintf(stderr, "Jerasure error: # ones in the bitmatrix (%d) doesn't match cauchy_n_ones() (%d).\n",
no, cauchy_n_ones(n, w));
exit(1);
}
printf("# Ones: %d\n", cauchy_n_ones(n, w));
return 0;
}
int main(int argc, char **argv)
{
int n, i, no, w;
int *bitmatrix;
if (argc != 3) usage(NULL);
if (sscanf(argv[1], "%d", &n) == 0 || n <= 0) usage("Bad n");
if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w");
if (w < 30 && n >= (1 << w)) usage("n is too big");
bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n);
no = 0;
for (i = 0; i < w*w; i++) no += bitmatrix[i];
printf("# Ones: %d\n", cauchy_n_ones(n, w));
printf("\n");
printf("Bitmatrix has %d ones\n", no);
printf("\n");
jerasure_print_bitmatrix(bitmatrix, w, w, w);
return 0;
}
int
rain_encode_noalloc (struct rain_encoding_s *encoding, uint8_t **data,
uint8_t **parity)
{
assert(encoding != NULL);
// Prepare the jerasure structures
int *bit_matrix=NULL, *matrix=NULL, **schedule=NULL;
if (encoding->algo == JALG_liberation) {
matrix = NULL;
bit_matrix = liber8tion_coding_bitmatrix(encoding->k);
schedule = jerasure_smart_bitmatrix_to_schedule(encoding->k, encoding->m,
encoding->w, bit_matrix);
}
else if (encoding->algo == JALG_crs) {
matrix = cauchy_good_general_coding_matrix(
encoding->k, encoding->m, encoding->w);
bit_matrix = jerasure_matrix_to_bitmatrix(
encoding->k, encoding->m, encoding->w, matrix);
schedule = jerasure_smart_bitmatrix_to_schedule(
encoding->k, encoding->m, encoding->w, bit_matrix);
}
// Compute now ... damned, no return code to check
jerasure_schedule_encode(encoding->k, encoding->m, encoding->w, schedule,
(char**) data, (char**) parity,
encoding->block_size, encoding->packet_size);
if (schedule)
jerasure_free_schedule(schedule);
if (bit_matrix)
free(bit_matrix);
if (matrix)
free(matrix);
return 1;
}
int decoder(char *argv,char *dup_argv, int ssd_no[] ) {
FILE *fp; // File pointer
/* Jerasure arguments */
char **data;
char **coding;
int *erasures;
int *erased;
int *matrix;
int *bitmatrix;
/* Parameters */
int k = 5;
int m = 3;
int w = 4;
int packetsize = 1024;
int buffersize = k*w*1024*sizeof(int);
int d=0;
int i, j; // loop control variables
int blocksize; // size of individual files
int origsize; // size of file before padding
int total; // used to write data, not padding to file
struct stat status; // used to find size of individual files
int numerased; // number of erased files
/* Used to recreate file names */
char *temp;
char *cs1, *cs2;
char *dup_cs1, *dup_cs2;
char *fname;
int md;
char *curdir;
/* Used to time decoding */
struct timeval t1, t2, t3, t4;
struct timezone tz;
double tsec;
double totalsec;
signal(SIGQUIT, ctrl_bs_handler);
matrix = NULL;
bitmatrix = NULL;
totalsec = 0.0;
/* Start timing */
gettimeofday(&t1, &tz);
/* Error checking parameters */
//if (argc != 2) {
// fprintf(stderr, "usage: inputfile\n");
// exit(0);
//}
curdir = (char *)malloc(sizeof(char)*100);
getcwd(curdir, 100);
printf("argv=%s dup_argv=%s\n",argv,dup_argv);
/* Begin recreation of file names */
cs1 = (char*)malloc(sizeof(char)*strlen(argv));
cs2 = strrchr(argv, '/');
if (cs2 != NULL) {
cs2++;
strcpy(cs1, cs2);
}
else {
strcpy(cs1, argv);
}
cs2 = strchr(cs1, '.');
if (cs2 != NULL) {
*cs2 = '\0';
}
cs2 = (char*)malloc(sizeof(char)*strlen(argv));
fname = strchr(argv, '.');
if(fname != NULL)
strcpy(cs2, fname);
else
*cs2='\0';
//
if(dup_argv != NULL){
dup_cs1 = (char*)malloc(sizeof(char)*strlen(dup_argv));
dup_cs2 = strrchr(dup_argv, '/');
if (dup_cs2 != NULL) {
dup_cs2++;
strcpy(dup_cs1, dup_cs2);
}
else {
strcpy(dup_cs1, dup_argv);
}
dup_cs2 = strchr(dup_cs1, '.');
if (dup_cs2 != NULL) {
*dup_cs2 = '\0';
}
dup_cs2 = (char*)malloc(sizeof(char)*strlen(dup_argv));
fname = strchr(dup_argv, '.');
if(fname!=NULL)
strcpy(dup_cs2, fname);
else
*dup_cs2='\0';
}
//
printf("CRS_decoder.c: cs1=%s cs2=%s\n",cs1,cs2);
printf("CRS_decoder.c: dup_cs1=%s dup_cs2=%s\n",dup_cs1,dup_cs2);
fname = (char *)malloc(sizeof(char*)*(100+strlen(argv)+10));
/* Read in parameters from metadata file */
sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1);
fp = fopen(fname, "rb");
temp = (char *)malloc(sizeof(char)*(strlen(argv)+10));
fscanf(fp, "%s", temp);
if (fscanf(fp, "%d", &origsize) != 1) {
fprintf(stderr, "Original size is not valid\n");
exit(0);
}
fclose(fp);
if(origsize%buffersize == 0)
readins = origsize/buffersize;
else
readins = (origsize/buffersize)+1;
/* Allocate memory */
erased = (int *)malloc(sizeof(int)*(k+m));
for (i = 0; i < k+m; i++)
erased[i] = 0;
erasures = (int *)malloc(sizeof(int)*(k+m));
data = (char **)malloc(sizeof(char *)*k);
coding = (char **)malloc(sizeof(char *)*m);
if (buffersize != origsize) {
for (i = 0; i < k; i++) {
data[i] = (char *)malloc(sizeof(char)*(buffersize/k));
}
for (i = 0; i < m; i++) {
coding[i] = (char *)malloc(sizeof(char)*(buffersize/k));
}
blocksize = buffersize/k;
}
sprintf(temp, "%d", k);
md = strlen(temp);
gettimeofday(&t3, &tz);
/* Create coding matrix or bitmatrix */
matrix = cauchy_good_general_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
gettimeofday(&t4, &tz);
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
/* Begin decoding process */
total = 0;
n = 1;
while (n <= readins) {
numerased = 0;
/* Open files, check for erasures, read in data/coding */
for (i = 1; i <= k; i++) {
sprintf(fname, "/media/newSSD%d/server/%s_k%0*d%s", ssd_no[i-1], cs1, md, i, cs2);
fp = fopen(fname, "rb");
if (fp == NULL) {
erased[i-1] = 1;
erasures[numerased] = i-1;
numerased++;
//printf("%s failed\n", fname);
}
else {
if (buffersize == origsize) {
stat(fname, &status);
blocksize = status.st_size;
data[i-1] = (char *)malloc(sizeof(char)*blocksize);
fread(data[i-1], sizeof(char), blocksize, fp);
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
fread(data[i-1], sizeof(char), buffersize/k, fp);
}
fclose(fp);
d++;
}
}
for (i = 1; i <= m ; i++) {
sprintf(fname, "/media/newSSD%d/server/%s_m%0*d%s", ssd_no[k+i-1], cs1, md, i, cs2);
fp = fopen(fname, "rb");
if (fp == NULL || d > k ) {
erased[k+(i-1)] = 1;
erasures[numerased] = k+i-1;
numerased++;
//printf("%s failed\n", fname);
if(fp != NULL) fclose(fp);
}
else {
if (buffersize == origsize) {
stat(fname, &status);
blocksize = status.st_size;
coding[i-1] = (char *)malloc(sizeof(char)*blocksize);
fread(coding[i-1], sizeof(char), blocksize, fp);
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
fread(coding[i-1], sizeof(char), blocksize, fp);
}
fclose(fp);
d++;
}
}
/* Finish allocating data/coding if needed */
if (n == 1) {
for (i = 0; i < numerased; i++) {
if (erasures[i] < k) {
data[erasures[i]] = (char *)malloc(sizeof(char)*blocksize);
}
else {
coding[erasures[i]-k] = (char *)malloc(sizeof(char)*blocksize);
}
}
}
erasures[numerased] = -1;
gettimeofday(&t3, &tz);
/* Choose proper decoding method */
i = jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1);
gettimeofday(&t4, &tz);
/* Exit if decoding was unsuccessful */
if (i == -1) {
fprintf(stderr, "Unsuccessful!\n");
exit(0);
}
/* Create decoded file */
if(dup_argv == NULL)
sprintf(fname, "%s/Coding/%s_decoded%s", curdir, cs1, cs2);
else
sprintf(fname, "%s/Coding/%s_decoded%s", curdir, dup_cs1, dup_cs2);
if (n == 1) {
fp = fopen(fname, "wb");
}
else {
fp = fopen(fname, "ab");
}
for (i = 0; i < k; i++) {
if (total+blocksize <= origsize) {
fwrite(data[i], sizeof(char), blocksize, fp);
total+= blocksize;
}
else {
for (j = 0; j < blocksize; j++) {
if (total < origsize) {
fprintf(fp, "%c", data[i][j]);
total++;
}
else {
break;
}
}
}
}
n++;
fclose(fp);
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
}
/* Free allocated memory */
if(dup_argv != NULL)
{
free(dup_cs1);
free(dup_cs2);
}
free(cs1);
free(fname);
free(data);
free(coding);
free(erasures);
free(erased);
/* Stop timing and print time */
gettimeofday(&t2, &tz);
tsec = 0;
tsec += t2.tv_usec;
tsec -= t1.tv_usec;
tsec /= 1000000.0;
tsec += t2.tv_sec;
tsec -= t1.tv_sec;
printf("Decoding (MB/sec): %0.10f\n", (origsize/1024/1024)/totalsec);
printf("De_Total (MB/sec): %0.10f\n\n", (origsize/1024/1024)/tsec);
}
int main (int argc, char **argv) {
FILE *fp, *fp2; // file pointers
char *block; // padding file
int size, newsize; // size of file and temp size
struct stat status; // finding file size
enum Coding_Technique tech; // coding technique (parameter)
int k, m, w, packetsize; // parameters
int buffersize; // paramter
int i; // loop control variables
int blocksize; // size of k+m files
int total;
int extra;
/* Jerasure Arguments */
char **data;
char **coding;
int *matrix;
int *bitmatrix;
int **schedule;
/* Creation of file name variables */
char temp[5];
char *s1, *s2, *extension;
char *fname;
int md;
char *curdir;
/* Timing variables */
struct timing t1, t2, t3, t4;
double tsec;
double totalsec;
struct timing start;
/* Find buffersize */
int up, down;
signal(SIGQUIT, ctrl_bs_handler);
/* Start timing */
timing_set(&t1);
totalsec = 0.0;
matrix = NULL;
bitmatrix = NULL;
schedule = NULL;
/* Error check Arguments*/
if (argc != 8) {
fprintf(stderr, "usage: inputfile k m coding_technique w packetsize buffersize\n");
fprintf(stderr, "\nChoose one of the following coding techniques: \nreed_sol_van, \nreed_sol_r6_op, \ncauchy_orig, \ncauchy_good, \nliberation, \nblaum_roth, \nliber8tion");
fprintf(stderr, "\n\nPacketsize is ignored for the reed_sol's");
fprintf(stderr, "\nBuffersize of 0 means the buffersize is chosen automatically.\n");
fprintf(stderr, "\nIf you just want to test speed, use an inputfile of \"-number\" where number is the size of the fake file you want to test.\n\n");
exit(0);
}
/* Conversion of parameters and error checking */
if (sscanf(argv[2], "%d", &k) == 0 || k <= 0) {
fprintf(stderr, "Invalid value for k\n");
exit(0);
}
if (sscanf(argv[3], "%d", &m) == 0 || m < 0) {
fprintf(stderr, "Invalid value for m\n");
exit(0);
}
if (sscanf(argv[5],"%d", &w) == 0 || w <= 0) {
fprintf(stderr, "Invalid value for w.\n");
exit(0);
}
if (argc == 6) {
packetsize = 0;
}
else {
if (sscanf(argv[6], "%d", &packetsize) == 0 || packetsize < 0) {
fprintf(stderr, "Invalid value for packetsize.\n");
exit(0);
}
}
if (argc != 8) {
buffersize = 0;
}
else {
if (sscanf(argv[7], "%d", &buffersize) == 0 || buffersize < 0) {
fprintf(stderr, "Invalid value for buffersize\n");
exit(0);
}
}
/* Determine proper buffersize by finding the closest valid buffersize to the input value */
if (buffersize != 0) {
if (packetsize != 0 && buffersize%(sizeof(long)*w*k*packetsize) != 0) {
up = buffersize;
down = buffersize;
while (up%(sizeof(long)*w*k*packetsize) != 0 && (down%(sizeof(long)*w*k*packetsize) != 0)) {
up++;
if (down == 0) {
down--;
}
}
if (up%(sizeof(long)*w*k*packetsize) == 0) {
buffersize = up;
}
else {
if (down != 0) {
buffersize = down;
}
}
}
else if (packetsize == 0 && buffersize%(sizeof(long)*w*k) != 0) {
up = buffersize;
down = buffersize;
while (up%(sizeof(long)*w*k) != 0 && down%(sizeof(long)*w*k) != 0) {
up++;
down--;
}
if (up%(sizeof(long)*w*k) == 0) {
buffersize = up;
}
else {
buffersize = down;
}
}
}
/* Setting of coding technique and error checking */
if (strcmp(argv[4], "no_coding") == 0) {
tech = No_Coding;
}
else if (strcmp(argv[4], "reed_sol_van") == 0) {
tech = Reed_Sol_Van;
if (w != 8 && w != 16 && w != 32) {
fprintf(stderr, "w must be one of {8, 16, 32}\n");
exit(0);
}
}
else if (strcmp(argv[4], "reed_sol_r6_op") == 0) {
if (m != 2) {
fprintf(stderr, "m must be equal to 2\n");
exit(0);
}
if (w != 8 && w != 16 && w != 32) {
fprintf(stderr, "w must be one of {8, 16, 32}\n");
exit(0);
}
tech = Reed_Sol_R6_Op;
}
else if (strcmp(argv[4], "cauchy_orig") == 0) {
tech = Cauchy_Orig;
if (packetsize == 0) {
fprintf(stderr, "Must include packetsize.\n");
exit(0);
}
}
else if (strcmp(argv[4], "cauchy_good") == 0) {
tech = Cauchy_Good;
if (packetsize == 0) {
fprintf(stderr, "Must include packetsize.\n");
exit(0);
}
}
else if (strcmp(argv[4], "liberation") == 0) {
if (k > w) {
fprintf(stderr, "k must be less than or equal to w\n");
exit(0);
}
if (w <= 2 || !(w%2) || !is_prime(w)) {
fprintf(stderr, "w must be greater than two and w must be prime\n");
exit(0);
}
if (packetsize == 0) {
fprintf(stderr, "Must include packetsize.\n");
exit(0);
}
if ((packetsize%(sizeof(long))) != 0) {
fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n");
exit(0);
}
tech = Liberation;
}
else if (strcmp(argv[4], "blaum_roth") == 0) {
if (k > w) {
fprintf(stderr, "k must be less than or equal to w\n");
exit(0);
}
if (w <= 2 || !((w+1)%2) || !is_prime(w+1)) {
fprintf(stderr, "w must be greater than two and w+1 must be prime\n");
exit(0);
}
if (packetsize == 0) {
fprintf(stderr, "Must include packetsize.\n");
exit(0);
}
if ((packetsize%(sizeof(long))) != 0) {
fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n");
exit(0);
}
tech = Blaum_Roth;
}
else if (strcmp(argv[4], "liber8tion") == 0) {
if (packetsize == 0) {
fprintf(stderr, "Must include packetsize\n");
exit(0);
}
if (w != 8) {
fprintf(stderr, "w must equal 8\n");
exit(0);
}
if (m != 2) {
fprintf(stderr, "m must equal 2\n");
exit(0);
}
if (k > w) {
fprintf(stderr, "k must be less than or equal to w\n");
exit(0);
}
tech = Liber8tion;
}
else {
fprintf(stderr, "Not a valid coding technique. Choose one of the following: reed_sol_van, reed_sol_r6_op, cauchy_orig, cauchy_good, liberation, blaum_roth, liber8tion, no_coding\n");
exit(0);
}
/* Set global variable method for signal handler */
method = tech;
/* Get current working directory for construction of file names */
curdir = (char*)malloc(sizeof(char)*1000);
if (! getcwd(curdir, 1000)) {
curdir[0] = 0;
}
if (argv[1][0] != '-') {
/* Open file and error check */
fp = fopen(argv[1], "rb");
if (fp == NULL) {
fprintf(stderr, "Unable to open file.\n");
exit(0);
}
/* Create Coding directory */
i = mkdir("Coding", S_IRWXU);
if (i == -1 && errno != EEXIST) {
fprintf(stderr, "Unable to create Coding directory.\n");
exit(0);
}
/* Determine original size of file */
stat(argv[1], &status);
size = status.st_size;
} else {
if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) {
fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n");
exit(1);
}
fp = NULL;
MOA_Seed(time(0));
}
newsize = size;
/* Find new size by determining next closest multiple */
if (packetsize != 0) {
if (size%(k*w*packetsize*sizeof(long)) != 0) {
while (newsize%(k*w*packetsize*sizeof(long)) != 0)
newsize++;
}
}
else {
if (size%(k*w*sizeof(long)) != 0) {
while (newsize%(k*w*sizeof(long)) != 0)
newsize++;
}
}
if (buffersize != 0) {
while (newsize%buffersize != 0) {
newsize++;
}
}
/* Determine size of k+m files */
blocksize = newsize/k;
/* Allow for buffersize and determine number of read-ins */
if (size > buffersize && buffersize != 0) {
if (newsize%buffersize != 0) {
readins = newsize/buffersize;
}
else {
readins = newsize/buffersize;
}
block = (char *)malloc(sizeof(char)*buffersize);
blocksize = buffersize/k;
}
else {
readins = 1;
buffersize = size;
block = (char *)malloc(sizeof(char)*newsize);
}
/* Break inputfile name into the filename and extension */
s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+20));
s2 = strrchr(argv[1], '/');
if (s2 != NULL) {
s2++;
strcpy(s1, s2);
}
else {
strcpy(s1, argv[1]);
}
s2 = strchr(s1, '.');
if (s2 != NULL) {
extension = strdup(s2);
*s2 = '\0';
} else {
extension = strdup("");
}
/* Allocate for full file name */
fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+20));
sprintf(temp, "%d", k);
md = strlen(temp);
/* Allocate data and coding */
data = (char **)malloc(sizeof(char*)*k);
coding = (char **)malloc(sizeof(char*)*m);
for (i = 0; i < m; i++) {
coding[i] = (char *)malloc(sizeof(char)*blocksize);
if (coding[i] == NULL) { perror("malloc"); exit(1); }
}
/* Create coding matrix or bitmatrix and schedule */
timing_set(&t3);
switch(tech) {
case No_Coding:
break;
case Reed_Sol_Van:
matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
break;
case Reed_Sol_R6_Op:
break;
case Cauchy_Orig:
matrix = cauchy_original_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
break;
case Cauchy_Good:
matrix = cauchy_good_general_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
break;
case Liberation:
bitmatrix = liberation_coding_bitmatrix(k, w);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
break;
case Blaum_Roth:
bitmatrix = blaum_roth_coding_bitmatrix(k, w);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
break;
case Liber8tion:
bitmatrix = liber8tion_coding_bitmatrix(k);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
break;
case RDP:
case EVENODD:
assert(0);
}
timing_set(&start);
timing_set(&t4);
totalsec += timing_delta(&t3, &t4);
/* Read in data until finished */
n = 1;
total = 0;
while (n <= readins) {
/* Check if padding is needed, if so, add appropriate
number of zeros */
if (total < size && total+buffersize <= size) {
total += jfread(block, sizeof(char), buffersize, fp);
}
else if (total < size && total+buffersize > size) {
extra = jfread(block, sizeof(char), buffersize, fp);
for (i = extra; i < buffersize; i++) {
block[i] = '0';
}
}
else if (total == size) {
for (i = 0; i < buffersize; i++) {
block[i] = '0';
}
}
/* Set pointers to point to file data */
for (i = 0; i < k; i++) {
data[i] = block+(i*blocksize);
}
timing_set(&t3);
/* Encode according to coding method */
switch(tech) {
case No_Coding:
break;
case Reed_Sol_Van:
jerasure_matrix_encode(k, m, w, matrix, data, coding, blocksize);
break;
case Reed_Sol_R6_Op:
reed_sol_r6_encode(k, w, data, coding, blocksize);
break;
case Cauchy_Orig:
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
break;
case Cauchy_Good:
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
break;
case Liberation:
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
break;
case Blaum_Roth:
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
break;
case Liber8tion:
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
break;
case RDP:
case EVENODD:
assert(0);
}
timing_set(&t4);
/* Write data and encoded data to k+m files */
for (i = 1; i <= k; i++) {
if (fp == NULL) {
bzero(data[i-1], blocksize);
} else {
sprintf(fname, "%s/Coding/%s_k%0*d%s", curdir, s1, md, i, extension);
if (n == 1) {
fp2 = fopen(fname, "wb");
}
else {
fp2 = fopen(fname, "ab");
}
fwrite(data[i-1], sizeof(char), blocksize, fp2);
fclose(fp2);
}
}
for (i = 1; i <= m; i++) {
if (fp == NULL) {
bzero(data[i-1], blocksize);
} else {
sprintf(fname, "%s/Coding/%s_m%0*d%s", curdir, s1, md, i, extension);
if (n == 1) {
fp2 = fopen(fname, "wb");
}
else {
fp2 = fopen(fname, "ab");
}
fwrite(coding[i-1], sizeof(char), blocksize, fp2);
fclose(fp2);
}
}
n++;
/* Calculate encoding time */
totalsec += timing_delta(&t3, &t4);
}
/* Create metadata file */
if (fp != NULL) {
sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1);
fp2 = fopen(fname, "wb");
fprintf(fp2, "%s\n", argv[1]);
fprintf(fp2, "%d\n", size);
fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize);
fprintf(fp2, "%s\n", argv[4]);
fprintf(fp2, "%d\n", tech);
fprintf(fp2, "%d\n", readins);
fclose(fp2);
}
/* Free allocated memory */
free(s1);
free(fname);
free(block);
free(curdir);
/* Calculate rate in MB/sec and print */
timing_set(&t2);
tsec = timing_delta(&t1, &t2);
printf("Encoding (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/totalsec);
printf("En_Total (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/tsec);
return 0;
}
int main (int argc, char **argv) {
FILE *fp, *fp2; // file pointers
char *memblock; // reading in file
char *block; // padding file
int size, newsize; // size of file and temp size
struct stat status; // finding file size
enum Coding_Technique tech; // coding technique (parameter)
int k, m, w, packetsize; // parameters
int buffersize; // paramter
int i, j; // loop control variables
int blocksize; // size of k+m files
int total;
int extra;
/* Jerasure Arguments */
char **data;
char **coding;
int *matrix;
int *bitmatrix;
int **schedule;
int *erasure;
int *erased;
/* Creation of file name variables */
char temp[5];
char *s1, *s2;
char *fname;
int md;
char *curdir;
/* Timing variables */
struct timeval t1, t2, t3, t4;
struct timezone tz;
double tsec;
double totalsec;
struct timeval start, stop;
/* Find buffersize */
int up, down;
/* Modifications */
int _size, sizeCorrect = 0, numFiles = 0, codingFactor, testValue;
/* Start timing */
gettimeofday(&t1, &tz);
totalsec = 0.0;
matrix = NULL;
bitmatrix = NULL;
schedule = NULL;
printf("\nFilename: %s\n", argv[1]);
printf("Coding Technique: cauchy_good\n");
printf("Input desired size of file segments (in MB): ");
scanf("%d", &_size);
printf("Input codingFactor: ");
scanf("%d", &codingFactor);
// printf("Input packetsize: ");
// scanf("%d", &packetsize);
// printf("Input buffersize: ");
// scanf("%d", &buffersize);
w = 16;
packetsize = 32;
buffersize = 50000;
//_size = _size * 1024 * 1024;
/* Set global variable method for signal handler */
tech = Cauchy_Good;
method = tech;
/* Get current working directory for construction of file names */
curdir = (char*)malloc(sizeof(char)*1000);
getcwd(curdir, 1000);
if (argv[1][0] != '-') {
/* Open file and error check */
fp = fopen(argv[1], "rb");
if (fp == NULL) {
fprintf(stderr, "Unable to open file.\n");
exit(0);
}
/* Create Coding directory */
i = mkdir("Coding", S_IRWXU);
if (i == -1 && errno != EEXIST) {
fprintf(stderr, "Unable to create Coding directory.\n");
exit(0);
}
/* Determine original size of file */
stat(argv[1], &status);
size = status.st_size;
} else {
if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) {
fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n");
exit(1);
}
fp = NULL;
srand48(time(0));
}
k = size/_size;
m = (codingFactor-1) * k;
/* Determine proper buffersize by finding the closest valid buffersize to the input value */
if (buffersize != 0) {
if (packetsize != 0 && buffersize%(sizeof(int)*w*k*packetsize) != 0) {
up = buffersize;
down = buffersize;
while (up%(sizeof(int)*w*k*packetsize) != 0 && (down%(sizeof(int)*w*k*packetsize) != 0)) {
up++;
if (down == 0) {
down--;
}
}
if (up%(sizeof(int)*w*k*packetsize) == 0) {
buffersize = up;
}
else {
if (down != 0) {
buffersize = down;
}
}
}
else if (packetsize == 0 && buffersize%(sizeof(int)*w*k) != 0) {
up = buffersize;
down = buffersize;
while (up%(sizeof(int)*w*k) != 0 && down%(sizeof(int)*w*k) != 0) {
testValue = up%(sizeof(int)*w*k);
printf("up%% = %d\n", testValue);
up++;
down--;
}
if (up%(sizeof(int)*w*k) == 0) {
buffersize = up;
}
else {
buffersize = down;
}
}
}
newsize = size;
/* Find new size by determining next closest multiple */
if (packetsize != 0) {
if (size%(k*w*packetsize*sizeof(int)) != 0) {
while (newsize%(k*w*packetsize*sizeof(int)) != 0)
newsize++;
}
}
else {
if (size%(k*w*sizeof(int)) != 0) {
while (newsize%(k*w*sizeof(int)) != 0)
newsize++;
}
}
if (buffersize != 0) {
while (newsize%buffersize != 0) {
newsize++;
}
}
/* Determine size of k+m files */
blocksize = newsize/k;
printf("\n");
printf("Size of file segments: %d \tbytes\n", blocksize);
printf("Size of buffer: %d \tbytes\n", buffersize);
printf("Size of file: %d \tbytes\n", size);
printf("Size of word: %d \t\tbytes\n\n", w);
printf("Number of data files: %d\n", k);
printf("Number of coding files: %d\n", m);
/* Allow for buffersize and determine number of read-ins */
if (size > buffersize && buffersize != 0) {
if (newsize%buffersize != 0) {
readins = newsize/buffersize;
}
else {
readins = newsize/buffersize;
}
block = (char *)malloc(sizeof(char)*buffersize);
blocksize = buffersize/k;
}
else {
readins = 1;
buffersize = size;
block = (char *)malloc(sizeof(char)*newsize);
}
/* Break inputfile name into the filename and extension */
s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+10));
s2 = strrchr(argv[1], '/');
if (s2 != NULL) {
s2++;
strcpy(s1, s2);
}
else {
strcpy(s1, argv[1]);
}
s2 = strchr(s1, '.');
if (s2 != NULL) {
*s2 = '\0';
}
fname = strchr(argv[1], '.');
s2 = (char*)malloc(sizeof(char)*(strlen(argv[1])+5));
if (fname != NULL) {
strcpy(s2, fname);
}
/* Allocate for full file name */
fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+10));
sprintf(temp, "%d", k+m);
md = strlen(temp);
/* Allocate data and coding */
data = (char **)malloc(sizeof(char*)*k);
coding = (char **)malloc(sizeof(char*)*m);
for (i = 0; i < m; i++) {
coding[i] = (char *)malloc(sizeof(char)*blocksize);
}
/* Create coding matrix or bitmatrix and schedule */
gettimeofday(&t3, &tz);
matrix = cauchy_good_general_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
gettimeofday(&start, &tz);
gettimeofday(&t4, &tz);
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
/* Read in data until finished */
n = 1;
total = 0;
printf("\nABOUT TO ENCODE SCHEDULE\n");
while (n <= readins) {
/* Check if padding is needed, if so, add appropriate
number of zeros */
if (total < size && total+buffersize <= size) {
total += jfread(block, sizeof(char), buffersize, fp);
}
else if (total < size && total+buffersize > size) {
extra = jfread(block, sizeof(char), buffersize, fp);
for (i = extra; i < buffersize; i++) {
block[i] = '0';
}
}
else if (total == size) {
for (i = 0; i < buffersize; i++) {
block[i] = '0';
}
}
/* Set pointers to point to file data */
for (i = 0; i < k; i++) {
data[i] = block+(i*blocksize);
}
gettimeofday(&t3, &tz);
jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize);
gettimeofday(&t4, &tz);
/* Write data and encoded data to k+m files */
for (i = 1; i <= k; i++) {
if (fp == NULL) {
bzero(data[i-1], blocksize);
} else {
sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2);
if (n == 1) {
fp2 = fopen(fname, "wb");
}
else {
fp2 = fopen(fname, "ab");
}
fwrite(data[i-1], sizeof(char), blocksize, fp2);
fclose(fp2);
}
}
for (i; i <= k+m; i++) {
if (fp == NULL) {
bzero(data[i-1-k], blocksize);
} else {
sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2);
if (n == 1) {
fp2 = fopen(fname, "wb");
}
else {
fp2 = fopen(fname, "ab");
}
fwrite(coding[i-1-k], sizeof(char), blocksize, fp2);
fclose(fp2);
}
}
n++;
/* Calculate encoding time */
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
}
/* Create metadata file */
if (fp != NULL) {
sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1);
fp2 = fopen(fname, "wb");
fprintf(fp2, "%s\n", argv[1]);
fprintf(fp2, "%d\n", size);
fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize);
fprintf(fp2, "%s\n", argv[4]);
fprintf(fp2, "%d\n", tech);
fprintf(fp2, "%d\n", readins);
fclose(fp2);
}
/* Free allocated memory */
free(s2);
free(s1);
free(fname);
free(block);
free(curdir);
/* Calculate rate in MB/sec and print */
gettimeofday(&t2, &tz);
tsec = 0.0;
tsec += t2.tv_usec;
tsec -= t1.tv_usec;
tsec /= 1000000.0;
tsec += t2.tv_sec;
tsec -= t1.tv_sec;
printf("Encoding (MB/sec): %0.10f\n", (size/1024/1024)/totalsec);
printf("En_Total (MB/sec): %0.10f\n", (size/1024/1024)/tsec);
}
int main (int argc, char **argv) {
FILE *fp; // File pointer
char *dummy;
int err;
/* Jerasure arguments */
char **data;
char **coding;
int *erasures;
int *erased;
int *matrix;
int *bitmatrix;
/* Parameters */
int k, m, w, packetsize, buffersize;
enum Coding_Technique tech;
char *c_tech;
int i, j; // loop control variables
int blocksize; // size of individual files
int origsize; // size of file before padding
int total; // used to write data, not padding to file
struct stat status; // used to find size of individual files
int numerased; // number of erased files
/* Used to recreate file names */
char *temp;
char *cs1, *cs2, *extension;
char *fname;
int md;
char *curdir;
/* Used to time decoding */
struct timeval t1, t2, t3, t4;
struct timezone tz;
double tsec;
double totalsec;
signal(SIGQUIT, ctrl_bs_handler);
matrix = NULL;
bitmatrix = NULL;
totalsec = 0.0;
blocksize = -1;
/* Start timing */
gettimeofday(&t1, &tz);
/* Error checking parameters */
if (argc != 2) {
fprintf(stderr, "usage: inputfile\n");
exit(0);
}
curdir = (char *)malloc(sizeof(char)*100);
dummy = getcwd(curdir, 100);
/* Begin recreation of file names */
cs1 = (char*)malloc(sizeof(char)*strlen(argv[1]));
cs2 = strrchr(argv[1], '/');
if (cs2 != NULL) {
cs2++;
strcpy(cs1, cs2);
}
else {
strcpy(cs1, argv[1]);
}
cs2 = strchr(cs1, '.');
if (cs2 != NULL) {
extension = strdup(cs2);
*cs2 = '\0';
} else {
extension = strdup("");
}
fname = (char *)malloc(sizeof(char*)*(100+strlen(argv[1])+10));
/* Read in parameters from metadata file */
sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1);
fp = fopen(fname, "rb");
if (fp == NULL) {
fprintf(stderr, "Error: no metadata file %s\n", fname);
exit(1);
}
temp = (char *)malloc(sizeof(char)*(strlen(argv[1])+10));
err = fscanf(fp, "%s", temp);
if (fscanf(fp, "%d", &origsize) != 1) {
fprintf(stderr, "Original size is not valid\n");
exit(0);
}
if (fscanf(fp, "%d %d %d %d %d", &k, &m, &w, &packetsize, &buffersize) != 5) {
fprintf(stderr, "Parameters are not correct\n");
exit(0);
}
c_tech = (char *)malloc(sizeof(char)*(strlen(argv[1])+10));
err = fscanf(fp, "%s", c_tech);
err = fscanf(fp, "%d", &tech);
method = tech;
err = fscanf(fp, "%d", &readins);
fclose(fp);
/* Allocate memory */
erased = (int *)malloc(sizeof(int)*(k+m));
for (i = 0; i < k+m; i++)
erased[i] = 0;
erasures = (int *)malloc(sizeof(int)*(k+m));
data = (char **)malloc(sizeof(char *)*k);
coding = (char **)malloc(sizeof(char *)*m);
if (buffersize != origsize) {
for (i = 0; i < k; i++) {
data[i] = (char *)malloc(sizeof(char)*(buffersize/k));
}
for (i = 0; i < m; i++) {
coding[i] = (char *)malloc(sizeof(char)*(buffersize/k));
}
blocksize = buffersize/k;
}
sprintf(temp, "%d", k);
md = strlen(temp);
gettimeofday(&t3, &tz);
/* Create coding matrix or bitmatrix */
switch(tech) {
case No_Coding:
case RDP:
case EVENODD:
break;
case Reed_Sol_Van:
matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
break;
case Reed_Sol_R6_Op:
matrix = reed_sol_r6_coding_matrix(k, w);
break;
case Cauchy_Orig:
matrix = cauchy_original_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
break;
case Cauchy_Good:
matrix = cauchy_good_general_coding_matrix(k, m, w);
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
break;
case Liberation:
bitmatrix = liberation_coding_bitmatrix(k, w);
break;
case Blaum_Roth:
bitmatrix = blaum_roth_coding_bitmatrix(k, w);
break;
case Liber8tion:
bitmatrix = liber8tion_coding_bitmatrix(k);
break;
default:
fprintf(stderr, "unsupported coding technique used\n");
break;
}
gettimeofday(&t4, &tz);
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
/* Begin decoding process */
total = 0;
n = 1;
while (n <= readins) {
numerased = 0;
/* Open files, check for erasures, read in data/coding */
for (i = 1; i <= k; i++) {
sprintf(fname, "%s/Coding/%s_k%0*d%s", curdir, cs1, md, i, extension);
fp = fopen(fname, "rb");
if (fp == NULL) {
erased[i-1] = 1;
erasures[numerased] = i-1;
numerased++;
//printf("%s failed\n", fname);
}
else {
if (buffersize == origsize) {
stat(fname, &status);
blocksize = status.st_size;
data[i-1] = (char *)malloc(sizeof(char)*blocksize);
err = fread(data[i-1], sizeof(char), blocksize, fp);
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
err = fread(data[i-1], sizeof(char), buffersize/k, fp);
}
fclose(fp);
}
}
for (i = 1; i <= m; i++) {
sprintf(fname, "%s/Coding/%s_m%0*d%s", curdir, cs1, md, i, extension);
fp = fopen(fname, "rb");
if (fp == NULL) {
erased[k+(i-1)] = 1;
erasures[numerased] = k+i-1;
numerased++;
//printf("%s failed\n", fname);
}
else {
if (buffersize == origsize) {
stat(fname, &status);
blocksize = status.st_size;
coding[i-1] = (char *)malloc(sizeof(char)*blocksize);
err = fread(coding[i-1], sizeof(char), blocksize, fp);
}
else {
fseek(fp, blocksize*(n-1), SEEK_SET);
err = fread(coding[i-1], sizeof(char), blocksize, fp);
}
fclose(fp);
}
}
/* Finish allocating data/coding if needed */
if (n == 1) {
for (i = 0; i < numerased; i++) {
if (erasures[i] < k) {
data[erasures[i]] = (char *)malloc(sizeof(char)*blocksize);
}
else {
coding[erasures[i]-k] = (char *)malloc(sizeof(char)*blocksize);
}
}
}
erasures[numerased] = -1;
gettimeofday(&t3, &tz);
/* Choose proper decoding method */
if (tech == Reed_Sol_Van || tech == Reed_Sol_R6_Op) {
i = jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, blocksize);
}
else if (tech == Cauchy_Orig || tech == Cauchy_Good || tech == Liberation || tech == Blaum_Roth || tech == Liber8tion) {
i = jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1);
}
else {
fprintf(stderr, "Not a valid coding technique.\n");
exit(0);
}
gettimeofday(&t4, &tz);
/* Exit if decoding was unsuccessful */
if (i == -1) {
fprintf(stderr, "Unsuccessful!\n");
exit(0);
}
/* Create decoded file */
sprintf(fname, "%s/Coding/%s_decoded%s", curdir, cs1, extension);
if (n == 1) {
fp = fopen(fname, "wb");
}
else {
fp = fopen(fname, "ab");
}
for (i = 0; i < k; i++) {
if (total+blocksize <= origsize) {
fwrite(data[i], sizeof(char), blocksize, fp);
total+= blocksize;
}
else {
for (j = 0; j < blocksize; j++) {
if (total < origsize) {
fprintf(fp, "%c", data[i][j]);
total++;
}
else {
break;
}
}
}
}
n++;
fclose(fp);
tsec = 0.0;
tsec += t4.tv_usec;
tsec -= t3.tv_usec;
tsec /= 1000000.0;
tsec += t4.tv_sec;
tsec -= t3.tv_sec;
totalsec += tsec;
}
/* Free allocated memory */
free(cs1);
free(extension);
free(fname);
free(data);
free(coding);
free(erasures);
free(erased);
/* Stop timing and print time */
gettimeofday(&t2, &tz);
tsec = 0;
tsec += t2.tv_usec;
tsec -= t1.tv_usec;
tsec /= 1000000.0;
tsec += t2.tv_sec;
tsec -= t1.tv_sec;
printf("Decoding (MB/sec): %0.10f\n", (origsize/1024/1024)/totalsec);
printf("De_Total (MB/sec): %0.10f\n\n", (origsize/1024/1024)/tsec);
return(0);
}
int decode_MSR_product_matrix_no_output(char **input, size_t input_size, int* erasures, struct coding_info *info)
{
clock_t clk, tclk;
int i, j,c1,c2,tdone,inv;
int n = info->req.n;
int d = info->req.d;
int k = info->req.k;
int w = info->req.w;
int subpacket_size = input_size/(d-k+1);
int num_of_long = subpacket_size/sizeof(long);
long *src_pos,*des_pos;
int *vector_A=NULL;
char **ptrs;
int **decode_schedule=NULL;
int *erased = NULL;
int *bitmatrix_temp = malloc(sizeof(int)*(k-1)*(k-1)*w*w*4);
char *data_transformed = malloc(input_size*k); // this is the buffer for tranformed data, i.e., matrix M. The output is the systematic part of
// codingmatrix*M, and
// we will regenerate the erased data from M using the encoding matrix, which will be written to *output.
int *pseudo_erasures = malloc(sizeof(int)*(n*2)); // in order to recompute M, we view it as a systematic MDS code,
// sometimes (n+k,k), sometimes (n+d-k+1,d-k+1), thus we over allocate
char **M_ptrs = malloc(sizeof(void*)*d*(d-k+1)); // this is the pointer matrix to elements in M.
char **data_ptrs = malloc(sizeof(void*)*n);
char **coding_ptrs = malloc(sizeof(void*)*n);
int* remaining = malloc(sizeof(int)*k); // not erased devices
char *buffer1 = malloc(subpacket_size*k*(k-1)*2); // need subpacketsize*k>=(max(4,k-1)+k-1)*sizeof(int), thus put factor of 2 to guarentee it
int *buffer1_int = (int*)buffer1; // alternative pointer for buffer1
char *buffer2 = malloc(subpacket_size*k*(k-1));
if(data_transformed==NULL||pseudo_erasures==NULL||data_ptrs==NULL
||coding_ptrs==NULL||buffer1==NULL||buffer2==NULL||M_ptrs==NULL||remaining==NULL){
printf("Can not allocate memory\n");
jerasure_free_schedule(decode_schedule);
if(data_ptrs!=NULL)free(data_ptrs);
if(coding_ptrs!=NULL)free(coding_ptrs);
if(pseudo_erasures!=NULL)free(pseudo_erasures);
if(data_transformed!=NULL)free(data_transformed);
if(M_ptrs!=NULL)free(M_ptrs);
if(buffer1!=NULL)free(buffer1);
if(buffer2!=NULL)free(buffer2);
if(remaining!=NULL)free(remaining);
return(-1);
}
//set up pointers for matrix M
// first k-1 rows, only have S1
for(i=0,c2=0;i<k-1;i++){
c1 = i*(d-k+1);
for(j=i;j<k-1;j++,c2++)
M_ptrs[c1+j] = data_transformed + subpacket_size*c2;
for(j=k-1;j<d-k+1;j++)
M_ptrs[c1+j] = NULL;
for(j=0;j<i;j++)
M_ptrs[c1+j] = M_ptrs[j*(d-k+1)+i]; // symmetric matrix, thus there is (i,j) and (j,i) point to the same pointer.
}
// next k-1 rows
for(i=0;i<k-1;i++){
c1 = (k-1+i)*(d-k+1);
for(j=i;j<d-k+1;j++,c2++)
M_ptrs[c1+j] = data_transformed + subpacket_size*c2;
for(j=0;j<i;j++)
M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i];
}
// next 1 and (d-2k+1) rows, may not exist
if(d>2*k-2)
{
c1 = (2*k-2)*(d-k+1);
for(j=k-1;j<d-k+1;j++,c2++)
M_ptrs[c1+j] = data_transformed + subpacket_size*c2;
for(j=0;j<i;j++)
M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+k-1];
for(i=2*k-1;i<d;i++){
c1 = i*(d-k+1);
for(j=0;j<k;j++)
M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i-k+1];
for(j=k;j<d-k+1;j++)
M_ptrs[c1+j] = NULL;
}
}
// first decode the last d-2k+1 columns of T and Z: view it as an (n+k,k) MDS code. Note strictly speaking this might
// not be a real (n+k,k) MDS code, but it hardly matters.
// before decoding operation, prepare for the pseudoerasure location array
if(d>2*k-2){ // only when d>2k-2, T and Z exist
for(i=0;i<k;i++)
pseudo_erasures[i] = i;
for(i=0;i<n;i++){
if(erasures[i]==-1){
pseudo_erasures[i+k] = -1;
break;
}
pseudo_erasures[i+k] = erasures[i] + k;
}
// make the decoding schedule: we can save the trouble of manually generate this schedule, at the expense of
// more computation
decode_schedule = jerasure_generate_decoding_schedule(k, n, w, info->subbitmatrix_array[0], pseudo_erasures, 1);
for(i=0;i<d-2*k+1;i++){
for(j=0;j<k;j++)
data_ptrs[j] = M_ptrs[i+k+(d-k+1)*(k-1+j)];
for(j=0;j<n;j++)
coding_ptrs[j] = input[j]+subpacket_size*(k+i);
ptrs = set_up_ptrs_for_scheduled_decoding(k, n, pseudo_erasures, data_ptrs,coding_ptrs);
if (ptrs == NULL){
printf("Can not allocate memory\n");
goto complete;
}
// assume packetsize = ALIGNMENT
for (tdone = 0; tdone < subpacket_size; tdone += ALIGNMENT*w) {
jerasure_do_scheduled_operations(ptrs, decode_schedule, ALIGNMENT);
for (c1 = 0; c1 < k+n; c1++) ptrs[c1] += (ALIGNMENT*w);
}
free(ptrs);
}
//next decode the first column of T and Z: we view this as an (n+d-k+1,d-k+1) erasure codes
// first setup the pseudo erasure location vector
for(i=0;i<k;i++)
pseudo_erasures[i] = i; // in the first columne of the Z matrix, the last d-2k+1 elements are known, but the first k elements are not
for(i=0;i<n;i++)
{
if(erasures[i]==-1){
pseudo_erasures[i+k] = -1;
break;
}
pseudo_erasures[i+k] = erasures[i]+d-k+1;
}
for(j=0;j<d-k+1;j++)
data_ptrs[j] = M_ptrs[(d-k+1)*(k-1+j)+k-1];
for(j=0;j<n;j++)
coding_ptrs[j] = input[j]+subpacket_size*(k-1);
jerasure_schedule_decode_lazy(d-k+1,n,w,
info->subbitmatrix_array[1],pseudo_erasures,data_ptrs,
coding_ptrs,subpacket_size,ALIGNMENT,0);
}
//clk = clock();
// now this is the hard part: to decode S1 and S2. The algorithm used here is slightly different from that in the paper:
// instead of right multiply \Phi_{DC}', we only right multiply the sub-matrix of \Phi_{DC}' without of the last row
// setting up remaining device array to facilitate decoding
erased = jerasure_erasures_to_erased(k, n-k, erasures);
for(i=0,c1=0;i<n&&c1<k;i++){
if(erased[i]==0){
remaining[c1] = i;
c1++;
}
}
//compute C_{DC}-\Delta_{DC}*T'
for(i=0;i<k-1;i++){ // has k-1 columns
for(j=0;j<d-2*k+2;j++)
data_ptrs[j] = M_ptrs[(2*k-2+j)*(d-k+1)+i];
for(j=0;j<k;j++)
coding_ptrs[j] = buffer1+(j*(k-1)+i)*subpacket_size;
for(j=0;j<k;j++){
jerasure_bitmatrix_dotprod(d-2*k+2, w, info->subbitmatrix_array[2]+remaining[j]*(d-2*k+2)*w*w, NULL, j+d-2*k+2,
data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
src_pos = (long*)(input[remaining[j]]+i*subpacket_size);
des_pos = (long*)(coding_ptrs[j]);
for(c2=0;c2<num_of_long;c2++)
des_pos[c2] ^= src_pos[c2];
}
}
// right multiply \Phi_{DC}': this will be P.
// result is in buffer2
for(j=0;j<k;j++){ //j-th row
for(i=0;i<k-1;i++)
data_ptrs[i] = buffer1+(j*(k-1)+i)*subpacket_size;
for(i=0;i<k-1;i++)
coding_ptrs[i] = buffer2 +(j*(k-1)+i)*subpacket_size;
for(i=0;i<k-1;i++){
jerasure_bitmatrix_dotprod(k-1, w, info->subbitmatrix_array[3]+remaining[i]*(k-1)*w*w, NULL, i+k-1,
data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
}
}
// now solve for the off-diagonal terms
for(i=0;i<k-1;i++){
for(j=i+1;j<k-1;j++){
// solve for S1 tilde off-diagonal
// here we directly use the fact that Lambda = [0 1 2 3 ....];
int** temp_schedule;
inv = galois_single_divide(1,remaining[i]^remaining[j],w);
buffer1_int[0] = buffer1_int[1] = inv;
data_ptrs[0] = buffer2+(i*(k-1)+j)*subpacket_size;
data_ptrs[1] = buffer2+(j*(k-1)+i)*subpacket_size;
coding_ptrs[0] = M_ptrs[i*(d-k+1)+j];
jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp);
temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp);
jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT);
if(temp_schedule!=NULL){
jerasure_free_schedule(temp_schedule);
temp_schedule = NULL;
}
// solve for S2 tilde off-diagonal
buffer1_int[0] = galois_single_multiply(remaining[j],inv,w);
buffer1_int[1] = galois_single_multiply(remaining[i],inv,w);
coding_ptrs[0] = M_ptrs[(i+k-1)*(d-k+1)+j];
jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp);
temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp);
jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT);
if(temp_schedule!=NULL){
jerasure_free_schedule(temp_schedule);
temp_schedule = NULL;
}
}
}
//tclk = clock()-clk;
//printf("~S1 and ~S2 off-diagonal decoded %.3e clocks \n", (double)tclk);
//clk = clock();
// compute the A vector: A*\Phi_{DC1} = \Phi_{DC2}, note this is always possible because \Phi_{DC1} is alway full rank by construction
// we first reuse buffer1 here to form \Phi_{DC1} matrix and then the compute its inverse
for(i=0;i<k-1;i++){
memcpy(buffer1_int+(k-1)*(k-1+i),(void*)(info->matrix+remaining[i]*d+k-1),(k-1)*sizeof(int));
}
jerasure_invert_matrix(buffer1_int+(k-1)*(k-1),buffer1_int,k-1,w);
vector_A = jerasure_matrix_multiply(info->matrix+remaining[k-1]*d+k-1,buffer1_int,1,k-1,k-1,k-1,w);
if(vector_A==NULL)
goto complete;
for(i=0;i<k-1;i++){
buffer1_int[i+(k-1)*(k-1)] = galois_single_multiply(vector_A[i],remaining[k-1],w);
buffer1_int[i+k*(k-1)] = vector_A[i];
}
for(i=0;i<k-1;i++){
memset(buffer1_int+(k-1)*(k+1),0,sizeof(int)*2*(k-1));
buffer1_int[(k-1)*(k+1)+i] = remaining[i];
buffer1_int[(k-1)*(k+2)+i] = 1;
pseudo_erasures[0] = i;
pseudo_erasures[1] = k-1+i;
pseudo_erasures[2] = -1;
for(j=0;j<2*k-2;j++)
data_ptrs[j] = M_ptrs[i+j*(d-k+1)];
coding_ptrs[0] = buffer2+((k-1)*(k-1)+i)*subpacket_size;
coding_ptrs[1] = buffer2+(i*(k-1)+i)*subpacket_size;
jerasure_matrix_to_bitmatrix_noallocate(2*k-2,2,w,buffer1_int+(k-1)*(k-1),bitmatrix_temp);
jerasure_schedule_decode_lazy(2*k-2,2,w,bitmatrix_temp,pseudo_erasures,data_ptrs,coding_ptrs,subpacket_size,ALIGNMENT,0);
}
//tclk = clock()-clk;
//printf("~S1 and ~S2 decoded %.3e clocks \n", (double)tclk);
// now we have both \tilde{S_1} and \tilde{S_2} in M_ptrs, need to recover S1 and S2 from them
// this is done by multiply \tilde{S_1} left and right by inv(\Phi_{DC1}).
// right-multiply for S1
int* bitmatrix_inv = jerasure_matrix_to_bitmatrix(k-1,k-1,w,buffer1_int);
int** inv_schedule = jerasure_smart_bitmatrix_to_schedule(k-1, k-1, w, bitmatrix_inv);
// right-multiply for S1
for(i=0;i<k-1;i++){
for(j=0;j<k-1;j++)
data_ptrs[j] = M_ptrs[i*(d-k+1)+j];
for(j=0;j<k-1;j++)
coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size;
jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
}
// left-multiply for S1
for(j=0;j<k-1;j++){
for(i=0;i<k-1;i++)
data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size;
for(i=0;i<k-1;i++)
coding_ptrs[i] = M_ptrs[i*(d-k+1)+j];
jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
}
// right-multiply for S2
for(i=0;i<k-1;i++){
for(j=0;j<k-1;j++)
data_ptrs[j] = M_ptrs[(i+k-1)*(d-k+1)+j];
for(j=0;j<k-1;j++)
coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size;
jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
}
// left-multiply for S2
for(j=0;j<k-1;j++){
for(i=0;i<k-1;i++)
data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size;
//for(i=j;i<k-1;i++)
for(i=0;i<k-1;i++)
coding_ptrs[i] = M_ptrs[(i+k-1)*(d-k+1)+j];
jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT);
}
// having S1,S2,T, now can also fill the first k-1 column of the output
for(i=0;i<k-1;i++){
for(j=0;j<d;j++)
data_ptrs[j] = M_ptrs[j*(d-k+1)+i];
for(j=0;j<n;j++)
coding_ptrs[j] = input[j]+i*subpacket_size;
for(j=0;j<n;j++){
if(erased[j]==1)
jerasure_bitmatrix_encode(d,1,w,info->bitmatrix+(j*d*w*w),data_ptrs,coding_ptrs+j,subpacket_size,ALIGNMENT);
}
}
// clean up
complete:
if(decode_schedule)
jerasure_free_schedule(decode_schedule);
if(inv_schedule)
jerasure_free_schedule(inv_schedule);
free(data_ptrs);
free(coding_ptrs);
if(erased)free(erased);
free(pseudo_erasures);
free(data_transformed);
free(M_ptrs);
free(buffer1);
free(buffer2);
free(remaining);
free(bitmatrix_temp);
if(vector_A!=NULL)free(vector_A);
return(1);
}