static int do_rehydrate(struct rain_encoding_s *enc, uint8_t **data, uint8_t **coding, int *erasures) { /* Creating coding matrix or bitmatrix */ int *bit_matrix=NULL, *matrix=NULL; if (enc->algo == JALG_liberation) bit_matrix = liber8tion_coding_bitmatrix(enc->k); else if (enc->algo == JALG_crs) { matrix = cauchy_good_general_coding_matrix(enc->k, enc->m, enc->w); bit_matrix = jerasure_matrix_to_bitmatrix(enc->k, enc->m, enc->w, matrix); } /* Choose proper decoding method */ jerasure_schedule_decode_lazy(enc->k, enc->m, enc->w, bit_matrix, erasures, (char**)data, (char**)coding, enc->block_size, enc->packet_size, 1); /* Freeing previously allocated memory */ if (bit_matrix) free(bit_matrix); if (matrix) free(matrix); return 1; }
int repair_decode_MSR_product_matrix(char **input, size_t input_size, char *output, size_t output_size, int to_device_ID, int* helpers, struct coding_info *info) { int i,counter; int d = info->req.d; int k = info->req.k; int w = info->req.w; int subpacket_size = input_size; char** coding_ptrs = malloc(sizeof(void*)*(d-k+1)); int *repair_matrix = malloc(sizeof(int)*d*d); int *repair_matrix_inv = malloc(sizeof(int)*d*d); int *combination_matrix = malloc(sizeof(int)*(d-k+1)*d); for(i=0,counter=0;i<d&&helpers[i]>=0;i++,counter++) memcpy(repair_matrix+d*i,info->matrix+d*helpers[i],sizeof(int)*d); if(counter<d){ printf("Insufficient number of helpers.\n"); return(-1); } jerasure_invert_matrix(repair_matrix,repair_matrix_inv,d,w); memset(combination_matrix,0,sizeof(int)*(d-k+1)*d); for(i=0;i<k-1;i++){ *(combination_matrix+(d*i)+i) = to_device_ID; *(combination_matrix+(d*i)+i+k-1) = 1; } for(i=k-1;i<d-k+1;i++) *(combination_matrix+(d*i)+i+k-1) = 1; int *coding_matrix = jerasure_matrix_multiply(combination_matrix,repair_matrix_inv,d-k+1,d,d,d,w); for(i=0; i<d-k+1 ;i++) coding_ptrs[i] = output + i*subpacket_size; int *coding_bitmatrix = jerasure_matrix_to_bitmatrix(d,d-k+1,w,coding_matrix); jerasure_bitmatrix_encode(d,d-k+1,w,coding_bitmatrix,(char**)input,coding_ptrs,subpacket_size,ALIGNMENT); free(coding_ptrs); free(coding_matrix); free(repair_matrix); free(repair_matrix_inv); free(combination_matrix); free(coding_bitmatrix); return(1); }
int main(int argc, char **argv) { int n; int i, no, w; int *bitmatrix; if (argc != 3) usage(NULL); if (sscanf(argv[1], "0x%x", &n) == 0) { if (sscanf(argv[1], "%d", &n) == 0) usage("Bad n"); } if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (w == 31) { if (n & 0x80000000L) usage("Bad n/w combination (n not between 0 and 2^w-1)\n"); } else if (w < 31) { if (n >= (1 << w)) usage("Bad n/w combination (n not between 0 and 2^w-1)\n"); } bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n); printf("<HTML><title>cauchy_01 %u %d</title>\n", w, n); printf("<HTML><h3>cauchy_01 %u %d</h3>\n", w, n); printf("<pre>\n"); if (w == 32) { printf("Converted the value 0x%x to the following bitmatrix:\n\n", n); } else { printf("Converted the value %d (0x%x) to the following bitmatrix:\n\n", n, n); } jerasure_print_bitmatrix(bitmatrix, w, w, w); printf("\n"); no = 0; for (i = 0; i < w*w; i++) no += bitmatrix[i]; if (no != cauchy_n_ones(n, w)) { fprintf(stderr, "Jerasure error: # ones in the bitmatrix (%d) doesn't match cauchy_n_ones() (%d).\n", no, cauchy_n_ones(n, w)); exit(1); } printf("# Ones: %d\n", cauchy_n_ones(n, w)); return 0; }
int main(int argc, char **argv) { int n, i, no, w; int *bitmatrix; if (argc != 3) usage(NULL); if (sscanf(argv[1], "%d", &n) == 0 || n <= 0) usage("Bad n"); if (sscanf(argv[2], "%d", &w) == 0 || w <= 0 || w > 32) usage("Bad w"); if (w < 30 && n >= (1 << w)) usage("n is too big"); bitmatrix = jerasure_matrix_to_bitmatrix(1, 1, w, &n); no = 0; for (i = 0; i < w*w; i++) no += bitmatrix[i]; printf("# Ones: %d\n", cauchy_n_ones(n, w)); printf("\n"); printf("Bitmatrix has %d ones\n", no); printf("\n"); jerasure_print_bitmatrix(bitmatrix, w, w, w); return 0; }
int rain_encode_noalloc (struct rain_encoding_s *encoding, uint8_t **data, uint8_t **parity) { assert(encoding != NULL); // Prepare the jerasure structures int *bit_matrix=NULL, *matrix=NULL, **schedule=NULL; if (encoding->algo == JALG_liberation) { matrix = NULL; bit_matrix = liber8tion_coding_bitmatrix(encoding->k); schedule = jerasure_smart_bitmatrix_to_schedule(encoding->k, encoding->m, encoding->w, bit_matrix); } else if (encoding->algo == JALG_crs) { matrix = cauchy_good_general_coding_matrix( encoding->k, encoding->m, encoding->w); bit_matrix = jerasure_matrix_to_bitmatrix( encoding->k, encoding->m, encoding->w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule( encoding->k, encoding->m, encoding->w, bit_matrix); } // Compute now ... damned, no return code to check jerasure_schedule_encode(encoding->k, encoding->m, encoding->w, schedule, (char**) data, (char**) parity, encoding->block_size, encoding->packet_size); if (schedule) jerasure_free_schedule(schedule); if (bit_matrix) free(bit_matrix); if (matrix) free(matrix); return 1; }
int decoder(char *argv,char *dup_argv, int ssd_no[] ) { FILE *fp; // File pointer /* Jerasure arguments */ char **data; char **coding; int *erasures; int *erased; int *matrix; int *bitmatrix; /* Parameters */ int k = 5; int m = 3; int w = 4; int packetsize = 1024; int buffersize = k*w*1024*sizeof(int); int d=0; int i, j; // loop control variables int blocksize; // size of individual files int origsize; // size of file before padding int total; // used to write data, not padding to file struct stat status; // used to find size of individual files int numerased; // number of erased files /* Used to recreate file names */ char *temp; char *cs1, *cs2; char *dup_cs1, *dup_cs2; char *fname; int md; char *curdir; /* Used to time decoding */ struct timeval t1, t2, t3, t4; struct timezone tz; double tsec; double totalsec; signal(SIGQUIT, ctrl_bs_handler); matrix = NULL; bitmatrix = NULL; totalsec = 0.0; /* Start timing */ gettimeofday(&t1, &tz); /* Error checking parameters */ //if (argc != 2) { // fprintf(stderr, "usage: inputfile\n"); // exit(0); //} curdir = (char *)malloc(sizeof(char)*100); getcwd(curdir, 100); printf("argv=%s dup_argv=%s\n",argv,dup_argv); /* Begin recreation of file names */ cs1 = (char*)malloc(sizeof(char)*strlen(argv)); cs2 = strrchr(argv, '/'); if (cs2 != NULL) { cs2++; strcpy(cs1, cs2); } else { strcpy(cs1, argv); } cs2 = strchr(cs1, '.'); if (cs2 != NULL) { *cs2 = '\0'; } cs2 = (char*)malloc(sizeof(char)*strlen(argv)); fname = strchr(argv, '.'); if(fname != NULL) strcpy(cs2, fname); else *cs2='\0'; // if(dup_argv != NULL){ dup_cs1 = (char*)malloc(sizeof(char)*strlen(dup_argv)); dup_cs2 = strrchr(dup_argv, '/'); if (dup_cs2 != NULL) { dup_cs2++; strcpy(dup_cs1, dup_cs2); } else { strcpy(dup_cs1, dup_argv); } dup_cs2 = strchr(dup_cs1, '.'); if (dup_cs2 != NULL) { *dup_cs2 = '\0'; } dup_cs2 = (char*)malloc(sizeof(char)*strlen(dup_argv)); fname = strchr(dup_argv, '.'); if(fname!=NULL) strcpy(dup_cs2, fname); else *dup_cs2='\0'; } // printf("CRS_decoder.c: cs1=%s cs2=%s\n",cs1,cs2); printf("CRS_decoder.c: dup_cs1=%s dup_cs2=%s\n",dup_cs1,dup_cs2); fname = (char *)malloc(sizeof(char*)*(100+strlen(argv)+10)); /* Read in parameters from metadata file */ sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1); fp = fopen(fname, "rb"); temp = (char *)malloc(sizeof(char)*(strlen(argv)+10)); fscanf(fp, "%s", temp); if (fscanf(fp, "%d", &origsize) != 1) { fprintf(stderr, "Original size is not valid\n"); exit(0); } fclose(fp); if(origsize%buffersize == 0) readins = origsize/buffersize; else readins = (origsize/buffersize)+1; /* Allocate memory */ erased = (int *)malloc(sizeof(int)*(k+m)); for (i = 0; i < k+m; i++) erased[i] = 0; erasures = (int *)malloc(sizeof(int)*(k+m)); data = (char **)malloc(sizeof(char *)*k); coding = (char **)malloc(sizeof(char *)*m); if (buffersize != origsize) { for (i = 0; i < k; i++) { data[i] = (char *)malloc(sizeof(char)*(buffersize/k)); } for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*(buffersize/k)); } blocksize = buffersize/k; } sprintf(temp, "%d", k); md = strlen(temp); gettimeofday(&t3, &tz); /* Create coding matrix or bitmatrix */ matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); gettimeofday(&t4, &tz); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; /* Begin decoding process */ total = 0; n = 1; while (n <= readins) { numerased = 0; /* Open files, check for erasures, read in data/coding */ for (i = 1; i <= k; i++) { sprintf(fname, "/media/newSSD%d/server/%s_k%0*d%s", ssd_no[i-1], cs1, md, i, cs2); fp = fopen(fname, "rb"); if (fp == NULL) { erased[i-1] = 1; erasures[numerased] = i-1; numerased++; //printf("%s failed\n", fname); } else { if (buffersize == origsize) { stat(fname, &status); blocksize = status.st_size; data[i-1] = (char *)malloc(sizeof(char)*blocksize); fread(data[i-1], sizeof(char), blocksize, fp); } else { fseek(fp, blocksize*(n-1), SEEK_SET); fread(data[i-1], sizeof(char), buffersize/k, fp); } fclose(fp); d++; } } for (i = 1; i <= m ; i++) { sprintf(fname, "/media/newSSD%d/server/%s_m%0*d%s", ssd_no[k+i-1], cs1, md, i, cs2); fp = fopen(fname, "rb"); if (fp == NULL || d > k ) { erased[k+(i-1)] = 1; erasures[numerased] = k+i-1; numerased++; //printf("%s failed\n", fname); if(fp != NULL) fclose(fp); } else { if (buffersize == origsize) { stat(fname, &status); blocksize = status.st_size; coding[i-1] = (char *)malloc(sizeof(char)*blocksize); fread(coding[i-1], sizeof(char), blocksize, fp); } else { fseek(fp, blocksize*(n-1), SEEK_SET); fread(coding[i-1], sizeof(char), blocksize, fp); } fclose(fp); d++; } } /* Finish allocating data/coding if needed */ if (n == 1) { for (i = 0; i < numerased; i++) { if (erasures[i] < k) { data[erasures[i]] = (char *)malloc(sizeof(char)*blocksize); } else { coding[erasures[i]-k] = (char *)malloc(sizeof(char)*blocksize); } } } erasures[numerased] = -1; gettimeofday(&t3, &tz); /* Choose proper decoding method */ i = jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1); gettimeofday(&t4, &tz); /* Exit if decoding was unsuccessful */ if (i == -1) { fprintf(stderr, "Unsuccessful!\n"); exit(0); } /* Create decoded file */ if(dup_argv == NULL) sprintf(fname, "%s/Coding/%s_decoded%s", curdir, cs1, cs2); else sprintf(fname, "%s/Coding/%s_decoded%s", curdir, dup_cs1, dup_cs2); if (n == 1) { fp = fopen(fname, "wb"); } else { fp = fopen(fname, "ab"); } for (i = 0; i < k; i++) { if (total+blocksize <= origsize) { fwrite(data[i], sizeof(char), blocksize, fp); total+= blocksize; } else { for (j = 0; j < blocksize; j++) { if (total < origsize) { fprintf(fp, "%c", data[i][j]); total++; } else { break; } } } } n++; fclose(fp); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; } /* Free allocated memory */ if(dup_argv != NULL) { free(dup_cs1); free(dup_cs2); } free(cs1); free(fname); free(data); free(coding); free(erasures); free(erased); /* Stop timing and print time */ gettimeofday(&t2, &tz); tsec = 0; tsec += t2.tv_usec; tsec -= t1.tv_usec; tsec /= 1000000.0; tsec += t2.tv_sec; tsec -= t1.tv_sec; printf("Decoding (MB/sec): %0.10f\n", (origsize/1024/1024)/totalsec); printf("De_Total (MB/sec): %0.10f\n\n", (origsize/1024/1024)/tsec); }
int main (int argc, char **argv) { FILE *fp, *fp2; // file pointers char *block; // padding file int size, newsize; // size of file and temp size struct stat status; // finding file size enum Coding_Technique tech; // coding technique (parameter) int k, m, w, packetsize; // parameters int buffersize; // paramter int i; // loop control variables int blocksize; // size of k+m files int total; int extra; /* Jerasure Arguments */ char **data; char **coding; int *matrix; int *bitmatrix; int **schedule; /* Creation of file name variables */ char temp[5]; char *s1, *s2, *extension; char *fname; int md; char *curdir; /* Timing variables */ struct timing t1, t2, t3, t4; double tsec; double totalsec; struct timing start; /* Find buffersize */ int up, down; signal(SIGQUIT, ctrl_bs_handler); /* Start timing */ timing_set(&t1); totalsec = 0.0; matrix = NULL; bitmatrix = NULL; schedule = NULL; /* Error check Arguments*/ if (argc != 8) { fprintf(stderr, "usage: inputfile k m coding_technique w packetsize buffersize\n"); fprintf(stderr, "\nChoose one of the following coding techniques: \nreed_sol_van, \nreed_sol_r6_op, \ncauchy_orig, \ncauchy_good, \nliberation, \nblaum_roth, \nliber8tion"); fprintf(stderr, "\n\nPacketsize is ignored for the reed_sol's"); fprintf(stderr, "\nBuffersize of 0 means the buffersize is chosen automatically.\n"); fprintf(stderr, "\nIf you just want to test speed, use an inputfile of \"-number\" where number is the size of the fake file you want to test.\n\n"); exit(0); } /* Conversion of parameters and error checking */ if (sscanf(argv[2], "%d", &k) == 0 || k <= 0) { fprintf(stderr, "Invalid value for k\n"); exit(0); } if (sscanf(argv[3], "%d", &m) == 0 || m < 0) { fprintf(stderr, "Invalid value for m\n"); exit(0); } if (sscanf(argv[5],"%d", &w) == 0 || w <= 0) { fprintf(stderr, "Invalid value for w.\n"); exit(0); } if (argc == 6) { packetsize = 0; } else { if (sscanf(argv[6], "%d", &packetsize) == 0 || packetsize < 0) { fprintf(stderr, "Invalid value for packetsize.\n"); exit(0); } } if (argc != 8) { buffersize = 0; } else { if (sscanf(argv[7], "%d", &buffersize) == 0 || buffersize < 0) { fprintf(stderr, "Invalid value for buffersize\n"); exit(0); } } /* Determine proper buffersize by finding the closest valid buffersize to the input value */ if (buffersize != 0) { if (packetsize != 0 && buffersize%(sizeof(long)*w*k*packetsize) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(long)*w*k*packetsize) != 0 && (down%(sizeof(long)*w*k*packetsize) != 0)) { up++; if (down == 0) { down--; } } if (up%(sizeof(long)*w*k*packetsize) == 0) { buffersize = up; } else { if (down != 0) { buffersize = down; } } } else if (packetsize == 0 && buffersize%(sizeof(long)*w*k) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(long)*w*k) != 0 && down%(sizeof(long)*w*k) != 0) { up++; down--; } if (up%(sizeof(long)*w*k) == 0) { buffersize = up; } else { buffersize = down; } } } /* Setting of coding technique and error checking */ if (strcmp(argv[4], "no_coding") == 0) { tech = No_Coding; } else if (strcmp(argv[4], "reed_sol_van") == 0) { tech = Reed_Sol_Van; if (w != 8 && w != 16 && w != 32) { fprintf(stderr, "w must be one of {8, 16, 32}\n"); exit(0); } } else if (strcmp(argv[4], "reed_sol_r6_op") == 0) { if (m != 2) { fprintf(stderr, "m must be equal to 2\n"); exit(0); } if (w != 8 && w != 16 && w != 32) { fprintf(stderr, "w must be one of {8, 16, 32}\n"); exit(0); } tech = Reed_Sol_R6_Op; } else if (strcmp(argv[4], "cauchy_orig") == 0) { tech = Cauchy_Orig; if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } } else if (strcmp(argv[4], "cauchy_good") == 0) { tech = Cauchy_Good; if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } } else if (strcmp(argv[4], "liberation") == 0) { if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } if (w <= 2 || !(w%2) || !is_prime(w)) { fprintf(stderr, "w must be greater than two and w must be prime\n"); exit(0); } if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } if ((packetsize%(sizeof(long))) != 0) { fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n"); exit(0); } tech = Liberation; } else if (strcmp(argv[4], "blaum_roth") == 0) { if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } if (w <= 2 || !((w+1)%2) || !is_prime(w+1)) { fprintf(stderr, "w must be greater than two and w+1 must be prime\n"); exit(0); } if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } if ((packetsize%(sizeof(long))) != 0) { fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n"); exit(0); } tech = Blaum_Roth; } else if (strcmp(argv[4], "liber8tion") == 0) { if (packetsize == 0) { fprintf(stderr, "Must include packetsize\n"); exit(0); } if (w != 8) { fprintf(stderr, "w must equal 8\n"); exit(0); } if (m != 2) { fprintf(stderr, "m must equal 2\n"); exit(0); } if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } tech = Liber8tion; } else { fprintf(stderr, "Not a valid coding technique. Choose one of the following: reed_sol_van, reed_sol_r6_op, cauchy_orig, cauchy_good, liberation, blaum_roth, liber8tion, no_coding\n"); exit(0); } /* Set global variable method for signal handler */ method = tech; /* Get current working directory for construction of file names */ curdir = (char*)malloc(sizeof(char)*1000); if (! getcwd(curdir, 1000)) { curdir[0] = 0; } if (argv[1][0] != '-') { /* Open file and error check */ fp = fopen(argv[1], "rb"); if (fp == NULL) { fprintf(stderr, "Unable to open file.\n"); exit(0); } /* Create Coding directory */ i = mkdir("Coding", S_IRWXU); if (i == -1 && errno != EEXIST) { fprintf(stderr, "Unable to create Coding directory.\n"); exit(0); } /* Determine original size of file */ stat(argv[1], &status); size = status.st_size; } else { if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) { fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n"); exit(1); } fp = NULL; MOA_Seed(time(0)); } newsize = size; /* Find new size by determining next closest multiple */ if (packetsize != 0) { if (size%(k*w*packetsize*sizeof(long)) != 0) { while (newsize%(k*w*packetsize*sizeof(long)) != 0) newsize++; } } else { if (size%(k*w*sizeof(long)) != 0) { while (newsize%(k*w*sizeof(long)) != 0) newsize++; } } if (buffersize != 0) { while (newsize%buffersize != 0) { newsize++; } } /* Determine size of k+m files */ blocksize = newsize/k; /* Allow for buffersize and determine number of read-ins */ if (size > buffersize && buffersize != 0) { if (newsize%buffersize != 0) { readins = newsize/buffersize; } else { readins = newsize/buffersize; } block = (char *)malloc(sizeof(char)*buffersize); blocksize = buffersize/k; } else { readins = 1; buffersize = size; block = (char *)malloc(sizeof(char)*newsize); } /* Break inputfile name into the filename and extension */ s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+20)); s2 = strrchr(argv[1], '/'); if (s2 != NULL) { s2++; strcpy(s1, s2); } else { strcpy(s1, argv[1]); } s2 = strchr(s1, '.'); if (s2 != NULL) { extension = strdup(s2); *s2 = '\0'; } else { extension = strdup(""); } /* Allocate for full file name */ fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+20)); sprintf(temp, "%d", k); md = strlen(temp); /* Allocate data and coding */ data = (char **)malloc(sizeof(char*)*k); coding = (char **)malloc(sizeof(char*)*m); for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*blocksize); if (coding[i] == NULL) { perror("malloc"); exit(1); } } /* Create coding matrix or bitmatrix and schedule */ timing_set(&t3); switch(tech) { case No_Coding: break; case Reed_Sol_Van: matrix = reed_sol_vandermonde_coding_matrix(k, m, w); break; case Reed_Sol_R6_Op: break; case Cauchy_Orig: matrix = cauchy_original_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Cauchy_Good: matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Liberation: bitmatrix = liberation_coding_bitmatrix(k, w); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Blaum_Roth: bitmatrix = blaum_roth_coding_bitmatrix(k, w); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Liber8tion: bitmatrix = liber8tion_coding_bitmatrix(k); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case RDP: case EVENODD: assert(0); } timing_set(&start); timing_set(&t4); totalsec += timing_delta(&t3, &t4); /* Read in data until finished */ n = 1; total = 0; while (n <= readins) { /* Check if padding is needed, if so, add appropriate number of zeros */ if (total < size && total+buffersize <= size) { total += jfread(block, sizeof(char), buffersize, fp); } else if (total < size && total+buffersize > size) { extra = jfread(block, sizeof(char), buffersize, fp); for (i = extra; i < buffersize; i++) { block[i] = '0'; } } else if (total == size) { for (i = 0; i < buffersize; i++) { block[i] = '0'; } } /* Set pointers to point to file data */ for (i = 0; i < k; i++) { data[i] = block+(i*blocksize); } timing_set(&t3); /* Encode according to coding method */ switch(tech) { case No_Coding: break; case Reed_Sol_Van: jerasure_matrix_encode(k, m, w, matrix, data, coding, blocksize); break; case Reed_Sol_R6_Op: reed_sol_r6_encode(k, w, data, coding, blocksize); break; case Cauchy_Orig: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Cauchy_Good: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Liberation: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Blaum_Roth: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Liber8tion: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case RDP: case EVENODD: assert(0); } timing_set(&t4); /* Write data and encoded data to k+m files */ for (i = 1; i <= k; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_k%0*d%s", curdir, s1, md, i, extension); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(data[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } for (i = 1; i <= m; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_m%0*d%s", curdir, s1, md, i, extension); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(coding[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } n++; /* Calculate encoding time */ totalsec += timing_delta(&t3, &t4); } /* Create metadata file */ if (fp != NULL) { sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1); fp2 = fopen(fname, "wb"); fprintf(fp2, "%s\n", argv[1]); fprintf(fp2, "%d\n", size); fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize); fprintf(fp2, "%s\n", argv[4]); fprintf(fp2, "%d\n", tech); fprintf(fp2, "%d\n", readins); fclose(fp2); } /* Free allocated memory */ free(s1); free(fname); free(block); free(curdir); /* Calculate rate in MB/sec and print */ timing_set(&t2); tsec = timing_delta(&t1, &t2); printf("Encoding (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/totalsec); printf("En_Total (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/tsec); return 0; }
int main (int argc, char **argv) { FILE *fp, *fp2; // file pointers char *memblock; // reading in file char *block; // padding file int size, newsize; // size of file and temp size struct stat status; // finding file size enum Coding_Technique tech; // coding technique (parameter) int k, m, w, packetsize; // parameters int buffersize; // paramter int i, j; // loop control variables int blocksize; // size of k+m files int total; int extra; /* Jerasure Arguments */ char **data; char **coding; int *matrix; int *bitmatrix; int **schedule; int *erasure; int *erased; /* Creation of file name variables */ char temp[5]; char *s1, *s2; char *fname; int md; char *curdir; /* Timing variables */ struct timeval t1, t2, t3, t4; struct timezone tz; double tsec; double totalsec; struct timeval start, stop; /* Find buffersize */ int up, down; /* Modifications */ int _size, sizeCorrect = 0, numFiles = 0, codingFactor, testValue; /* Start timing */ gettimeofday(&t1, &tz); totalsec = 0.0; matrix = NULL; bitmatrix = NULL; schedule = NULL; printf("\nFilename: %s\n", argv[1]); printf("Coding Technique: cauchy_good\n"); printf("Input desired size of file segments (in MB): "); scanf("%d", &_size); printf("Input codingFactor: "); scanf("%d", &codingFactor); // printf("Input packetsize: "); // scanf("%d", &packetsize); // printf("Input buffersize: "); // scanf("%d", &buffersize); w = 16; packetsize = 32; buffersize = 50000; //_size = _size * 1024 * 1024; /* Set global variable method for signal handler */ tech = Cauchy_Good; method = tech; /* Get current working directory for construction of file names */ curdir = (char*)malloc(sizeof(char)*1000); getcwd(curdir, 1000); if (argv[1][0] != '-') { /* Open file and error check */ fp = fopen(argv[1], "rb"); if (fp == NULL) { fprintf(stderr, "Unable to open file.\n"); exit(0); } /* Create Coding directory */ i = mkdir("Coding", S_IRWXU); if (i == -1 && errno != EEXIST) { fprintf(stderr, "Unable to create Coding directory.\n"); exit(0); } /* Determine original size of file */ stat(argv[1], &status); size = status.st_size; } else { if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) { fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n"); exit(1); } fp = NULL; srand48(time(0)); } k = size/_size; m = (codingFactor-1) * k; /* Determine proper buffersize by finding the closest valid buffersize to the input value */ if (buffersize != 0) { if (packetsize != 0 && buffersize%(sizeof(int)*w*k*packetsize) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(int)*w*k*packetsize) != 0 && (down%(sizeof(int)*w*k*packetsize) != 0)) { up++; if (down == 0) { down--; } } if (up%(sizeof(int)*w*k*packetsize) == 0) { buffersize = up; } else { if (down != 0) { buffersize = down; } } } else if (packetsize == 0 && buffersize%(sizeof(int)*w*k) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(int)*w*k) != 0 && down%(sizeof(int)*w*k) != 0) { testValue = up%(sizeof(int)*w*k); printf("up%% = %d\n", testValue); up++; down--; } if (up%(sizeof(int)*w*k) == 0) { buffersize = up; } else { buffersize = down; } } } newsize = size; /* Find new size by determining next closest multiple */ if (packetsize != 0) { if (size%(k*w*packetsize*sizeof(int)) != 0) { while (newsize%(k*w*packetsize*sizeof(int)) != 0) newsize++; } } else { if (size%(k*w*sizeof(int)) != 0) { while (newsize%(k*w*sizeof(int)) != 0) newsize++; } } if (buffersize != 0) { while (newsize%buffersize != 0) { newsize++; } } /* Determine size of k+m files */ blocksize = newsize/k; printf("\n"); printf("Size of file segments: %d \tbytes\n", blocksize); printf("Size of buffer: %d \tbytes\n", buffersize); printf("Size of file: %d \tbytes\n", size); printf("Size of word: %d \t\tbytes\n\n", w); printf("Number of data files: %d\n", k); printf("Number of coding files: %d\n", m); /* Allow for buffersize and determine number of read-ins */ if (size > buffersize && buffersize != 0) { if (newsize%buffersize != 0) { readins = newsize/buffersize; } else { readins = newsize/buffersize; } block = (char *)malloc(sizeof(char)*buffersize); blocksize = buffersize/k; } else { readins = 1; buffersize = size; block = (char *)malloc(sizeof(char)*newsize); } /* Break inputfile name into the filename and extension */ s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+10)); s2 = strrchr(argv[1], '/'); if (s2 != NULL) { s2++; strcpy(s1, s2); } else { strcpy(s1, argv[1]); } s2 = strchr(s1, '.'); if (s2 != NULL) { *s2 = '\0'; } fname = strchr(argv[1], '.'); s2 = (char*)malloc(sizeof(char)*(strlen(argv[1])+5)); if (fname != NULL) { strcpy(s2, fname); } /* Allocate for full file name */ fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+10)); sprintf(temp, "%d", k+m); md = strlen(temp); /* Allocate data and coding */ data = (char **)malloc(sizeof(char*)*k); coding = (char **)malloc(sizeof(char*)*m); for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*blocksize); } /* Create coding matrix or bitmatrix and schedule */ gettimeofday(&t3, &tz); matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); gettimeofday(&start, &tz); gettimeofday(&t4, &tz); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; /* Read in data until finished */ n = 1; total = 0; printf("\nABOUT TO ENCODE SCHEDULE\n"); while (n <= readins) { /* Check if padding is needed, if so, add appropriate number of zeros */ if (total < size && total+buffersize <= size) { total += jfread(block, sizeof(char), buffersize, fp); } else if (total < size && total+buffersize > size) { extra = jfread(block, sizeof(char), buffersize, fp); for (i = extra; i < buffersize; i++) { block[i] = '0'; } } else if (total == size) { for (i = 0; i < buffersize; i++) { block[i] = '0'; } } /* Set pointers to point to file data */ for (i = 0; i < k; i++) { data[i] = block+(i*blocksize); } gettimeofday(&t3, &tz); jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); gettimeofday(&t4, &tz); /* Write data and encoded data to k+m files */ for (i = 1; i <= k; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(data[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } for (i; i <= k+m; i++) { if (fp == NULL) { bzero(data[i-1-k], blocksize); } else { sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(coding[i-1-k], sizeof(char), blocksize, fp2); fclose(fp2); } } n++; /* Calculate encoding time */ tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; } /* Create metadata file */ if (fp != NULL) { sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1); fp2 = fopen(fname, "wb"); fprintf(fp2, "%s\n", argv[1]); fprintf(fp2, "%d\n", size); fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize); fprintf(fp2, "%s\n", argv[4]); fprintf(fp2, "%d\n", tech); fprintf(fp2, "%d\n", readins); fclose(fp2); } /* Free allocated memory */ free(s2); free(s1); free(fname); free(block); free(curdir); /* Calculate rate in MB/sec and print */ gettimeofday(&t2, &tz); tsec = 0.0; tsec += t2.tv_usec; tsec -= t1.tv_usec; tsec /= 1000000.0; tsec += t2.tv_sec; tsec -= t1.tv_sec; printf("Encoding (MB/sec): %0.10f\n", (size/1024/1024)/totalsec); printf("En_Total (MB/sec): %0.10f\n", (size/1024/1024)/tsec); }
int main (int argc, char **argv) { FILE *fp; // File pointer char *dummy; int err; /* Jerasure arguments */ char **data; char **coding; int *erasures; int *erased; int *matrix; int *bitmatrix; /* Parameters */ int k, m, w, packetsize, buffersize; enum Coding_Technique tech; char *c_tech; int i, j; // loop control variables int blocksize; // size of individual files int origsize; // size of file before padding int total; // used to write data, not padding to file struct stat status; // used to find size of individual files int numerased; // number of erased files /* Used to recreate file names */ char *temp; char *cs1, *cs2, *extension; char *fname; int md; char *curdir; /* Used to time decoding */ struct timeval t1, t2, t3, t4; struct timezone tz; double tsec; double totalsec; signal(SIGQUIT, ctrl_bs_handler); matrix = NULL; bitmatrix = NULL; totalsec = 0.0; blocksize = -1; /* Start timing */ gettimeofday(&t1, &tz); /* Error checking parameters */ if (argc != 2) { fprintf(stderr, "usage: inputfile\n"); exit(0); } curdir = (char *)malloc(sizeof(char)*100); dummy = getcwd(curdir, 100); /* Begin recreation of file names */ cs1 = (char*)malloc(sizeof(char)*strlen(argv[1])); cs2 = strrchr(argv[1], '/'); if (cs2 != NULL) { cs2++; strcpy(cs1, cs2); } else { strcpy(cs1, argv[1]); } cs2 = strchr(cs1, '.'); if (cs2 != NULL) { extension = strdup(cs2); *cs2 = '\0'; } else { extension = strdup(""); } fname = (char *)malloc(sizeof(char*)*(100+strlen(argv[1])+10)); /* Read in parameters from metadata file */ sprintf(fname, "%s/Coding/%s_meta.txt", curdir, cs1); fp = fopen(fname, "rb"); if (fp == NULL) { fprintf(stderr, "Error: no metadata file %s\n", fname); exit(1); } temp = (char *)malloc(sizeof(char)*(strlen(argv[1])+10)); err = fscanf(fp, "%s", temp); if (fscanf(fp, "%d", &origsize) != 1) { fprintf(stderr, "Original size is not valid\n"); exit(0); } if (fscanf(fp, "%d %d %d %d %d", &k, &m, &w, &packetsize, &buffersize) != 5) { fprintf(stderr, "Parameters are not correct\n"); exit(0); } c_tech = (char *)malloc(sizeof(char)*(strlen(argv[1])+10)); err = fscanf(fp, "%s", c_tech); err = fscanf(fp, "%d", &tech); method = tech; err = fscanf(fp, "%d", &readins); fclose(fp); /* Allocate memory */ erased = (int *)malloc(sizeof(int)*(k+m)); for (i = 0; i < k+m; i++) erased[i] = 0; erasures = (int *)malloc(sizeof(int)*(k+m)); data = (char **)malloc(sizeof(char *)*k); coding = (char **)malloc(sizeof(char *)*m); if (buffersize != origsize) { for (i = 0; i < k; i++) { data[i] = (char *)malloc(sizeof(char)*(buffersize/k)); } for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*(buffersize/k)); } blocksize = buffersize/k; } sprintf(temp, "%d", k); md = strlen(temp); gettimeofday(&t3, &tz); /* Create coding matrix or bitmatrix */ switch(tech) { case No_Coding: case RDP: case EVENODD: break; case Reed_Sol_Van: matrix = reed_sol_vandermonde_coding_matrix(k, m, w); break; case Reed_Sol_R6_Op: matrix = reed_sol_r6_coding_matrix(k, w); break; case Cauchy_Orig: matrix = cauchy_original_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); break; case Cauchy_Good: matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); break; case Liberation: bitmatrix = liberation_coding_bitmatrix(k, w); break; case Blaum_Roth: bitmatrix = blaum_roth_coding_bitmatrix(k, w); break; case Liber8tion: bitmatrix = liber8tion_coding_bitmatrix(k); break; default: fprintf(stderr, "unsupported coding technique used\n"); break; } gettimeofday(&t4, &tz); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; /* Begin decoding process */ total = 0; n = 1; while (n <= readins) { numerased = 0; /* Open files, check for erasures, read in data/coding */ for (i = 1; i <= k; i++) { sprintf(fname, "%s/Coding/%s_k%0*d%s", curdir, cs1, md, i, extension); fp = fopen(fname, "rb"); if (fp == NULL) { erased[i-1] = 1; erasures[numerased] = i-1; numerased++; //printf("%s failed\n", fname); } else { if (buffersize == origsize) { stat(fname, &status); blocksize = status.st_size; data[i-1] = (char *)malloc(sizeof(char)*blocksize); err = fread(data[i-1], sizeof(char), blocksize, fp); } else { fseek(fp, blocksize*(n-1), SEEK_SET); err = fread(data[i-1], sizeof(char), buffersize/k, fp); } fclose(fp); } } for (i = 1; i <= m; i++) { sprintf(fname, "%s/Coding/%s_m%0*d%s", curdir, cs1, md, i, extension); fp = fopen(fname, "rb"); if (fp == NULL) { erased[k+(i-1)] = 1; erasures[numerased] = k+i-1; numerased++; //printf("%s failed\n", fname); } else { if (buffersize == origsize) { stat(fname, &status); blocksize = status.st_size; coding[i-1] = (char *)malloc(sizeof(char)*blocksize); err = fread(coding[i-1], sizeof(char), blocksize, fp); } else { fseek(fp, blocksize*(n-1), SEEK_SET); err = fread(coding[i-1], sizeof(char), blocksize, fp); } fclose(fp); } } /* Finish allocating data/coding if needed */ if (n == 1) { for (i = 0; i < numerased; i++) { if (erasures[i] < k) { data[erasures[i]] = (char *)malloc(sizeof(char)*blocksize); } else { coding[erasures[i]-k] = (char *)malloc(sizeof(char)*blocksize); } } } erasures[numerased] = -1; gettimeofday(&t3, &tz); /* Choose proper decoding method */ if (tech == Reed_Sol_Van || tech == Reed_Sol_R6_Op) { i = jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data, coding, blocksize); } else if (tech == Cauchy_Orig || tech == Cauchy_Good || tech == Liberation || tech == Blaum_Roth || tech == Liber8tion) { i = jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data, coding, blocksize, packetsize, 1); } else { fprintf(stderr, "Not a valid coding technique.\n"); exit(0); } gettimeofday(&t4, &tz); /* Exit if decoding was unsuccessful */ if (i == -1) { fprintf(stderr, "Unsuccessful!\n"); exit(0); } /* Create decoded file */ sprintf(fname, "%s/Coding/%s_decoded%s", curdir, cs1, extension); if (n == 1) { fp = fopen(fname, "wb"); } else { fp = fopen(fname, "ab"); } for (i = 0; i < k; i++) { if (total+blocksize <= origsize) { fwrite(data[i], sizeof(char), blocksize, fp); total+= blocksize; } else { for (j = 0; j < blocksize; j++) { if (total < origsize) { fprintf(fp, "%c", data[i][j]); total++; } else { break; } } } } n++; fclose(fp); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; } /* Free allocated memory */ free(cs1); free(extension); free(fname); free(data); free(coding); free(erasures); free(erased); /* Stop timing and print time */ gettimeofday(&t2, &tz); tsec = 0; tsec += t2.tv_usec; tsec -= t1.tv_usec; tsec /= 1000000.0; tsec += t2.tv_sec; tsec -= t1.tv_sec; printf("Decoding (MB/sec): %0.10f\n", (origsize/1024/1024)/totalsec); printf("De_Total (MB/sec): %0.10f\n\n", (origsize/1024/1024)/tsec); return(0); }
int decode_MSR_product_matrix_no_output(char **input, size_t input_size, int* erasures, struct coding_info *info) { clock_t clk, tclk; int i, j,c1,c2,tdone,inv; int n = info->req.n; int d = info->req.d; int k = info->req.k; int w = info->req.w; int subpacket_size = input_size/(d-k+1); int num_of_long = subpacket_size/sizeof(long); long *src_pos,*des_pos; int *vector_A=NULL; char **ptrs; int **decode_schedule=NULL; int *erased = NULL; int *bitmatrix_temp = malloc(sizeof(int)*(k-1)*(k-1)*w*w*4); char *data_transformed = malloc(input_size*k); // this is the buffer for tranformed data, i.e., matrix M. The output is the systematic part of // codingmatrix*M, and // we will regenerate the erased data from M using the encoding matrix, which will be written to *output. int *pseudo_erasures = malloc(sizeof(int)*(n*2)); // in order to recompute M, we view it as a systematic MDS code, // sometimes (n+k,k), sometimes (n+d-k+1,d-k+1), thus we over allocate char **M_ptrs = malloc(sizeof(void*)*d*(d-k+1)); // this is the pointer matrix to elements in M. char **data_ptrs = malloc(sizeof(void*)*n); char **coding_ptrs = malloc(sizeof(void*)*n); int* remaining = malloc(sizeof(int)*k); // not erased devices char *buffer1 = malloc(subpacket_size*k*(k-1)*2); // need subpacketsize*k>=(max(4,k-1)+k-1)*sizeof(int), thus put factor of 2 to guarentee it int *buffer1_int = (int*)buffer1; // alternative pointer for buffer1 char *buffer2 = malloc(subpacket_size*k*(k-1)); if(data_transformed==NULL||pseudo_erasures==NULL||data_ptrs==NULL ||coding_ptrs==NULL||buffer1==NULL||buffer2==NULL||M_ptrs==NULL||remaining==NULL){ printf("Can not allocate memory\n"); jerasure_free_schedule(decode_schedule); if(data_ptrs!=NULL)free(data_ptrs); if(coding_ptrs!=NULL)free(coding_ptrs); if(pseudo_erasures!=NULL)free(pseudo_erasures); if(data_transformed!=NULL)free(data_transformed); if(M_ptrs!=NULL)free(M_ptrs); if(buffer1!=NULL)free(buffer1); if(buffer2!=NULL)free(buffer2); if(remaining!=NULL)free(remaining); return(-1); } //set up pointers for matrix M // first k-1 rows, only have S1 for(i=0,c2=0;i<k-1;i++){ c1 = i*(d-k+1); for(j=i;j<k-1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=k-1;j<d-k+1;j++) M_ptrs[c1+j] = NULL; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[j*(d-k+1)+i]; // symmetric matrix, thus there is (i,j) and (j,i) point to the same pointer. } // next k-1 rows for(i=0;i<k-1;i++){ c1 = (k-1+i)*(d-k+1); for(j=i;j<d-k+1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i]; } // next 1 and (d-2k+1) rows, may not exist if(d>2*k-2) { c1 = (2*k-2)*(d-k+1); for(j=k-1;j<d-k+1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+k-1]; for(i=2*k-1;i<d;i++){ c1 = i*(d-k+1); for(j=0;j<k;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i-k+1]; for(j=k;j<d-k+1;j++) M_ptrs[c1+j] = NULL; } } // first decode the last d-2k+1 columns of T and Z: view it as an (n+k,k) MDS code. Note strictly speaking this might // not be a real (n+k,k) MDS code, but it hardly matters. // before decoding operation, prepare for the pseudoerasure location array if(d>2*k-2){ // only when d>2k-2, T and Z exist for(i=0;i<k;i++) pseudo_erasures[i] = i; for(i=0;i<n;i++){ if(erasures[i]==-1){ pseudo_erasures[i+k] = -1; break; } pseudo_erasures[i+k] = erasures[i] + k; } // make the decoding schedule: we can save the trouble of manually generate this schedule, at the expense of // more computation decode_schedule = jerasure_generate_decoding_schedule(k, n, w, info->subbitmatrix_array[0], pseudo_erasures, 1); for(i=0;i<d-2*k+1;i++){ for(j=0;j<k;j++) data_ptrs[j] = M_ptrs[i+k+(d-k+1)*(k-1+j)]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+subpacket_size*(k+i); ptrs = set_up_ptrs_for_scheduled_decoding(k, n, pseudo_erasures, data_ptrs,coding_ptrs); if (ptrs == NULL){ printf("Can not allocate memory\n"); goto complete; } // assume packetsize = ALIGNMENT for (tdone = 0; tdone < subpacket_size; tdone += ALIGNMENT*w) { jerasure_do_scheduled_operations(ptrs, decode_schedule, ALIGNMENT); for (c1 = 0; c1 < k+n; c1++) ptrs[c1] += (ALIGNMENT*w); } free(ptrs); } //next decode the first column of T and Z: we view this as an (n+d-k+1,d-k+1) erasure codes // first setup the pseudo erasure location vector for(i=0;i<k;i++) pseudo_erasures[i] = i; // in the first columne of the Z matrix, the last d-2k+1 elements are known, but the first k elements are not for(i=0;i<n;i++) { if(erasures[i]==-1){ pseudo_erasures[i+k] = -1; break; } pseudo_erasures[i+k] = erasures[i]+d-k+1; } for(j=0;j<d-k+1;j++) data_ptrs[j] = M_ptrs[(d-k+1)*(k-1+j)+k-1]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+subpacket_size*(k-1); jerasure_schedule_decode_lazy(d-k+1,n,w, info->subbitmatrix_array[1],pseudo_erasures,data_ptrs, coding_ptrs,subpacket_size,ALIGNMENT,0); } //clk = clock(); // now this is the hard part: to decode S1 and S2. The algorithm used here is slightly different from that in the paper: // instead of right multiply \Phi_{DC}', we only right multiply the sub-matrix of \Phi_{DC}' without of the last row // setting up remaining device array to facilitate decoding erased = jerasure_erasures_to_erased(k, n-k, erasures); for(i=0,c1=0;i<n&&c1<k;i++){ if(erased[i]==0){ remaining[c1] = i; c1++; } } //compute C_{DC}-\Delta_{DC}*T' for(i=0;i<k-1;i++){ // has k-1 columns for(j=0;j<d-2*k+2;j++) data_ptrs[j] = M_ptrs[(2*k-2+j)*(d-k+1)+i]; for(j=0;j<k;j++) coding_ptrs[j] = buffer1+(j*(k-1)+i)*subpacket_size; for(j=0;j<k;j++){ jerasure_bitmatrix_dotprod(d-2*k+2, w, info->subbitmatrix_array[2]+remaining[j]*(d-2*k+2)*w*w, NULL, j+d-2*k+2, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); src_pos = (long*)(input[remaining[j]]+i*subpacket_size); des_pos = (long*)(coding_ptrs[j]); for(c2=0;c2<num_of_long;c2++) des_pos[c2] ^= src_pos[c2]; } } // right multiply \Phi_{DC}': this will be P. // result is in buffer2 for(j=0;j<k;j++){ //j-th row for(i=0;i<k-1;i++) data_ptrs[i] = buffer1+(j*(k-1)+i)*subpacket_size; for(i=0;i<k-1;i++) coding_ptrs[i] = buffer2 +(j*(k-1)+i)*subpacket_size; for(i=0;i<k-1;i++){ jerasure_bitmatrix_dotprod(k-1, w, info->subbitmatrix_array[3]+remaining[i]*(k-1)*w*w, NULL, i+k-1, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } } // now solve for the off-diagonal terms for(i=0;i<k-1;i++){ for(j=i+1;j<k-1;j++){ // solve for S1 tilde off-diagonal // here we directly use the fact that Lambda = [0 1 2 3 ....]; int** temp_schedule; inv = galois_single_divide(1,remaining[i]^remaining[j],w); buffer1_int[0] = buffer1_int[1] = inv; data_ptrs[0] = buffer2+(i*(k-1)+j)*subpacket_size; data_ptrs[1] = buffer2+(j*(k-1)+i)*subpacket_size; coding_ptrs[0] = M_ptrs[i*(d-k+1)+j]; jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp); temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp); jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT); if(temp_schedule!=NULL){ jerasure_free_schedule(temp_schedule); temp_schedule = NULL; } // solve for S2 tilde off-diagonal buffer1_int[0] = galois_single_multiply(remaining[j],inv,w); buffer1_int[1] = galois_single_multiply(remaining[i],inv,w); coding_ptrs[0] = M_ptrs[(i+k-1)*(d-k+1)+j]; jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp); temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp); jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT); if(temp_schedule!=NULL){ jerasure_free_schedule(temp_schedule); temp_schedule = NULL; } } } //tclk = clock()-clk; //printf("~S1 and ~S2 off-diagonal decoded %.3e clocks \n", (double)tclk); //clk = clock(); // compute the A vector: A*\Phi_{DC1} = \Phi_{DC2}, note this is always possible because \Phi_{DC1} is alway full rank by construction // we first reuse buffer1 here to form \Phi_{DC1} matrix and then the compute its inverse for(i=0;i<k-1;i++){ memcpy(buffer1_int+(k-1)*(k-1+i),(void*)(info->matrix+remaining[i]*d+k-1),(k-1)*sizeof(int)); } jerasure_invert_matrix(buffer1_int+(k-1)*(k-1),buffer1_int,k-1,w); vector_A = jerasure_matrix_multiply(info->matrix+remaining[k-1]*d+k-1,buffer1_int,1,k-1,k-1,k-1,w); if(vector_A==NULL) goto complete; for(i=0;i<k-1;i++){ buffer1_int[i+(k-1)*(k-1)] = galois_single_multiply(vector_A[i],remaining[k-1],w); buffer1_int[i+k*(k-1)] = vector_A[i]; } for(i=0;i<k-1;i++){ memset(buffer1_int+(k-1)*(k+1),0,sizeof(int)*2*(k-1)); buffer1_int[(k-1)*(k+1)+i] = remaining[i]; buffer1_int[(k-1)*(k+2)+i] = 1; pseudo_erasures[0] = i; pseudo_erasures[1] = k-1+i; pseudo_erasures[2] = -1; for(j=0;j<2*k-2;j++) data_ptrs[j] = M_ptrs[i+j*(d-k+1)]; coding_ptrs[0] = buffer2+((k-1)*(k-1)+i)*subpacket_size; coding_ptrs[1] = buffer2+(i*(k-1)+i)*subpacket_size; jerasure_matrix_to_bitmatrix_noallocate(2*k-2,2,w,buffer1_int+(k-1)*(k-1),bitmatrix_temp); jerasure_schedule_decode_lazy(2*k-2,2,w,bitmatrix_temp,pseudo_erasures,data_ptrs,coding_ptrs,subpacket_size,ALIGNMENT,0); } //tclk = clock()-clk; //printf("~S1 and ~S2 decoded %.3e clocks \n", (double)tclk); // now we have both \tilde{S_1} and \tilde{S_2} in M_ptrs, need to recover S1 and S2 from them // this is done by multiply \tilde{S_1} left and right by inv(\Phi_{DC1}). // right-multiply for S1 int* bitmatrix_inv = jerasure_matrix_to_bitmatrix(k-1,k-1,w,buffer1_int); int** inv_schedule = jerasure_smart_bitmatrix_to_schedule(k-1, k-1, w, bitmatrix_inv); // right-multiply for S1 for(i=0;i<k-1;i++){ for(j=0;j<k-1;j++) data_ptrs[j] = M_ptrs[i*(d-k+1)+j]; for(j=0;j<k-1;j++) coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // left-multiply for S1 for(j=0;j<k-1;j++){ for(i=0;i<k-1;i++) data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size; for(i=0;i<k-1;i++) coding_ptrs[i] = M_ptrs[i*(d-k+1)+j]; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // right-multiply for S2 for(i=0;i<k-1;i++){ for(j=0;j<k-1;j++) data_ptrs[j] = M_ptrs[(i+k-1)*(d-k+1)+j]; for(j=0;j<k-1;j++) coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // left-multiply for S2 for(j=0;j<k-1;j++){ for(i=0;i<k-1;i++) data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size; //for(i=j;i<k-1;i++) for(i=0;i<k-1;i++) coding_ptrs[i] = M_ptrs[(i+k-1)*(d-k+1)+j]; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // having S1,S2,T, now can also fill the first k-1 column of the output for(i=0;i<k-1;i++){ for(j=0;j<d;j++) data_ptrs[j] = M_ptrs[j*(d-k+1)+i]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+i*subpacket_size; for(j=0;j<n;j++){ if(erased[j]==1) jerasure_bitmatrix_encode(d,1,w,info->bitmatrix+(j*d*w*w),data_ptrs,coding_ptrs+j,subpacket_size,ALIGNMENT); } } // clean up complete: if(decode_schedule) jerasure_free_schedule(decode_schedule); if(inv_schedule) jerasure_free_schedule(inv_schedule); free(data_ptrs); free(coding_ptrs); if(erased)free(erased); free(pseudo_erasures); free(data_transformed); free(M_ptrs); free(buffer1); free(buffer2); free(remaining); free(bitmatrix_temp); if(vector_A!=NULL)free(vector_A); return(1); }