vector<ERL_NIF_TERM> LiberationCoding::doEncode(ERL_NIF_TERM dataBin) { int *bitmatrix = liberation_coding_bitmatrix(k, w); int **smart = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); char* dataBlocks[k]; char* codeBlocks[m]; ErlNifBinary data; enif_inspect_binary(env, dataBin, &data); size_t dataSize = data.size; size_t blockSize = roundTo((roundTo(dataSize, k*w) / (k*w)), 16) * w; size_t offset = 0; size_t remain = dataSize; int filled = 0; while(remain >= blockSize) { dataBlocks[filled] = (char*)data.data + offset; offset += blockSize; remain -= blockSize; filled++; } ErlNifBinary tmp; enif_alloc_binary((k + m - filled) * blockSize + 16, &tmp); size_t align = (((size_t)data.data & 0x0f) - ((size_t)tmp.data & 0x0f) + 16) & 0x0f; char* alignedHead = (char*)tmp.data + align; memcpy(alignedHead, data.data + filled * blockSize, dataSize - filled * blockSize); offset = 0; for(int i = filled; i < k + m; ++i, offset += blockSize) { (i < k) ? dataBlocks[i] = alignedHead + offset: codeBlocks[i - k] = alignedHead + offset; } jerasure_schedule_encode(k, m, w, smart, dataBlocks, codeBlocks, blockSize, blockSize / w); vector<ERL_NIF_TERM> blockList; for(int i = 0; i < filled; ++i) { blockList.push_back(enif_make_sub_binary(env, dataBin, i * blockSize, blockSize)); } ERL_NIF_TERM tmpBin = enif_make_binary(env, &tmp); offset = 0; for(int i = filled; i < k + m; ++i, offset += blockSize) { blockList.push_back(enif_make_sub_binary(env, tmpBin, offset + align, blockSize)); } jerasure_free_schedule(smart); free(bitmatrix); return blockList; }
int rain_encode_noalloc (struct rain_encoding_s *encoding, uint8_t **data, uint8_t **parity) { assert(encoding != NULL); // Prepare the jerasure structures int *bit_matrix=NULL, *matrix=NULL, **schedule=NULL; if (encoding->algo == JALG_liberation) { matrix = NULL; bit_matrix = liber8tion_coding_bitmatrix(encoding->k); schedule = jerasure_smart_bitmatrix_to_schedule(encoding->k, encoding->m, encoding->w, bit_matrix); } else if (encoding->algo == JALG_crs) { matrix = cauchy_good_general_coding_matrix( encoding->k, encoding->m, encoding->w); bit_matrix = jerasure_matrix_to_bitmatrix( encoding->k, encoding->m, encoding->w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule( encoding->k, encoding->m, encoding->w, bit_matrix); } // Compute now ... damned, no return code to check jerasure_schedule_encode(encoding->k, encoding->m, encoding->w, schedule, (char**) data, (char**) parity, encoding->block_size, encoding->packet_size); if (schedule) jerasure_free_schedule(schedule); if (bit_matrix) free(bit_matrix); if (matrix) free(matrix); return 1; }
int main (int argc, char **argv) { FILE *fp, *fp2; // file pointers char *block; // padding file int size, newsize; // size of file and temp size struct stat status; // finding file size enum Coding_Technique tech; // coding technique (parameter) int k, m, w, packetsize; // parameters int buffersize; // paramter int i; // loop control variables int blocksize; // size of k+m files int total; int extra; /* Jerasure Arguments */ char **data; char **coding; int *matrix; int *bitmatrix; int **schedule; /* Creation of file name variables */ char temp[5]; char *s1, *s2, *extension; char *fname; int md; char *curdir; /* Timing variables */ struct timing t1, t2, t3, t4; double tsec; double totalsec; struct timing start; /* Find buffersize */ int up, down; signal(SIGQUIT, ctrl_bs_handler); /* Start timing */ timing_set(&t1); totalsec = 0.0; matrix = NULL; bitmatrix = NULL; schedule = NULL; /* Error check Arguments*/ if (argc != 8) { fprintf(stderr, "usage: inputfile k m coding_technique w packetsize buffersize\n"); fprintf(stderr, "\nChoose one of the following coding techniques: \nreed_sol_van, \nreed_sol_r6_op, \ncauchy_orig, \ncauchy_good, \nliberation, \nblaum_roth, \nliber8tion"); fprintf(stderr, "\n\nPacketsize is ignored for the reed_sol's"); fprintf(stderr, "\nBuffersize of 0 means the buffersize is chosen automatically.\n"); fprintf(stderr, "\nIf you just want to test speed, use an inputfile of \"-number\" where number is the size of the fake file you want to test.\n\n"); exit(0); } /* Conversion of parameters and error checking */ if (sscanf(argv[2], "%d", &k) == 0 || k <= 0) { fprintf(stderr, "Invalid value for k\n"); exit(0); } if (sscanf(argv[3], "%d", &m) == 0 || m < 0) { fprintf(stderr, "Invalid value for m\n"); exit(0); } if (sscanf(argv[5],"%d", &w) == 0 || w <= 0) { fprintf(stderr, "Invalid value for w.\n"); exit(0); } if (argc == 6) { packetsize = 0; } else { if (sscanf(argv[6], "%d", &packetsize) == 0 || packetsize < 0) { fprintf(stderr, "Invalid value for packetsize.\n"); exit(0); } } if (argc != 8) { buffersize = 0; } else { if (sscanf(argv[7], "%d", &buffersize) == 0 || buffersize < 0) { fprintf(stderr, "Invalid value for buffersize\n"); exit(0); } } /* Determine proper buffersize by finding the closest valid buffersize to the input value */ if (buffersize != 0) { if (packetsize != 0 && buffersize%(sizeof(long)*w*k*packetsize) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(long)*w*k*packetsize) != 0 && (down%(sizeof(long)*w*k*packetsize) != 0)) { up++; if (down == 0) { down--; } } if (up%(sizeof(long)*w*k*packetsize) == 0) { buffersize = up; } else { if (down != 0) { buffersize = down; } } } else if (packetsize == 0 && buffersize%(sizeof(long)*w*k) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(long)*w*k) != 0 && down%(sizeof(long)*w*k) != 0) { up++; down--; } if (up%(sizeof(long)*w*k) == 0) { buffersize = up; } else { buffersize = down; } } } /* Setting of coding technique and error checking */ if (strcmp(argv[4], "no_coding") == 0) { tech = No_Coding; } else if (strcmp(argv[4], "reed_sol_van") == 0) { tech = Reed_Sol_Van; if (w != 8 && w != 16 && w != 32) { fprintf(stderr, "w must be one of {8, 16, 32}\n"); exit(0); } } else if (strcmp(argv[4], "reed_sol_r6_op") == 0) { if (m != 2) { fprintf(stderr, "m must be equal to 2\n"); exit(0); } if (w != 8 && w != 16 && w != 32) { fprintf(stderr, "w must be one of {8, 16, 32}\n"); exit(0); } tech = Reed_Sol_R6_Op; } else if (strcmp(argv[4], "cauchy_orig") == 0) { tech = Cauchy_Orig; if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } } else if (strcmp(argv[4], "cauchy_good") == 0) { tech = Cauchy_Good; if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } } else if (strcmp(argv[4], "liberation") == 0) { if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } if (w <= 2 || !(w%2) || !is_prime(w)) { fprintf(stderr, "w must be greater than two and w must be prime\n"); exit(0); } if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } if ((packetsize%(sizeof(long))) != 0) { fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n"); exit(0); } tech = Liberation; } else if (strcmp(argv[4], "blaum_roth") == 0) { if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } if (w <= 2 || !((w+1)%2) || !is_prime(w+1)) { fprintf(stderr, "w must be greater than two and w+1 must be prime\n"); exit(0); } if (packetsize == 0) { fprintf(stderr, "Must include packetsize.\n"); exit(0); } if ((packetsize%(sizeof(long))) != 0) { fprintf(stderr, "packetsize must be a multiple of sizeof(long)\n"); exit(0); } tech = Blaum_Roth; } else if (strcmp(argv[4], "liber8tion") == 0) { if (packetsize == 0) { fprintf(stderr, "Must include packetsize\n"); exit(0); } if (w != 8) { fprintf(stderr, "w must equal 8\n"); exit(0); } if (m != 2) { fprintf(stderr, "m must equal 2\n"); exit(0); } if (k > w) { fprintf(stderr, "k must be less than or equal to w\n"); exit(0); } tech = Liber8tion; } else { fprintf(stderr, "Not a valid coding technique. Choose one of the following: reed_sol_van, reed_sol_r6_op, cauchy_orig, cauchy_good, liberation, blaum_roth, liber8tion, no_coding\n"); exit(0); } /* Set global variable method for signal handler */ method = tech; /* Get current working directory for construction of file names */ curdir = (char*)malloc(sizeof(char)*1000); if (! getcwd(curdir, 1000)) { curdir[0] = 0; } if (argv[1][0] != '-') { /* Open file and error check */ fp = fopen(argv[1], "rb"); if (fp == NULL) { fprintf(stderr, "Unable to open file.\n"); exit(0); } /* Create Coding directory */ i = mkdir("Coding", S_IRWXU); if (i == -1 && errno != EEXIST) { fprintf(stderr, "Unable to create Coding directory.\n"); exit(0); } /* Determine original size of file */ stat(argv[1], &status); size = status.st_size; } else { if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) { fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n"); exit(1); } fp = NULL; MOA_Seed(time(0)); } newsize = size; /* Find new size by determining next closest multiple */ if (packetsize != 0) { if (size%(k*w*packetsize*sizeof(long)) != 0) { while (newsize%(k*w*packetsize*sizeof(long)) != 0) newsize++; } } else { if (size%(k*w*sizeof(long)) != 0) { while (newsize%(k*w*sizeof(long)) != 0) newsize++; } } if (buffersize != 0) { while (newsize%buffersize != 0) { newsize++; } } /* Determine size of k+m files */ blocksize = newsize/k; /* Allow for buffersize and determine number of read-ins */ if (size > buffersize && buffersize != 0) { if (newsize%buffersize != 0) { readins = newsize/buffersize; } else { readins = newsize/buffersize; } block = (char *)malloc(sizeof(char)*buffersize); blocksize = buffersize/k; } else { readins = 1; buffersize = size; block = (char *)malloc(sizeof(char)*newsize); } /* Break inputfile name into the filename and extension */ s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+20)); s2 = strrchr(argv[1], '/'); if (s2 != NULL) { s2++; strcpy(s1, s2); } else { strcpy(s1, argv[1]); } s2 = strchr(s1, '.'); if (s2 != NULL) { extension = strdup(s2); *s2 = '\0'; } else { extension = strdup(""); } /* Allocate for full file name */ fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+20)); sprintf(temp, "%d", k); md = strlen(temp); /* Allocate data and coding */ data = (char **)malloc(sizeof(char*)*k); coding = (char **)malloc(sizeof(char*)*m); for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*blocksize); if (coding[i] == NULL) { perror("malloc"); exit(1); } } /* Create coding matrix or bitmatrix and schedule */ timing_set(&t3); switch(tech) { case No_Coding: break; case Reed_Sol_Van: matrix = reed_sol_vandermonde_coding_matrix(k, m, w); break; case Reed_Sol_R6_Op: break; case Cauchy_Orig: matrix = cauchy_original_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Cauchy_Good: matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Liberation: bitmatrix = liberation_coding_bitmatrix(k, w); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Blaum_Roth: bitmatrix = blaum_roth_coding_bitmatrix(k, w); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case Liber8tion: bitmatrix = liber8tion_coding_bitmatrix(k); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); break; case RDP: case EVENODD: assert(0); } timing_set(&start); timing_set(&t4); totalsec += timing_delta(&t3, &t4); /* Read in data until finished */ n = 1; total = 0; while (n <= readins) { /* Check if padding is needed, if so, add appropriate number of zeros */ if (total < size && total+buffersize <= size) { total += jfread(block, sizeof(char), buffersize, fp); } else if (total < size && total+buffersize > size) { extra = jfread(block, sizeof(char), buffersize, fp); for (i = extra; i < buffersize; i++) { block[i] = '0'; } } else if (total == size) { for (i = 0; i < buffersize; i++) { block[i] = '0'; } } /* Set pointers to point to file data */ for (i = 0; i < k; i++) { data[i] = block+(i*blocksize); } timing_set(&t3); /* Encode according to coding method */ switch(tech) { case No_Coding: break; case Reed_Sol_Van: jerasure_matrix_encode(k, m, w, matrix, data, coding, blocksize); break; case Reed_Sol_R6_Op: reed_sol_r6_encode(k, w, data, coding, blocksize); break; case Cauchy_Orig: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Cauchy_Good: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Liberation: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Blaum_Roth: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case Liber8tion: jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); break; case RDP: case EVENODD: assert(0); } timing_set(&t4); /* Write data and encoded data to k+m files */ for (i = 1; i <= k; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_k%0*d%s", curdir, s1, md, i, extension); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(data[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } for (i = 1; i <= m; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_m%0*d%s", curdir, s1, md, i, extension); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(coding[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } n++; /* Calculate encoding time */ totalsec += timing_delta(&t3, &t4); } /* Create metadata file */ if (fp != NULL) { sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1); fp2 = fopen(fname, "wb"); fprintf(fp2, "%s\n", argv[1]); fprintf(fp2, "%d\n", size); fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize); fprintf(fp2, "%s\n", argv[4]); fprintf(fp2, "%d\n", tech); fprintf(fp2, "%d\n", readins); fclose(fp2); } /* Free allocated memory */ free(s1); free(fname); free(block); free(curdir); /* Calculate rate in MB/sec and print */ timing_set(&t2); tsec = timing_delta(&t1, &t2); printf("Encoding (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/totalsec); printf("En_Total (MB/sec): %0.10f\n", (((double) size)/1024.0/1024.0)/tsec); return 0; }
int main (int argc, char **argv) { FILE *fp, *fp2; // file pointers char *memblock; // reading in file char *block; // padding file int size, newsize; // size of file and temp size struct stat status; // finding file size enum Coding_Technique tech; // coding technique (parameter) int k, m, w, packetsize; // parameters int buffersize; // paramter int i, j; // loop control variables int blocksize; // size of k+m files int total; int extra; /* Jerasure Arguments */ char **data; char **coding; int *matrix; int *bitmatrix; int **schedule; int *erasure; int *erased; /* Creation of file name variables */ char temp[5]; char *s1, *s2; char *fname; int md; char *curdir; /* Timing variables */ struct timeval t1, t2, t3, t4; struct timezone tz; double tsec; double totalsec; struct timeval start, stop; /* Find buffersize */ int up, down; /* Modifications */ int _size, sizeCorrect = 0, numFiles = 0, codingFactor, testValue; /* Start timing */ gettimeofday(&t1, &tz); totalsec = 0.0; matrix = NULL; bitmatrix = NULL; schedule = NULL; printf("\nFilename: %s\n", argv[1]); printf("Coding Technique: cauchy_good\n"); printf("Input desired size of file segments (in MB): "); scanf("%d", &_size); printf("Input codingFactor: "); scanf("%d", &codingFactor); // printf("Input packetsize: "); // scanf("%d", &packetsize); // printf("Input buffersize: "); // scanf("%d", &buffersize); w = 16; packetsize = 32; buffersize = 50000; //_size = _size * 1024 * 1024; /* Set global variable method for signal handler */ tech = Cauchy_Good; method = tech; /* Get current working directory for construction of file names */ curdir = (char*)malloc(sizeof(char)*1000); getcwd(curdir, 1000); if (argv[1][0] != '-') { /* Open file and error check */ fp = fopen(argv[1], "rb"); if (fp == NULL) { fprintf(stderr, "Unable to open file.\n"); exit(0); } /* Create Coding directory */ i = mkdir("Coding", S_IRWXU); if (i == -1 && errno != EEXIST) { fprintf(stderr, "Unable to create Coding directory.\n"); exit(0); } /* Determine original size of file */ stat(argv[1], &status); size = status.st_size; } else { if (sscanf(argv[1]+1, "%d", &size) != 1 || size <= 0) { fprintf(stderr, "Files starting with '-' should be sizes for randomly created input\n"); exit(1); } fp = NULL; srand48(time(0)); } k = size/_size; m = (codingFactor-1) * k; /* Determine proper buffersize by finding the closest valid buffersize to the input value */ if (buffersize != 0) { if (packetsize != 0 && buffersize%(sizeof(int)*w*k*packetsize) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(int)*w*k*packetsize) != 0 && (down%(sizeof(int)*w*k*packetsize) != 0)) { up++; if (down == 0) { down--; } } if (up%(sizeof(int)*w*k*packetsize) == 0) { buffersize = up; } else { if (down != 0) { buffersize = down; } } } else if (packetsize == 0 && buffersize%(sizeof(int)*w*k) != 0) { up = buffersize; down = buffersize; while (up%(sizeof(int)*w*k) != 0 && down%(sizeof(int)*w*k) != 0) { testValue = up%(sizeof(int)*w*k); printf("up%% = %d\n", testValue); up++; down--; } if (up%(sizeof(int)*w*k) == 0) { buffersize = up; } else { buffersize = down; } } } newsize = size; /* Find new size by determining next closest multiple */ if (packetsize != 0) { if (size%(k*w*packetsize*sizeof(int)) != 0) { while (newsize%(k*w*packetsize*sizeof(int)) != 0) newsize++; } } else { if (size%(k*w*sizeof(int)) != 0) { while (newsize%(k*w*sizeof(int)) != 0) newsize++; } } if (buffersize != 0) { while (newsize%buffersize != 0) { newsize++; } } /* Determine size of k+m files */ blocksize = newsize/k; printf("\n"); printf("Size of file segments: %d \tbytes\n", blocksize); printf("Size of buffer: %d \tbytes\n", buffersize); printf("Size of file: %d \tbytes\n", size); printf("Size of word: %d \t\tbytes\n\n", w); printf("Number of data files: %d\n", k); printf("Number of coding files: %d\n", m); /* Allow for buffersize and determine number of read-ins */ if (size > buffersize && buffersize != 0) { if (newsize%buffersize != 0) { readins = newsize/buffersize; } else { readins = newsize/buffersize; } block = (char *)malloc(sizeof(char)*buffersize); blocksize = buffersize/k; } else { readins = 1; buffersize = size; block = (char *)malloc(sizeof(char)*newsize); } /* Break inputfile name into the filename and extension */ s1 = (char*)malloc(sizeof(char)*(strlen(argv[1])+10)); s2 = strrchr(argv[1], '/'); if (s2 != NULL) { s2++; strcpy(s1, s2); } else { strcpy(s1, argv[1]); } s2 = strchr(s1, '.'); if (s2 != NULL) { *s2 = '\0'; } fname = strchr(argv[1], '.'); s2 = (char*)malloc(sizeof(char)*(strlen(argv[1])+5)); if (fname != NULL) { strcpy(s2, fname); } /* Allocate for full file name */ fname = (char*)malloc(sizeof(char)*(strlen(argv[1])+strlen(curdir)+10)); sprintf(temp, "%d", k+m); md = strlen(temp); /* Allocate data and coding */ data = (char **)malloc(sizeof(char*)*k); coding = (char **)malloc(sizeof(char*)*m); for (i = 0; i < m; i++) { coding[i] = (char *)malloc(sizeof(char)*blocksize); } /* Create coding matrix or bitmatrix and schedule */ gettimeofday(&t3, &tz); matrix = cauchy_good_general_coding_matrix(k, m, w); bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix); schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix); gettimeofday(&start, &tz); gettimeofday(&t4, &tz); tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; /* Read in data until finished */ n = 1; total = 0; printf("\nABOUT TO ENCODE SCHEDULE\n"); while (n <= readins) { /* Check if padding is needed, if so, add appropriate number of zeros */ if (total < size && total+buffersize <= size) { total += jfread(block, sizeof(char), buffersize, fp); } else if (total < size && total+buffersize > size) { extra = jfread(block, sizeof(char), buffersize, fp); for (i = extra; i < buffersize; i++) { block[i] = '0'; } } else if (total == size) { for (i = 0; i < buffersize; i++) { block[i] = '0'; } } /* Set pointers to point to file data */ for (i = 0; i < k; i++) { data[i] = block+(i*blocksize); } gettimeofday(&t3, &tz); jerasure_schedule_encode(k, m, w, schedule, data, coding, blocksize, packetsize); gettimeofday(&t4, &tz); /* Write data and encoded data to k+m files */ for (i = 1; i <= k; i++) { if (fp == NULL) { bzero(data[i-1], blocksize); } else { sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(data[i-1], sizeof(char), blocksize, fp2); fclose(fp2); } } for (i; i <= k+m; i++) { if (fp == NULL) { bzero(data[i-1-k], blocksize); } else { sprintf(fname, "%s/Coding/%s_%0*d%s", curdir, s1, md, i-1, s2); if (n == 1) { fp2 = fopen(fname, "wb"); } else { fp2 = fopen(fname, "ab"); } fwrite(coding[i-1-k], sizeof(char), blocksize, fp2); fclose(fp2); } } n++; /* Calculate encoding time */ tsec = 0.0; tsec += t4.tv_usec; tsec -= t3.tv_usec; tsec /= 1000000.0; tsec += t4.tv_sec; tsec -= t3.tv_sec; totalsec += tsec; } /* Create metadata file */ if (fp != NULL) { sprintf(fname, "%s/Coding/%s_meta.txt", curdir, s1); fp2 = fopen(fname, "wb"); fprintf(fp2, "%s\n", argv[1]); fprintf(fp2, "%d\n", size); fprintf(fp2, "%d %d %d %d %d\n", k, m, w, packetsize, buffersize); fprintf(fp2, "%s\n", argv[4]); fprintf(fp2, "%d\n", tech); fprintf(fp2, "%d\n", readins); fclose(fp2); } /* Free allocated memory */ free(s2); free(s1); free(fname); free(block); free(curdir); /* Calculate rate in MB/sec and print */ gettimeofday(&t2, &tz); tsec = 0.0; tsec += t2.tv_usec; tsec -= t1.tv_usec; tsec /= 1000000.0; tsec += t2.tv_sec; tsec -= t1.tv_sec; printf("Encoding (MB/sec): %0.10f\n", (size/1024/1024)/totalsec); printf("En_Total (MB/sec): %0.10f\n", (size/1024/1024)/tsec); }
int decode_MSR_product_matrix_no_output(char **input, size_t input_size, int* erasures, struct coding_info *info) { clock_t clk, tclk; int i, j,c1,c2,tdone,inv; int n = info->req.n; int d = info->req.d; int k = info->req.k; int w = info->req.w; int subpacket_size = input_size/(d-k+1); int num_of_long = subpacket_size/sizeof(long); long *src_pos,*des_pos; int *vector_A=NULL; char **ptrs; int **decode_schedule=NULL; int *erased = NULL; int *bitmatrix_temp = malloc(sizeof(int)*(k-1)*(k-1)*w*w*4); char *data_transformed = malloc(input_size*k); // this is the buffer for tranformed data, i.e., matrix M. The output is the systematic part of // codingmatrix*M, and // we will regenerate the erased data from M using the encoding matrix, which will be written to *output. int *pseudo_erasures = malloc(sizeof(int)*(n*2)); // in order to recompute M, we view it as a systematic MDS code, // sometimes (n+k,k), sometimes (n+d-k+1,d-k+1), thus we over allocate char **M_ptrs = malloc(sizeof(void*)*d*(d-k+1)); // this is the pointer matrix to elements in M. char **data_ptrs = malloc(sizeof(void*)*n); char **coding_ptrs = malloc(sizeof(void*)*n); int* remaining = malloc(sizeof(int)*k); // not erased devices char *buffer1 = malloc(subpacket_size*k*(k-1)*2); // need subpacketsize*k>=(max(4,k-1)+k-1)*sizeof(int), thus put factor of 2 to guarentee it int *buffer1_int = (int*)buffer1; // alternative pointer for buffer1 char *buffer2 = malloc(subpacket_size*k*(k-1)); if(data_transformed==NULL||pseudo_erasures==NULL||data_ptrs==NULL ||coding_ptrs==NULL||buffer1==NULL||buffer2==NULL||M_ptrs==NULL||remaining==NULL){ printf("Can not allocate memory\n"); jerasure_free_schedule(decode_schedule); if(data_ptrs!=NULL)free(data_ptrs); if(coding_ptrs!=NULL)free(coding_ptrs); if(pseudo_erasures!=NULL)free(pseudo_erasures); if(data_transformed!=NULL)free(data_transformed); if(M_ptrs!=NULL)free(M_ptrs); if(buffer1!=NULL)free(buffer1); if(buffer2!=NULL)free(buffer2); if(remaining!=NULL)free(remaining); return(-1); } //set up pointers for matrix M // first k-1 rows, only have S1 for(i=0,c2=0;i<k-1;i++){ c1 = i*(d-k+1); for(j=i;j<k-1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=k-1;j<d-k+1;j++) M_ptrs[c1+j] = NULL; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[j*(d-k+1)+i]; // symmetric matrix, thus there is (i,j) and (j,i) point to the same pointer. } // next k-1 rows for(i=0;i<k-1;i++){ c1 = (k-1+i)*(d-k+1); for(j=i;j<d-k+1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i]; } // next 1 and (d-2k+1) rows, may not exist if(d>2*k-2) { c1 = (2*k-2)*(d-k+1); for(j=k-1;j<d-k+1;j++,c2++) M_ptrs[c1+j] = data_transformed + subpacket_size*c2; for(j=0;j<i;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+k-1]; for(i=2*k-1;i<d;i++){ c1 = i*(d-k+1); for(j=0;j<k;j++) M_ptrs[c1+j] = M_ptrs[(j+k-1)*(d-k+1)+i-k+1]; for(j=k;j<d-k+1;j++) M_ptrs[c1+j] = NULL; } } // first decode the last d-2k+1 columns of T and Z: view it as an (n+k,k) MDS code. Note strictly speaking this might // not be a real (n+k,k) MDS code, but it hardly matters. // before decoding operation, prepare for the pseudoerasure location array if(d>2*k-2){ // only when d>2k-2, T and Z exist for(i=0;i<k;i++) pseudo_erasures[i] = i; for(i=0;i<n;i++){ if(erasures[i]==-1){ pseudo_erasures[i+k] = -1; break; } pseudo_erasures[i+k] = erasures[i] + k; } // make the decoding schedule: we can save the trouble of manually generate this schedule, at the expense of // more computation decode_schedule = jerasure_generate_decoding_schedule(k, n, w, info->subbitmatrix_array[0], pseudo_erasures, 1); for(i=0;i<d-2*k+1;i++){ for(j=0;j<k;j++) data_ptrs[j] = M_ptrs[i+k+(d-k+1)*(k-1+j)]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+subpacket_size*(k+i); ptrs = set_up_ptrs_for_scheduled_decoding(k, n, pseudo_erasures, data_ptrs,coding_ptrs); if (ptrs == NULL){ printf("Can not allocate memory\n"); goto complete; } // assume packetsize = ALIGNMENT for (tdone = 0; tdone < subpacket_size; tdone += ALIGNMENT*w) { jerasure_do_scheduled_operations(ptrs, decode_schedule, ALIGNMENT); for (c1 = 0; c1 < k+n; c1++) ptrs[c1] += (ALIGNMENT*w); } free(ptrs); } //next decode the first column of T and Z: we view this as an (n+d-k+1,d-k+1) erasure codes // first setup the pseudo erasure location vector for(i=0;i<k;i++) pseudo_erasures[i] = i; // in the first columne of the Z matrix, the last d-2k+1 elements are known, but the first k elements are not for(i=0;i<n;i++) { if(erasures[i]==-1){ pseudo_erasures[i+k] = -1; break; } pseudo_erasures[i+k] = erasures[i]+d-k+1; } for(j=0;j<d-k+1;j++) data_ptrs[j] = M_ptrs[(d-k+1)*(k-1+j)+k-1]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+subpacket_size*(k-1); jerasure_schedule_decode_lazy(d-k+1,n,w, info->subbitmatrix_array[1],pseudo_erasures,data_ptrs, coding_ptrs,subpacket_size,ALIGNMENT,0); } //clk = clock(); // now this is the hard part: to decode S1 and S2. The algorithm used here is slightly different from that in the paper: // instead of right multiply \Phi_{DC}', we only right multiply the sub-matrix of \Phi_{DC}' without of the last row // setting up remaining device array to facilitate decoding erased = jerasure_erasures_to_erased(k, n-k, erasures); for(i=0,c1=0;i<n&&c1<k;i++){ if(erased[i]==0){ remaining[c1] = i; c1++; } } //compute C_{DC}-\Delta_{DC}*T' for(i=0;i<k-1;i++){ // has k-1 columns for(j=0;j<d-2*k+2;j++) data_ptrs[j] = M_ptrs[(2*k-2+j)*(d-k+1)+i]; for(j=0;j<k;j++) coding_ptrs[j] = buffer1+(j*(k-1)+i)*subpacket_size; for(j=0;j<k;j++){ jerasure_bitmatrix_dotprod(d-2*k+2, w, info->subbitmatrix_array[2]+remaining[j]*(d-2*k+2)*w*w, NULL, j+d-2*k+2, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); src_pos = (long*)(input[remaining[j]]+i*subpacket_size); des_pos = (long*)(coding_ptrs[j]); for(c2=0;c2<num_of_long;c2++) des_pos[c2] ^= src_pos[c2]; } } // right multiply \Phi_{DC}': this will be P. // result is in buffer2 for(j=0;j<k;j++){ //j-th row for(i=0;i<k-1;i++) data_ptrs[i] = buffer1+(j*(k-1)+i)*subpacket_size; for(i=0;i<k-1;i++) coding_ptrs[i] = buffer2 +(j*(k-1)+i)*subpacket_size; for(i=0;i<k-1;i++){ jerasure_bitmatrix_dotprod(k-1, w, info->subbitmatrix_array[3]+remaining[i]*(k-1)*w*w, NULL, i+k-1, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } } // now solve for the off-diagonal terms for(i=0;i<k-1;i++){ for(j=i+1;j<k-1;j++){ // solve for S1 tilde off-diagonal // here we directly use the fact that Lambda = [0 1 2 3 ....]; int** temp_schedule; inv = galois_single_divide(1,remaining[i]^remaining[j],w); buffer1_int[0] = buffer1_int[1] = inv; data_ptrs[0] = buffer2+(i*(k-1)+j)*subpacket_size; data_ptrs[1] = buffer2+(j*(k-1)+i)*subpacket_size; coding_ptrs[0] = M_ptrs[i*(d-k+1)+j]; jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp); temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp); jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT); if(temp_schedule!=NULL){ jerasure_free_schedule(temp_schedule); temp_schedule = NULL; } // solve for S2 tilde off-diagonal buffer1_int[0] = galois_single_multiply(remaining[j],inv,w); buffer1_int[1] = galois_single_multiply(remaining[i],inv,w); coding_ptrs[0] = M_ptrs[(i+k-1)*(d-k+1)+j]; jerasure_matrix_to_bitmatrix_noallocate(2,1,w,buffer1_int,bitmatrix_temp); temp_schedule = jerasure_smart_bitmatrix_to_schedule(2, 1, w, bitmatrix_temp); jerasure_schedule_encode(2, 1, w, temp_schedule, data_ptrs, coding_ptrs,subpacket_size, ALIGNMENT); if(temp_schedule!=NULL){ jerasure_free_schedule(temp_schedule); temp_schedule = NULL; } } } //tclk = clock()-clk; //printf("~S1 and ~S2 off-diagonal decoded %.3e clocks \n", (double)tclk); //clk = clock(); // compute the A vector: A*\Phi_{DC1} = \Phi_{DC2}, note this is always possible because \Phi_{DC1} is alway full rank by construction // we first reuse buffer1 here to form \Phi_{DC1} matrix and then the compute its inverse for(i=0;i<k-1;i++){ memcpy(buffer1_int+(k-1)*(k-1+i),(void*)(info->matrix+remaining[i]*d+k-1),(k-1)*sizeof(int)); } jerasure_invert_matrix(buffer1_int+(k-1)*(k-1),buffer1_int,k-1,w); vector_A = jerasure_matrix_multiply(info->matrix+remaining[k-1]*d+k-1,buffer1_int,1,k-1,k-1,k-1,w); if(vector_A==NULL) goto complete; for(i=0;i<k-1;i++){ buffer1_int[i+(k-1)*(k-1)] = galois_single_multiply(vector_A[i],remaining[k-1],w); buffer1_int[i+k*(k-1)] = vector_A[i]; } for(i=0;i<k-1;i++){ memset(buffer1_int+(k-1)*(k+1),0,sizeof(int)*2*(k-1)); buffer1_int[(k-1)*(k+1)+i] = remaining[i]; buffer1_int[(k-1)*(k+2)+i] = 1; pseudo_erasures[0] = i; pseudo_erasures[1] = k-1+i; pseudo_erasures[2] = -1; for(j=0;j<2*k-2;j++) data_ptrs[j] = M_ptrs[i+j*(d-k+1)]; coding_ptrs[0] = buffer2+((k-1)*(k-1)+i)*subpacket_size; coding_ptrs[1] = buffer2+(i*(k-1)+i)*subpacket_size; jerasure_matrix_to_bitmatrix_noallocate(2*k-2,2,w,buffer1_int+(k-1)*(k-1),bitmatrix_temp); jerasure_schedule_decode_lazy(2*k-2,2,w,bitmatrix_temp,pseudo_erasures,data_ptrs,coding_ptrs,subpacket_size,ALIGNMENT,0); } //tclk = clock()-clk; //printf("~S1 and ~S2 decoded %.3e clocks \n", (double)tclk); // now we have both \tilde{S_1} and \tilde{S_2} in M_ptrs, need to recover S1 and S2 from them // this is done by multiply \tilde{S_1} left and right by inv(\Phi_{DC1}). // right-multiply for S1 int* bitmatrix_inv = jerasure_matrix_to_bitmatrix(k-1,k-1,w,buffer1_int); int** inv_schedule = jerasure_smart_bitmatrix_to_schedule(k-1, k-1, w, bitmatrix_inv); // right-multiply for S1 for(i=0;i<k-1;i++){ for(j=0;j<k-1;j++) data_ptrs[j] = M_ptrs[i*(d-k+1)+j]; for(j=0;j<k-1;j++) coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // left-multiply for S1 for(j=0;j<k-1;j++){ for(i=0;i<k-1;i++) data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size; for(i=0;i<k-1;i++) coding_ptrs[i] = M_ptrs[i*(d-k+1)+j]; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // right-multiply for S2 for(i=0;i<k-1;i++){ for(j=0;j<k-1;j++) data_ptrs[j] = M_ptrs[(i+k-1)*(d-k+1)+j]; for(j=0;j<k-1;j++) coding_ptrs[j] = buffer2+(i*(k-1)+j)*subpacket_size; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // left-multiply for S2 for(j=0;j<k-1;j++){ for(i=0;i<k-1;i++) data_ptrs[i] = buffer2+(i*(k-1)+j)*subpacket_size; //for(i=j;i<k-1;i++) for(i=0;i<k-1;i++) coding_ptrs[i] = M_ptrs[(i+k-1)*(d-k+1)+j]; jerasure_schedule_encode(k-1, k-1, w, inv_schedule, data_ptrs, coding_ptrs, subpacket_size, ALIGNMENT); } // having S1,S2,T, now can also fill the first k-1 column of the output for(i=0;i<k-1;i++){ for(j=0;j<d;j++) data_ptrs[j] = M_ptrs[j*(d-k+1)+i]; for(j=0;j<n;j++) coding_ptrs[j] = input[j]+i*subpacket_size; for(j=0;j<n;j++){ if(erased[j]==1) jerasure_bitmatrix_encode(d,1,w,info->bitmatrix+(j*d*w*w),data_ptrs,coding_ptrs+j,subpacket_size,ALIGNMENT); } } // clean up complete: if(decode_schedule) jerasure_free_schedule(decode_schedule); if(inv_schedule) jerasure_free_schedule(inv_schedule); free(data_ptrs); free(coding_ptrs); if(erased)free(erased); free(pseudo_erasures); free(data_transformed); free(M_ptrs); free(buffer1); free(buffer2); free(remaining); free(bitmatrix_temp); if(vector_A!=NULL)free(vector_A); return(1); }