int inv_test(u8 * in, u8 * inv, u8 * sav, int n)
{
memcpy(sav, in, n * n);
if (gf_invert_matrix(in, inv, n)) {
printf("Given singular matrix\n");
print_matrix(sav, n);
return -1;
}
matrix_mult(inv, sav, in, n);
if (is_ident(in, n)) {
printf("fail\n");
print_matrix(sav, n);
print_matrix(inv, n);
print_matrix(in, n);
return -1;
}
putchar('.');
return 0;
}
// Generate decode matrix from encode matrix
static int gf_gen_decode_matrix(unsigned char *encode_matrix,
unsigned char *decode_matrix,
unsigned char *invert_matrix,
unsigned int *decode_index,
unsigned char *src_err_list,
unsigned char *src_in_err,
int nerrs, int nsrcerrs, int k, int m)
{
int i, j, p;
int r;
unsigned char *backup, *b, s;
int incr = 0;
b = malloc(MMAX * KMAX);
backup = malloc(MMAX * KMAX);
if (b == NULL || backup == NULL) {
printf("Test failure! Error with malloc\n");
free(b);
free(backup);
return -1;
}
// Construct matrix b by removing error rows
for (i = 0, r = 0; i < k; i++, r++) {
while (src_in_err[r])
r++;
for (j = 0; j < k; j++) {
b[k * i + j] = encode_matrix[k * r + j];
backup[k * i + j] = encode_matrix[k * r + j];
}
decode_index[i] = r;
}
incr = 0;
while (gf_invert_matrix(b, invert_matrix, k) < 0) {
if (nerrs == (m - k)) {
free(b);
free(backup);
printf("BAD MATRIX\n");
return NO_INVERT_MATRIX;
}
incr++;
memcpy(b, backup, MMAX * KMAX);
for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) {
if (src_err_list[i] == (decode_index[k - 1] + incr)) {
// skip the erased parity line
incr++;
continue;
}
}
if (decode_index[k - 1] + incr >= m) {
free(b);
free(backup);
printf("BAD MATRIX\n");
return NO_INVERT_MATRIX;
}
decode_index[k - 1] += incr;
for (j = 0; j < k; j++)
b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j];
};
for (i = 0; i < nsrcerrs; i++) {
for (j = 0; j < k; j++) {
decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j];
}
}
/* src_err_list from encode_matrix * invert of b for parity decoding */
for (p = nsrcerrs; p < nerrs; p++) {
for (i = 0; i < k; i++) {
s = 0;
for (j = 0; j < k; j++)
s ^= gf_mul(invert_matrix[j * k + i],
encode_matrix[k * src_err_list[p] + j]);
decode_matrix[k * p + i] = s;
}
}
free(b);
free(backup);
return 0;
}
int main(int argc, char *argv[])
{
int i, j, rtest, m, k, nerrs, r;
void *buf;
u8 *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
u8 a[MMAX * KMAX], b[MMAX * KMAX], c[MMAX * KMAX], d[MMAX * KMAX];
u8 g_tbls[KMAX * TEST_SOURCES * 32], src_in_err[TEST_SOURCES];
u8 src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];
struct perf start, stop;
// Pick test parameters
m = 14;
k = 10;
nerrs = 4;
const u8 err_list[] = {2, 4, 5, 7};
printf("erasure_code_base_perf: %dx%d %d\n", m, TEST_LEN(m), nerrs);
if (m > MMAX || k > KMAX || nerrs > (m - k)){
printf(" Input test parameter error\n");
return -1;
}
memcpy(src_err_list, err_list, nerrs);
memset(src_in_err, 0, TEST_SOURCES);
for (i = 0; i < nerrs; i++)
src_in_err[src_err_list[i]] = 1;
// Allocate the arrays
for (i = 0; i < m; i++) {
if (posix_memalign(&buf, 64, TEST_LEN(m))) {
printf("alloc error: Fail\n");
return -1;
}
buffs[i] = buf;
}
for (i = 0; i < (m - k); i++) {
if (posix_memalign(&buf, 64, TEST_LEN(m))) {
printf("alloc error: Fail\n");
return -1;
}
temp_buffs[i] = buf;
}
// Make random data
for (i = 0; i < k; i++)
for (j = 0; j < TEST_LEN(m); j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
ec_init_tables(k, m - k, &a[k * k], g_tbls);
ec_encode_data_base(TEST_LEN(m), k, m - k, g_tbls, buffs, &buffs[k]);
// Start encode test
perf_start(&start);
for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
// Make parity vects
ec_init_tables(k, m - k, &a[k * k], g_tbls);
ec_encode_data_base(TEST_LEN(m), k, m - k, g_tbls, buffs, &buffs[k]);
}
perf_stop(&stop);
printf("erasure_code_base_encode" TEST_TYPE_STR ": ");
perf_print(stop, start, (long long)(TEST_LEN(m)) * (m) * rtest);
// Start decode test
perf_start(&start);
for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
// Construct b by removing error rows
for (i = 0, r = 0; i < k; i++, r++) {
while (src_in_err[r])
r++;
recov[i] = buffs[r];
for (j = 0; j < k; j++)
b[k * i + j] = a[k * r + j];
}
if (gf_invert_matrix(b, d, k) < 0) {
printf("BAD MATRIX\n");
return -1;
}
for (i = 0; i < nerrs; i++)
for (j = 0; j < k; j++)
c[k * i + j] = d[k * src_err_list[i] + j];
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data_base(TEST_LEN(m), k, nerrs, g_tbls, recov, temp_buffs);
}
perf_stop(&stop);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_buffs[i], buffs[src_err_list[i]], TEST_LEN(m))) {
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
return -1;
}
}
printf("erasure_code_base_decode" TEST_TYPE_STR ": ");
perf_print(stop, start, (long long)(TEST_LEN(m)) * (k + nerrs) * rtest);
printf("done all: Pass\n");
return 0;
}
int main(int argc, char *argv[])
{
int i, j, rtest, srcs, m, k, nerrs, r, err;
void *buf;
u8 g[TEST_SOURCES], g_tbls[TEST_SOURCES*32], src_in_err[TEST_SOURCES];
u8 *dest, *dest_ref, *temp_buff, *buffs[TEST_SOURCES];
u8 a[MMAX*KMAX], b[MMAX*KMAX], d[MMAX*KMAX];
u8 src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];
int align, size;
unsigned char *efence_buffs[TEST_SOURCES];
unsigned int offset;
u8 *ubuffs[TEST_SOURCES];
u8 *udest_ptr;
printf("gf_vect_dot_prod_sse: %dx%d ", TEST_SOURCES, TEST_LEN);
mk_gf_field();
// Allocate the arrays
for(i=0; i<TEST_SOURCES; i++){
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
buffs[i] = buf;
}
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
dest = buf;
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
dest_ref = buf;
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
temp_buff = buf;
// Test of all zeros
for(i=0; i<TEST_SOURCES; i++)
memset(buffs[i], 0, TEST_LEN);
memset(dest, 0, TEST_LEN);
memset(temp_buff, 0, TEST_LEN);
memset(dest_ref, 0, TEST_LEN);
memset(g, 0, TEST_SOURCES);
for(i=0; i<TEST_SOURCES; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(TEST_LEN, TEST_SOURCES, &g_tbls[0], buffs, dest_ref);
gf_vect_dot_prod_sse(TEST_LEN, TEST_SOURCES, g_tbls, buffs, dest);
if (0 != memcmp(dest_ref, dest, TEST_LEN)){
printf("Fail zero vect_dot_prod_sse test\n");
dump_matrix(buffs, 5, TEST_SOURCES);
printf("dprod_base:");
dump(dest_ref, 25);
printf("dprod_sse:");
dump(dest, 25);;
return -1;
}
else
putchar('.');
// Rand data test
for(rtest=0; rtest<RANDOMS; rtest++){
for(i=0; i<TEST_SOURCES; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
for (i=0; i<TEST_SOURCES; i++)
g[i] = rand();
for(i=0; i<TEST_SOURCES; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(TEST_LEN, TEST_SOURCES, &g_tbls[0], buffs, dest_ref);
gf_vect_dot_prod_sse(TEST_LEN, TEST_SOURCES, g_tbls, buffs, dest);
if (0 != memcmp(dest_ref, dest, TEST_LEN)){
printf("Fail rand vect_dot_prod_sse test 1\n");
dump_matrix(buffs, 5, TEST_SOURCES);
printf("dprod_base:");
dump(dest_ref, 25);
printf("dprod_sse:");
dump(dest, 25);
return -1;
}
putchar('.');
}
// Rand data test with varied parameters
for(rtest=0; rtest < RANDOMS; rtest++){
for (srcs = TEST_SOURCES; srcs > 0; srcs--){
for(i=0; i<srcs; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
for (i=0; i<srcs; i++)
g[i] = rand();
for(i=0; i<srcs; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(TEST_LEN, srcs, &g_tbls[0], buffs, dest_ref);
gf_vect_dot_prod_sse(TEST_LEN, srcs, g_tbls, buffs, dest);
if (0 != memcmp(dest_ref, dest, TEST_LEN)){
printf("Fail rand vect_dot_prod_sse test 2\n");
dump_matrix(buffs, 5, srcs);
printf("dprod_base:");
dump(dest_ref, 5);
printf("dprod_sse:");
dump(dest, 5);
return -1;
}
putchar('.');
}
}
// Test erasure code using gf_vect_dot_prod
// Pick a first test
m = 9;
k = 5;
if (m > MMAX || k > KMAX)
return -1;
gf_gen_rs_matrix(a, m, k);
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
// Make parity vects
for (i=k; i<m; i++) {
for (j=0; j<k; j++)
gf_vect_mul_init(a[k*i+j], &g_tbls[j*32]);
#ifndef USEREF
gf_vect_dot_prod_sse(TEST_LEN,
k, g_tbls, buffs, buffs[i]);
#else
gf_vect_dot_prod_base(TEST_LEN,
k, &g_tbls[0], buffs, buffs[i]);
#endif
}
// Random buffers in erasure
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
// construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r]) {
r++;
continue;
}
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix((u8*)b, (u8*)d, k) < 0)
printf("BAD MATRIX\n");
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r]) {
r++;
continue;
}
recov[i] = buffs[r];
}
// Recover data
for(i=0; i<nerrs; i++){
for (j=0; j<k; j++)
gf_vect_mul_init(d[k*src_err_list[i]+j], &g_tbls[j*32]);
#ifndef USEREF
gf_vect_dot_prod_sse(TEST_LEN,
k, g_tbls, recov, temp_buff);
#else
gf_vect_dot_prod_base(TEST_LEN,
k, &g_tbls[0], recov, temp_buff);
#endif
if (0 != memcmp(temp_buff, buffs[src_err_list[i]],
TEST_LEN)){
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf("recov %d:",src_err_list[i]);
dump(temp_buff, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
return -1;
}
}
// Do more random tests
for (rtest = 0; rtest < RANDOMS; rtest++){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
gf_gen_rs_matrix(a, m, k);
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
// Make parity vects
for (i=k; i<m; i++) {
for (j=0; j<k; j++)
gf_vect_mul_init(a[k*i+j], &g_tbls[j*32]);
#ifndef USEREF
gf_vect_dot_prod_sse(TEST_LEN, k, g_tbls, buffs, buffs[i]);
#else
gf_vect_dot_prod_base(TEST_LEN, k, &g_tbls[0], buffs, buffs[i]);
#endif
}
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r]) {
r++;
continue;
}
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix((u8*)b, (u8*)d, k) < 0)
printf("BAD MATRIX\n");
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r]) {
r++;
continue;
}
recov[i] = buffs[r];
}
// Recover data
for(i=0; i<nerrs; i++){
for (j=0; j<k; j++)
gf_vect_mul_init(d[k*src_err_list[i]+j], &g_tbls[j*32]);
#ifndef USEREF
gf_vect_dot_prod_sse(TEST_LEN, k, g_tbls, recov, temp_buff);
#else
gf_vect_dot_prod_base(TEST_LEN, k, &g_tbls[0], recov, temp_buff);
#endif
if (0 != memcmp(temp_buff, buffs[src_err_list[i]],
TEST_LEN)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((u8*)a, m, k);
printf("inv b:\n");
dump_u8xu8((u8*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buff, 25);
return -1;
}
}
putchar('.');
}
// Run tests at end of buffer for Electric Fence
align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
for(size=EFENCE_TEST_MIN_SIZE; size<=TEST_SIZE; size+=align){
for(i=0; i<TEST_SOURCES; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
for(i=0; i<TEST_SOURCES; i++) // Line up TEST_SIZE from end
efence_buffs[i] = buffs[i] + TEST_LEN - size;
for (i=0; i<TEST_SOURCES; i++)
g[i] = rand();
for(i=0; i<TEST_SOURCES; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(size, TEST_SOURCES, &g_tbls[0], efence_buffs, dest_ref);
gf_vect_dot_prod_sse(size, TEST_SOURCES, g_tbls, efence_buffs, dest);
if (0 != memcmp(dest_ref, dest, size)){
printf("Fail rand vect_dot_prod_sse test 3\n");
dump_matrix(efence_buffs, 5, TEST_SOURCES);
printf("dprod_base:");
dump(dest_ref, align);
printf("dprod_sse:");
dump(dest, align);
return -1;
}
putchar('.');
}
// Test rand ptr alignment if available
for(rtest=0; rtest<RANDOMS; rtest++){
size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;
srcs = rand() % TEST_SOURCES;
if (srcs == 0)
continue;
offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
// Add random offsets
for(i=0; i<srcs; i++)
ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
udest_ptr = dest + (rand() & (PTR_ALIGN_CHK_B - offset));
memset(dest, 0, TEST_LEN); // zero pad to check write-over
for(i=0; i<srcs; i++)
for(j=0; j<size; j++)
ubuffs[i][j] = rand();
for (i=0; i<srcs; i++)
g[i] = rand();
for(i=0; i<srcs; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(size, srcs, &g_tbls[0], ubuffs, dest_ref);
gf_vect_dot_prod_sse(size, srcs, g_tbls, ubuffs, udest_ptr);
if (memcmp(dest_ref, udest_ptr, size)){
printf("Fail rand vect_dot_prod_sse test ualign srcs=%d\n", srcs);
dump_matrix(ubuffs, 5, TEST_SOURCES);
printf("dprod_base:");
dump(dest_ref, 25);
printf("dprod_sse:");
dump(udest_ptr, 25);
return -1;
}
// Confirm that padding around dests is unchanged
memset(dest_ref, 0, PTR_ALIGN_CHK_B); // Make reference zero buff
offset = udest_ptr - dest;
if (memcmp(dest, dest_ref, offset)){
printf("Fail rand ualign pad start\n");
return -1;
}
if (memcmp(dest + offset + size, dest_ref, PTR_ALIGN_CHK_B - offset)){
printf("Fail rand ualign pad end\n");
return -1;
}
putchar('.');
}
// Test all size alignment
align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
for(size=TEST_LEN; size>15; size-=align){
srcs = TEST_SOURCES;
for(i=0; i<srcs; i++)
for(j=0; j<size; j++)
buffs[i][j] = rand();
for (i=0; i<srcs; i++)
g[i] = rand();
for(i=0; i<srcs; i++)
gf_vect_mul_init(g[i], &g_tbls[i*32]);
gf_vect_dot_prod_base(size, srcs, &g_tbls[0], buffs, dest_ref);
gf_vect_dot_prod_sse(size, srcs, g_tbls, buffs, dest);
if (memcmp(dest_ref, dest, size)){
printf("Fail rand vect_dot_prod_sse test ualign len=%d\n", size);
dump_matrix(buffs, 5, TEST_SOURCES);
printf("dprod_base:");
dump(dest_ref, 25);
printf("dprod_sse:");
dump(dest, 25);
return -1;
}
}
printf("done all: Pass\n");
return 0;
}
main(int argc, char **argv)
{
int i, j, k, *vdm, *inv, *prod, cache_size;
int rows, cols, blocksize, orig_size;
int n, m, sz, *factors, tmp, factor, *exists, *map;
char *stem, *filename,*file_name;
char **buffer, *buf_file, *block;
struct stat buf;
Condensed_Matrix *cm;
int *mat, *id;
FILE *f;
if (argc != 3) /*modified by supriya*/{
fprintf(stderr, "usage: rs_decode_file stem file_name\n");
exit(1);
}
file_name=argv[2];/*added by supriya*/
stem = argv[1];
buf_file = (char *) malloc(sizeof(char)*(strlen(stem)+30));
if (buf_file == NULL) { perror("malloc - buf_file"); exit(1); }
sprintf(buf_file, "%s-info.txt", stem, i);
f = fopen(buf_file, "r");
if (f == NULL) { perror(buf_file); exit(1); }
if (fscanf(f, "%d\n", &orig_size) != 1) { fprintf(stderr, "Error reading info file 1\n"); exit(1); }
if (fscanf(f, "%d\n", &sz) != 1) { fprintf(stderr, "Error reading info file 2\n"); exit(1); }
if (fscanf(f, "%d\n", &blocksize) != 1) { fprintf(stderr, "Error reading info file 3\n"); exit(1); }
if (fscanf(f, "%d\n", &n) != 1) { fprintf(stderr, "Error reading info file 4\n"); exit(1); }
if (fscanf(f, "%d\n", &m) != 1) { fprintf(stderr, "Error reading info file 5\n"); exit(1); }
vdm = gf_read_matrix(f, &rows, &cols);
if (vdm == NULL) { fprintf(stderr, "Error reading info file matrix\n"); exit(1); }
fclose(f);
if (rows != n+m) {
fprintf(stderr, "Error in %s - rows != n+m\n", buf_file);
exit(1);
}
if (cols != n) {
fprintf(stderr, "Error in %s - cols != n\n", buf_file);
exit(1);
}
exists = (int *) malloc(sizeof(int) * rows);
if (exists == NULL) { perror("malloc - exists"); exit(1); }
factors = (int *) malloc(sizeof(int) * rows);
if (factors == NULL) { perror("malloc - factors"); exit(1); }
map = (int *) malloc(sizeof(int) * rows);
if (map == NULL) { perror("malloc - map"); exit(1); }
buffer = (char **) malloc(sizeof(char *)*n);
for (i = 0; i < n; i++) {
buffer[i] = (char *) malloc(blocksize);
if (buffer[i] == NULL) {
perror("Allocating buffer to store the whole file");
exit(1);
}
}
/* Added by : Supriya */
/*Socket program in order to fetch all the encoded files from different nodes*/
int clientsocket;/* Socket descriptor for client */
int enable=1,num=n,flag=0,p=0,pos,port=7855;
char file[40]="",stem_filename[40]="",ip_addr[40]="",ack[10];
char sup[1024]="",c,line[100]="",search_string[30]="";
int bytes_received,y;
strcpy(search_string,file_name);
//printf("\n%s***",search_string);
struct sockaddr_in serverAddr;/* client address */
socklen_t addr_size;
/*opening the master to file*/
FILE *fp;
fp=fopen("Master_file.txt","r");
if(!fp)
{
perror("\ncould not find the file");
exit(0);
}
while ( c!= EOF )/* read a line */
{
c=fscanf( fp, "%s",line ) ;
//printf("%s\n",line);
if(strstr(line,search_string)){
p=1;
break;
}
}
//printf("ssss");
pos=ftell(fp);
//printf("%d\n",n);
pos=pos-strlen(search_string);
fseek( fp, pos, SEEK_SET );
if(p!=1){
perror("\nNo such File exists");
exit(0);
}
//printf("%d\n",fseek( fp, n, SEEK_SET ));
//c=fscanf( fp, "%s",line ) ;
//printf("%s\n",line);
printf("%d\n",num);
while(num>0)
{
strcpy(sup,"");
/*reading the master file to get the ip address of different nodes*/
fscanf(fp,"%s",file);
fscanf(fp,"%s",stem_filename);
fscanf(fp,"%s",ip_addr);
//fscanf(fp,"%s",port);
//printf("%s...%s\n",file,stem_filename);
/* Create socket for connections */
clientsocket=socket(PF_INET,SOCK_STREAM,0);
printf("\n\nSocket creation: %s\n",strerror(errno));
//printf("%d\n",clientsocket);
port=port+1;
num=num-1;
/* Construct local address structure */
serverAddr.sin_family=AF_INET; /* Internet address family */
serverAddr.sin_port = htons(port);/*server port*/
serverAddr.sin_addr.s_addr=inet_addr(ip_addr);/*server address*/
memset(serverAddr.sin_zero,'\0',sizeof serverAddr.sin_zero);/* Zero out structure */
printf("Before connect\n");
/*reuse the same port*/
if (setsockopt(clientsocket, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0)
error("setsockopt(SO_REUSEADDR) failed");
/*connect to the nodes to receive the encoded files*/
connect(clientsocket,(struct sockaddr*)&serverAddr,sizeof(serverAddr));
printf("Connection: %s\n",strerror(errno));
//printf("Connected to the node with ip address : %s\n",ip_addr);
printf("After connect\n");
send(clientsocket,"Decode",7,0);
bytes_received=recv(clientsocket,ack,sizeof(ack),0);
send(clientsocket,stem_filename,strlen(stem_filename),0);
bytes_received=recv(clientsocket,sup,sizeof(sup),0);
sup[bytes_received]='\0';
printf("**************");
/*checking if the file has been received or not */
if(strcmp(sup,"File does not exist")!=0){
/*write the received content into file*/
FILE *ft;
ft=fopen(stem_filename,"w");
//flag = fwrite(sup, sizeof(char), bytes_received, f);
printf("content: %s\n",sup);
fprintf(ft,"%s",sup);
flag++;
fclose(ft);
}
else{
printf("%s \n",sup);
}
//printf("%d\t%d\n",num,port);
}
fclose(fp);
//fflush(stdin);
if(flag<n){
fprintf(stderr, "Only %d fragments -- need %d. Sorry\n", flag, n);
exit(1);
}
/*end of socket program*/
j = 0;
for (i = 0; i < rows && j < cols; i++) {
sprintf(buf_file, "%s-%04d.rs", stem, i);
if (stat(buf_file, &buf) != 0) {
map[i] = -1;
} else {
if (buf.st_size != blocksize) {
map[i] = -1;
} else {
map[i] = j++;
f = fopen(buf_file, "r");
if (f == NULL) { perror(buf_file); exit(1); }
k = fread(buffer[map[i]], 1, blocksize, f);
if (k != blocksize) {
fprintf(stderr, "%s -- stat says %d bytes, but only read %d\n",
buf_file, buf.st_size, k);
exit(1);
}
}
}
}
printf("\n********************");
/*if (j < cols) {
fprintf(stderr, "Only %d fragments -- need %d. Sorry\n", j, cols);
exit(1);
}*/
j = 0;
for (i = 0; i < cols; i++) if (map[i] == -1) j++;
fprintf(stderr, "Blocks to decode: %d\n", j);
if (j == 0) {
cache_size = orig_size;
for (i = 0; i < cols; i++) {
if (cache_size > 0) {
fwrite(buffer[i], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
}
}
exit(0);
}
block = (char *) malloc(sizeof(char)*blocksize);
if (block == NULL) { perror("malloc - block"); exit(1); }
for (i = 0; i < rows; i++) exists[i] = (map[i] != -1);
cm = gf_condense_dispersal_matrix(vdm, exists, rows, cols);
mat = cm->condensed_matrix;
id = cm->row_identities;
/* Fix it so that map[i] for i = 0 to cols-1 is defined correctly.
map[i] is the index of buffer[] that holds the blocks for row i in
the condensed matrix */
for (i = 0; i < cols; i++) {
if (map[i] == -1) map[i] = map[id[i]];
}
fprintf(stderr, "Inverting condensed dispersal matrix ... "); fflush(stderr);
inv = gf_invert_matrix(mat, cols);
if (inv == NULL) {
fprintf(stderr, "\n\nError -- matrix unvertible\n");
exit(1);
}
fprintf(stderr, "Done\n"); fflush(stderr);
fprintf(stderr, "\nCondensed matrix:\n\n");
gf_fprint_matrix(stderr, mat, cols, cols);
fprintf(stderr, "\nInverted matrix:\n\n");
gf_fprint_matrix(stderr, inv, cols, cols);
for(i = 0; i < rows; i++) factors[i] = 1;
cache_size = orig_size;
for (i = 0; i < cols && cache_size > 0; i++) {
if (id[i] < cols) {
fprintf(stderr, "Writing block %d from memory ... ", i); fflush(stderr);
if (factors[i] != 1) {
tmp = gf_single_divide(1, factors[i]);
/* fprintf(stderr, "Factor = %3d. Tmp = %3d. Before[0] = %3d. ",
factors[i], tmp, (unsigned char) buffer[map[i]][0]); */
factors[i] = 1;
gf_mult_region(buffer[map[i]], blocksize, tmp);
/* fprintf(stderr, "After[0] = %3d.\n", (unsigned char) buffer[map[i]][0]); */
} else {
/* fprintf(stderr, "Factor = %3d. Buffer[0] = %3d.\b", factors[i],
(unsigned char) buffer[map[i]][0]); */
}
fwrite(buffer[map[i]], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
fprintf(stderr, "Done\n"); fflush(stderr);
} else {
fprintf(stderr, "Decoding block %d ... ", i); fflush(stderr);
memset(block, 0, blocksize);
for (j = 0; j < cols; j++) {
tmp = inv[i*cols+j];
factor = gf_single_divide(tmp, factors[j]);
/* fprintf(stderr, "Factors[%d] = %3d. Tmp = %3d. Factor = %3d\n Before[j][0] = %3d. ",
j, factors[j], tmp, factor, (unsigned char) buffer[map[j]][0]); */
factors[j] = tmp;
gf_mult_region(buffer[map[j]], blocksize, factor);
/* fprintf(stderr, "After[j][0] = %3d. ", (unsigned char) buffer[map[j]][0]);
fprintf(stderr, "Before-block[0] = %3d. ", (unsigned char) block[0]); */
gf_add_parity(buffer[map[j]], block, blocksize);
/* fprintf(stderr, "After-block[0] = %3d.\n", (unsigned char) block[0]); */
}
fprintf(stderr, "writing ... "); fflush(stderr);
fwrite(block, 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
fprintf(stderr, "Done\n"); fflush(stderr);
}
}
}
int main(int argc, char *argv[])
{
int i, j, rtest, m, k, nerrs, r, err;
void *buf;
u8 *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
u8 a[MMAX*KMAX], b[MMAX*KMAX], c[MMAX*KMAX], d[MMAX*KMAX];
u8 g_tbls[KMAX*TEST_SOURCES*32], src_in_err[TEST_SOURCES];
u8 src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];
struct perf start, stop;
m = 32;
k = 28;
printf("erasure_code_sse_perf: %dx%d ",
m, (TEST_LEN(m)));
// Allocate the arrays
for(i=0; i<TEST_SOURCES; i++) {
if (posix_memalign(&buf, 64, TEST_LEN(m))) {
printf("alloc error: Fail");
return -1;
}
buffs[i] = buf;
}
for (i=0; i<TEST_SOURCES; i++) {
if (posix_memalign(&buf, 64, TEST_LEN(m))) {
printf("alloc error: Fail");
return -1;
}
temp_buffs[i] = buf;
}
// Test erasure code by encode and recovery
// Pick a first test
if (m > MMAX || k > KMAX)
return -1;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<(TEST_LEN(m)); j++)
buffs[i][j] = rand();
memset(src_in_err, 0, TEST_SOURCES);
srand(1);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++) {
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0) { // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
printf("Test erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\n");
perf_start(&start);
for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse((TEST_LEN(m)),
k, m-k, g_tbls, buffs, &buffs[k]);
}
perf_stop(&stop);
printf("erasure_code_sse_encode" TEST_TYPE_STR ": ");
perf_print(stop,start,
(long long)(TEST_LEN(m))*(m)*rtest);
perf_start(&start);
for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++) {
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0) {
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++) {
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++) {
for(j=0; j<k; j++) {
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data_sse((TEST_LEN(m)),
k, nerrs, g_tbls, recov, &temp_buffs[k]);
}
perf_stop(&stop);
for(i=0; i<nerrs; i++) {
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]],
(TEST_LEN(m)))) {
printf("Fail error recovery (%d, %d, %d) - ",
m, k, nerrs);
printf(" - erase list = ");
for (j=0; j<nerrs; j++)
printf(" %d", src_err_list[j]);
printf("\na:\n");
dump_u8xu8((u8*)a, m, k);
printf("inv b:\n");
dump_u8xu8((u8*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
return -1;
}
}
printf("erasure_code_sse_decode" TEST_TYPE_STR ": ");
perf_print(stop,start,
(long long)(TEST_LEN(m))*(k+nerrs)*rtest);
printf("done all: Pass\n");
return 0;
}
void RsDecodeFile(int argc, char **argv)
{
int i, j, k, *vdm, *inv, *prod, cache_size;
int rows, cols, blocksize, orig_size;
int n, m, sz, *factors, tmp, factor, *exists, *map;
char *stem, *filename;
char **buffer, *buf_file, *block;
struct stat buf;
Condensed_Matrix *cm;
int *mat, *id;
FILE *f;
if (argc != 2) {
fprintf(stderr, "usage: rs_decode_file stem\n");
exit(1);
}
stem = argv[1];
buf_file = (char *) malloc(sizeof(char)*(strlen(stem)+30));
if (buf_file == NULL) { perror("malloc - buf_file"); exit(1); }
sprintf(buf_file, "%s-info.txt", stem, i);
f = fopen(buf_file, "r");
if (f == NULL) { perror(buf_file); exit(1); }
if (fscanf(f, "%d\n", &orig_size) != 1) { fprintf(stderr, "Error reading info file 1\n"); exit(1); }
if (fscanf(f, "%d\n", &sz) != 1) { fprintf(stderr, "Error reading info file 2\n"); exit(1); }
if (fscanf(f, "%d\n", &blocksize) != 1) { fprintf(stderr, "Error reading info file 3\n"); exit(1); }
if (fscanf(f, "%d\n", &n) != 1) { fprintf(stderr, "Error reading info file 4\n"); exit(1); }
if (fscanf(f, "%d\n", &m) != 1) { fprintf(stderr, "Error reading info file 5\n"); exit(1); }
vdm = gf_read_matrix(f, &rows, &cols);
if (vdm == NULL) { fprintf(stderr, "Error reading info file matrix\n"); exit(1); }
fclose(f);
if (rows != n+m) {
fprintf(stderr, "Error in %s - rows != n+m\n", buf_file);
exit(1);
}
if (cols != n) {
fprintf(stderr, "Error in %s - cols != n\n", buf_file);
exit(1);
}
exists = (int *) malloc(sizeof(int) * rows);
if (exists == NULL) { perror("malloc - exists"); exit(1); }
factors = (int *) malloc(sizeof(int) * rows);
if (factors == NULL) { perror("malloc - factors"); exit(1); }
map = (int *) malloc(sizeof(int) * rows);
if (map == NULL) { perror("malloc - map"); exit(1); }
buffer = (char **) malloc(sizeof(char *)*n);
for (i = 0; i < n; i++) {
buffer[i] = (char *) malloc(blocksize);
if (buffer[i] == NULL) {
perror("Allocating buffer to store the whole file");
exit(1);
}
}
j = 0;
for (i = 0; i < rows && j < cols; i++) {
sprintf(buf_file, "%s-%04d.rs", stem, i);
if (stat(buf_file, &buf) != 0) {
map[i] = -1;
} else {
if (buf.st_size != blocksize) {
map[i] = -1;
} else {
map[i] = j++;
f = fopen(buf_file, "r");
if (f == NULL) { perror(buf_file); exit(1); }
k = fread(buffer[map[i]], 1, blocksize, f);
if (k != blocksize) {
fprintf(stderr, "%s -- stat says %d bytes, but only read %d\n",
buf_file, buf.st_size, k);
exit(1);
}
}
}
}
if (j < cols) {
fprintf(stderr, "Only %d fragments -- need %d. Sorry\n", j, cols);
exit(1);
}
j = 0;
for (i = 0; i < cols; i++) if (map[i] == -1) j++;
fprintf(stderr, "Blocks to decode: %d\n", j);
if (j == 0) {
cache_size = orig_size;
for (i = 0; i < cols; i++) {
if (cache_size > 0) {
fwrite(buffer[i], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
}
}
exit(0);
}
block = (char *) malloc(sizeof(char)*blocksize);
if (block == NULL) { perror("malloc - block"); exit(1); }
for (i = 0; i < rows; i++) exists[i] = (map[i] != -1);
cm = gf_condense_dispersal_matrix(vdm, exists, rows, cols);
mat = cm->condensed_matrix;
id = cm->row_identities;
/* Fix it so that map[i] for i = 0 to cols-1 is defined correctly.
map[i] is the index of buffer[] that holds the blocks for row i in
the condensed matrix */
for (i = 0; i < cols; i++) {
if (map[i] == -1) map[i] = map[id[i]];
}
fprintf(stderr, "Inverting condensed dispersal matrix ... "); fflush(stderr);
inv = gf_invert_matrix(mat, cols);
if (inv == NULL) {
fprintf(stderr, "\n\nError -- matrix unvertible\n");
exit(1);
}
fprintf(stderr, "Done\n"); fflush(stderr);
fprintf(stderr, "\nCondensed matrix:\n\n");
gf_fprint_matrix(stderr, mat, cols, cols);
fprintf(stderr, "\nInverted matrix:\n\n");
gf_fprint_matrix(stderr, inv, cols, cols);
for(i = 0; i < rows; i++) factors[i] = 1;
cache_size = orig_size;
for (i = 0; i < cols && cache_size > 0; i++)
{
if (id[i] < cols)
{
fprintf(stderr, "Writing block %d from memory ... ", i); fflush(stderr);
if (factors[i] != 1)
{
tmp = gf_single_divide(1, factors[i]);
/* fprintf(stderr, "Factor = %3d. Tmp = %3d. Before[0] = %3d. ",
factors[i], tmp, (unsigned char) buffer[map[i]][0]); */
factors[i] = 1;
gf_mult_region(buffer[map[i]], blocksize, tmp);
/* fprintf(stderr, "After[0] = %3d.\n", (unsigned char) buffer[map[i]][0]); */
}
else
{
/* fprintf(stderr, "Factor = %3d. Buffer[0] = %3d.\b", factors[i],
(unsigned char) buffer[map[i]][0]); */
}
fwrite(buffer[map[i]], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
fprintf(stderr, "Done\n"); fflush(stderr);
}
else
{
fprintf(stderr, "Decoding block %d ... ", i); fflush(stderr);
memset(block, 0, blocksize);
for (j = 0; j < cols; j++)
{
tmp = inv[i*cols+j];
factor = gf_single_divide(tmp, factors[j]);
/* fprintf(stderr, "Factors[%d] = %3d. Tmp = %3d. Factor = %3d\n Before[j][0] = %3d. ",
j, factors[j], tmp, factor, (unsigned char) buffer[map[j]][0]); */
factors[j] = tmp;
gf_mult_region(buffer[map[j]], blocksize, factor);
/* fprintf(stderr, "After[j][0] = %3d. ", (unsigned char) buffer[map[j]][0]);
fprintf(stderr, "Before-block[0] = %3d. ", (unsigned char) block[0]); */
gf_add_parity(buffer[map[j]], block, blocksize);
/* fprintf(stderr, "After-block[0] = %3d.\n", (unsigned char) block[0]); */
}
fprintf(stderr, "writing ... "); fflush(stderr);
fwrite(block, 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
cache_size -= blocksize;
fprintf(stderr, "Done\n"); fflush(stderr);
}
}
}
int main(int argc, char *argv[])
{
int i, k, t;
u8 *test_mat, *save_mat, *invr_mat;
u8 test1[] = { 1, 1, 6,
1, 1, 1,
7, 1, 9
};
u8 test2[] = { 0, 1, 6,
1, 0, 1,
0, 1, 9
};
u8 test3[] = { 0, 0, 1,
1, 0, 0,
0, 1, 1
};
u8 test4[] = { 0, 1, 6, 7,
1, 1, 0, 0,
0, 1, 2, 3,
3, 2, 2, 3
}; // = row3+3*row2
printf("gf_inverse_test: max=%d ", KMAX);
test_mat = malloc(KMAX * KMAX);
save_mat = malloc(KMAX * KMAX);
invr_mat = malloc(KMAX * KMAX);
if (NULL == test_mat || NULL == save_mat || NULL == invr_mat)
return -1;
// Test with lots of leading 1's
k = 3;
memcpy(test_mat, test1, k * k);
if (inv_test(test_mat, invr_mat, save_mat, k))
return -1;
// Test with leading zeros
k = 3;
memcpy(test_mat, test2, k * k);
if (inv_test(test_mat, invr_mat, save_mat, k))
return -1;
// Test 3
k = 3;
memcpy(test_mat, test3, k * k);
if (inv_test(test_mat, invr_mat, save_mat, k))
return -1;
// Test 4 - try a singular matrix
k = 4;
memcpy(test_mat, test4, k * k);
if (!gf_invert_matrix(test_mat, invr_mat, k)) {
printf("Fail: didn't catch singular matrix\n");
print_matrix(test4, 4);
return -1;
}
// Do random test of size KMAX
k = KMAX;
for (i = 0; i < k * k; i++)
test_mat[i] = save_mat[i] = rand();
if (gf_invert_matrix(test_mat, invr_mat, k)) {
printf("rand picked a singular matrix, try again\n");
return -1;
}
matrix_mult(invr_mat, save_mat, test_mat, k);
if (is_ident(test_mat, k)) {
printf("fail\n");
print_matrix(save_mat, k);
print_matrix(invr_mat, k);
print_matrix(test_mat, k);
return -1;
}
// Do Randoms. Random size and coefficients
for (t = 0; t < RANDOMS; t++) {
k = rand() % KMAX;
for (i = 0; i < k * k; i++)
test_mat[i] = save_mat[i] = rand();
if (gf_invert_matrix(test_mat, invr_mat, k))
continue;
matrix_mult(invr_mat, save_mat, test_mat, k);
if (is_ident(test_mat, k)) {
printf("fail rand k=%d\n", k);
print_matrix(save_mat, k);
print_matrix(invr_mat, k);
print_matrix(test_mat, k);
return -1;
}
if (0 == (t % 8))
putchar('.');
}
printf(" Pass\n");
return 0;
}
JNIEXPORT jint JNICALL Java_org_apache_hadoop_raid_ReedSolomonDecoder_isaDecode
(JNIEnv *env, jclass myclass, jobjectArray alldata, jintArray erasures, jint blocksize){
Codec_Parameter_de * pCodecParameter = NULL;
pthread_once(&key_once, make_key);
jboolean isCopy;
int * erasured = NULL;
int i, j, p, r, k, errorlocation;
int alldataLen = env->GetArrayLength(alldata);
int erasureLen = env->GetArrayLength(erasures);
int src_error_list[erasureLen];
u8 s;
// Check all the parameters.
if(NULL == (pCodecParameter = (Codec_Parameter_de *)pthread_getspecific(keyDe))){
printf("ReedSolomonDecoder DE not initilized!\n");
return -3;
}
if(erasureLen > pCodecParameter->paritySize){
printf("Too many erasured data!\n");
return -4;
}
if(alldataLen != pCodecParameter->stripeSize + pCodecParameter->paritySize){
printf("Wrong data and parity data size.\n");
return -5;
}
for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize; j++){
pCodecParameter->erasured[j] = -1;
}
int * tmp = (int *)env->GetIntArrayElements(erasures, &isCopy);
int parityErrors = 0;
for(j = 0; j < erasureLen; j++){
if (tmp[j] >= pCodecParameter->paritySize) { // errors in parity will not be fixed
errorlocation = tmp[j] - pCodecParameter->paritySize;
pCodecParameter->erasured[errorlocation] = 1;
}
else if (tmp[j] >= 0){ // put error parity postion
pCodecParameter->erasured[tmp[j] + pCodecParameter->stripeSize] = 1;
parityErrors++;
}
}
// make the src_error_list in the right order
for(j = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++ ) {
if(pCodecParameter->erasured[j] == 1) src_error_list[r++] = j ;
}
for(j = pCodecParameter->paritySize, i = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++){
pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
if(pCodecParameter->erasured[j - pCodecParameter->paritySize] == -1){
pCodecParameter->recov[r++] = pCodecParameter->data[j];
}
else{
pCodecParameter->code[i++] = pCodecParameter->data[j];
}
}
//first parity length elements in alldata are saving parity data
for (j = 0; j < pCodecParameter->paritySize ; j++){
pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
if(pCodecParameter->erasured[j + pCodecParameter->stripeSize] == -1) {
pCodecParameter->recov[r++] = pCodecParameter->data[j];
} else {
pCodecParameter->code[i++] = pCodecParameter->data[j];
}
}
for(i = 0, r = 0; i < pCodecParameter->stripeSize; i++, r++){
while(pCodecParameter->erasured[r] == 1) r++;
for(j = 0; j < pCodecParameter->stripeSize; j++){
pCodecParameter->b[pCodecParameter->stripeSize * i + j] =
pCodecParameter->a[pCodecParameter->stripeSize * r + j];
}
}
//Construct d, the inverted matrix.
if(gf_invert_matrix(pCodecParameter->b, pCodecParameter->d, pCodecParameter->stripeSize) < 0){
printf("BAD MATRIX!\n");
return -6;
}
int srcErrors = erasureLen - parityErrors;
for(i = 0; i < srcErrors; i++){
for(j = 0; j < pCodecParameter->stripeSize; j++){
//store all the erasured line numbers's to the c.
pCodecParameter->c[pCodecParameter->stripeSize * i + j] =
pCodecParameter->d[pCodecParameter->stripeSize * src_error_list[i] + j];
}
}
// recover data
for(i = srcErrors, p = 0; i < erasureLen; i++, p++) {
for(j = 0; j < pCodecParameter->stripeSize; j++) {
pCodecParameter->e[pCodecParameter->stripeSize * p + j] = pCodecParameter->a[pCodecParameter->stripeSize * src_error_list[i] + j];
}
}
// e * invert - b
for(p = 0; p < parityErrors; p++) {
for(i = 0; i < pCodecParameter->stripeSize; i++) {
s = 0;
for(j = 0; j < pCodecParameter->stripeSize; j++)
s ^= gf_mul(pCodecParameter->d[j * pCodecParameter->stripeSize + i], pCodecParameter->e[pCodecParameter->stripeSize * p + j]);
pCodecParameter->c[pCodecParameter->stripeSize * (srcErrors + p) + i] = s;
}
}
ec_init_tables(pCodecParameter->stripeSize, erasureLen, pCodecParameter->c, pCodecParameter->g_tbls);
// Get all the repaired data into pCodecParameter->data, in the first erasuredLen rows.
ec_encode_data(blocksize, pCodecParameter->stripeSize, erasureLen, pCodecParameter->g_tbls,
pCodecParameter->recov, pCodecParameter->code);
// Set the repaired data to alldata.
for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize ; j++){
if(pCodecParameter->erasured[j - pCodecParameter->paritySize] != -1){
env->SetObjectArrayElement(alldata, j, pCodecParameter->datajbuf[j]);
}
}
if(isCopy){
env->ReleaseIntArrayElements(erasures, (jint *)tmp, JNI_ABORT);
}
return 0;
}
int main(int argc, char *argv[])
{
int i, j, rtest, m, k, nerrs, r, err;
void *buf;
unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES], *a, *b, *c, *d, *g_tbls;
unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
unsigned char *recov[TEST_SOURCES];
int rows, align, size;
unsigned char *efence_buffs[TEST_SOURCES];
unsigned int offset;
u8 *ubuffs[TEST_SOURCES];
u8 *temp_ubuffs[TEST_SOURCES];
printf("erasure_code_sse_test: %dx%d ", TEST_SOURCES, TEST_LEN);
// Allocate the arrays
for(i=0; i<TEST_SOURCES; i++){
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
buffs[i] = buf;
}
for(i=0; i<TEST_SOURCES; i++){
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
temp_buffs[i] = buf;
}
// Test erasure code by encode and recovery
a = malloc(MMAX*KMAX);
b = malloc(MMAX*KMAX);
c = malloc(MMAX*KMAX);
d = malloc(MMAX*KMAX);
g_tbls = malloc(KMAX*TEST_SOURCES*32);
if (a == NULL || b == NULL || c == NULL || d == NULL || g_tbls == NULL) {
printf("Test failure! Error with malloc\n");
return -1;
}
// Pick a first test
m = 9;
k = 5;
if (m > MMAX || k > KMAX)
return -1;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);
// Choose random buffers to be in erasure
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
// Construct matrix b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
// Generate decode matrix d as matrix inverse of b
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
// Pack recovery array as list of valid sources
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(TEST_LEN,
k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
return -1;
}
}
// Do more random tests
for (rtest = 0; rtest < RANDOMS; rtest++){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
return -1;
}
}
putchar('.');
}
// Run tests at end of buffer for Electric Fence
k = 16;
align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
if (k > KMAX)
return -1;
for(rows=1; rows<=16; rows++){
m = k+rows;
if (m > MMAX)
return -1;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
for(size=EFENCE_TEST_MIN_SIZE; size<=TEST_SIZE; size+=align){
for(i=0; i<m; i++) // Line up TEST_SIZE from end
efence_buffs[i] = buffs[i] + TEST_LEN - size;
gf_gen_rs_matrix(a, m, k);
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, efence_buffs, &efence_buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
// Generate decode matrix d as matrix inverse of b
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
// Pack recovery array as list of valid sources
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = efence_buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], efence_buffs[src_err_list[i]], size)){
printf("Efence: Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf("Test erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\n");
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], align);
printf("orig :");
dump(efence_buffs[src_err_list[i]],align);
return -1;
}
}
}
}
// Test rand ptr alignment if available
for(rtest=0; rtest<RANDOMS; rtest++){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;
offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
// Add random offsets
for(i=0; i<m; i++) {
memset(buffs[i], 0, TEST_LEN); // zero pad to check write-over
memset(temp_buffs[i], 0, TEST_LEN); // zero pad to check write-over
ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
temp_ubuffs[i] = temp_buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
}
for(i=0; i<k; i++)
for(j=0; j<size; j++)
ubuffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, ubuffs, &ubuffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = ubuffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_ubuffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_ubuffs[k+i], ubuffs[src_err_list[i]], size)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(ubuffs, m, 25);
printf("orig :");
dump(ubuffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_ubuffs[k+i], 25);
return -1;
}
}
// Confirm that padding around dests is unchanged
memset(temp_buffs[0], 0, PTR_ALIGN_CHK_B); // Make reference zero buff
for(i=0; i<m; i++){
offset = ubuffs[i] - buffs[i];
if (memcmp(buffs[i], temp_buffs[0], offset)){
printf("Fail rand ualign encode pad start\n");
return -1;
}
if (memcmp(buffs[i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
printf("Fail rand ualign encode pad end\n");
return -1;
}
}
for(i=0; i<nerrs; i++){
offset = temp_ubuffs[k+i] - temp_buffs[k+i];
if (memcmp(temp_buffs[k+i], temp_buffs[0], offset)){
printf("Fail rand ualign decode pad start\n");
return -1;
}
if (memcmp(temp_buffs[k+i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
printf("Fail rand ualign decode pad end\n");
return -1;
}
}
putchar('.');
}
// Test size alignment
align = (LEN_ALIGN_CHK_B != 0) ? 13 : 16;
for(size=TEST_LEN; size>0; size-=align){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
for(i=0; i<k; i++)
for(j=0; j<size; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], size)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
return -1;
}
}
}
printf("done EC tests: Pass\n");
return 0;
}
int minio_init_decoder (int32_t *error_index,
int k, int n, int errs,
unsigned char *encode_matrix,
unsigned char **decode_matrix,
unsigned char **decode_tbls,
uint32_t **decode_index)
{
int i, j, r, l;
uint32_t *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
unsigned char *input_matrix;
unsigned char *inverse_matrix;
unsigned char *tmp_decode_matrix;
unsigned char *tmp_decode_tbls;
input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);
for (i = 0, r = 0; i < k; i++, r++) {
while (_minio_src_index_in_error(r, error_index, errs))
r++;
for (j = 0; j < k; j++) {
input_matrix[k * i + j] = encode_matrix[k * r + j];
}
tmp_decode_index[i] = r;
}
// Not all vandermonde matrix can be inverted
if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
free(tmp_decode_matrix);
free(tmp_decode_tbls);
free(tmp_decode_index);
return -1;
}
for (l = 0; l < errs; l++) {
if (error_index[l] < k) {
// decoding matrix elements for data chunks
for (j = 0; j < k; j++) {
tmp_decode_matrix[k * l + j] =
inverse_matrix[k *
error_index[l] + j];
}
} else {
// decoding matrix element for coding chunks
for (i = 0; i < k; i++) {
unsigned char s = 0;
for (j = 0; j < k; j++) {
s ^= gf_mul(inverse_matrix[j * k + i],
encode_matrix[k *
error_index[l] + j]);
}
tmp_decode_matrix[k * l + i] = s;
}
}
}
ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
*decode_matrix = tmp_decode_matrix;
*decode_tbls = tmp_decode_tbls;
*decode_index = tmp_decode_index;
return 0;
}