コード例 #1
0
ファイル: gf_inverse_test.c プロジェクト: gbtucker/isa-l
int inv_test(u8 * in, u8 * inv, u8 * sav, int n)
{
	memcpy(sav, in, n * n);

	if (gf_invert_matrix(in, inv, n)) {
		printf("Given singular matrix\n");
		print_matrix(sav, n);
		return -1;
	}

	matrix_mult(inv, sav, in, n);

	if (is_ident(in, n)) {
		printf("fail\n");
		print_matrix(sav, n);
		print_matrix(inv, n);
		print_matrix(in, n);
		return -1;
	}
	putchar('.');

	return 0;
}
コード例 #2
0
// Generate decode matrix from encode matrix
static int gf_gen_decode_matrix(unsigned char *encode_matrix,
				unsigned char *decode_matrix,
				unsigned char *invert_matrix,
				unsigned int *decode_index,
				unsigned char *src_err_list,
				unsigned char *src_in_err,
				int nerrs, int nsrcerrs, int k, int m)
{
	int i, j, p;
	int r;
	unsigned char *backup, *b, s;
	int incr = 0;

	b = malloc(MMAX * KMAX);
	backup = malloc(MMAX * KMAX);

	if (b == NULL || backup == NULL) {
		printf("Test failure! Error with malloc\n");
		free(b);
		free(backup);
		return -1;
	}
	// Construct matrix b by removing error rows
	for (i = 0, r = 0; i < k; i++, r++) {
		while (src_in_err[r])
			r++;
		for (j = 0; j < k; j++) {
			b[k * i + j] = encode_matrix[k * r + j];
			backup[k * i + j] = encode_matrix[k * r + j];
		}
		decode_index[i] = r;
	}
	incr = 0;
	while (gf_invert_matrix(b, invert_matrix, k) < 0) {
		if (nerrs == (m - k)) {
			free(b);
			free(backup);
			printf("BAD MATRIX\n");
			return NO_INVERT_MATRIX;
		}
		incr++;
		memcpy(b, backup, MMAX * KMAX);
		for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) {
			if (src_err_list[i] == (decode_index[k - 1] + incr)) {
				// skip the erased parity line
				incr++;
				continue;
			}
		}
		if (decode_index[k - 1] + incr >= m) {
			free(b);
			free(backup);
			printf("BAD MATRIX\n");
			return NO_INVERT_MATRIX;
		}
		decode_index[k - 1] += incr;
		for (j = 0; j < k; j++)
			b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j];

	};

	for (i = 0; i < nsrcerrs; i++) {
		for (j = 0; j < k; j++) {
			decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j];
		}
	}
	/* src_err_list from encode_matrix * invert of b for parity decoding */
	for (p = nsrcerrs; p < nerrs; p++) {
		for (i = 0; i < k; i++) {
			s = 0;
			for (j = 0; j < k; j++)
				s ^= gf_mul(invert_matrix[j * k + i],
					    encode_matrix[k * src_err_list[p] + j]);

			decode_matrix[k * p + i] = s;
		}
	}
	free(b);
	free(backup);
	return 0;
}
コード例 #3
0
ファイル: erasure-code-base-perf.c プロジェクト: mattf/minio
int main(int argc, char *argv[])
{
        int i, j, rtest, m, k, nerrs, r;
        void *buf;
        u8 *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
        u8 a[MMAX * KMAX], b[MMAX * KMAX], c[MMAX * KMAX], d[MMAX * KMAX];
        u8 g_tbls[KMAX * TEST_SOURCES * 32], src_in_err[TEST_SOURCES];
        u8 src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];
        struct perf start, stop;

        // Pick test parameters
        m = 14;
        k = 10;
        nerrs = 4;
        const u8 err_list[] = {2, 4, 5, 7};

        printf("erasure_code_base_perf: %dx%d %d\n", m, TEST_LEN(m), nerrs);

        if (m > MMAX || k > KMAX || nerrs > (m - k)){
                printf(" Input test parameter error\n");
                return -1;
        }

        memcpy(src_err_list, err_list, nerrs);
        memset(src_in_err, 0, TEST_SOURCES);
        for (i = 0; i < nerrs; i++)
                src_in_err[src_err_list[i]] = 1;

        // Allocate the arrays
        for (i = 0; i < m; i++) {
                if (posix_memalign(&buf, 64, TEST_LEN(m))) {
                        printf("alloc error: Fail\n");
                        return -1;
                }
                buffs[i] = buf;
        }

        for (i = 0; i < (m - k); i++) {
                if (posix_memalign(&buf, 64, TEST_LEN(m))) {
                        printf("alloc error: Fail\n");
                        return -1;
                }
                temp_buffs[i] = buf;
        }

        // Make random data
        for (i = 0; i < k; i++)
                for (j = 0; j < TEST_LEN(m); j++)
                        buffs[i][j] = rand();

        gf_gen_rs_matrix(a, m, k);
        ec_init_tables(k, m - k, &a[k * k], g_tbls);
        ec_encode_data_base(TEST_LEN(m), k, m - k, g_tbls, buffs, &buffs[k]);

        // Start encode test
        perf_start(&start);
        for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
                // Make parity vects
                ec_init_tables(k, m - k, &a[k * k], g_tbls);
                ec_encode_data_base(TEST_LEN(m), k, m - k, g_tbls, buffs, &buffs[k]);
        }
        perf_stop(&stop);
        printf("erasure_code_base_encode" TEST_TYPE_STR ": ");
        perf_print(stop, start, (long long)(TEST_LEN(m)) * (m) * rtest);

        // Start decode test
        perf_start(&start);
        for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
                // Construct b by removing error rows
                for (i = 0, r = 0; i < k; i++, r++) {
                        while (src_in_err[r])
                                r++;
                        recov[i] = buffs[r];
                        for (j = 0; j < k; j++)
                                b[k * i + j] = a[k * r + j];
                }

                if (gf_invert_matrix(b, d, k) < 0) {
                        printf("BAD MATRIX\n");
                        return -1;
                }

                for (i = 0; i < nerrs; i++)
                        for (j = 0; j < k; j++)
                                c[k * i + j] = d[k * src_err_list[i] + j];

                // Recover data
                ec_init_tables(k, nerrs, c, g_tbls);
                ec_encode_data_base(TEST_LEN(m), k, nerrs, g_tbls, recov, temp_buffs);
        }
        perf_stop(&stop);

        for (i = 0; i < nerrs; i++) {
                if (0 != memcmp(temp_buffs[i], buffs[src_err_list[i]], TEST_LEN(m))) {
                        printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
                        return -1;
                }
        }

        printf("erasure_code_base_decode" TEST_TYPE_STR ": ");
        perf_print(stop, start, (long long)(TEST_LEN(m)) * (k + nerrs) * rtest);

        printf("done all: Pass\n");
        return 0;
}
コード例 #4
0
int main(int argc, char *argv[])
{
	int i, j, rtest, srcs, m, k, nerrs, r, err;
	void *buf;
	u8 g[TEST_SOURCES], g_tbls[TEST_SOURCES*32], src_in_err[TEST_SOURCES];
	u8 *dest, *dest_ref, *temp_buff, *buffs[TEST_SOURCES];
	u8 a[MMAX*KMAX], b[MMAX*KMAX], d[MMAX*KMAX];
	u8  src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];

	int align, size;
	unsigned char *efence_buffs[TEST_SOURCES];
	unsigned int offset;
	u8 *ubuffs[TEST_SOURCES];
	u8 *udest_ptr;

	printf("gf_vect_dot_prod_sse: %dx%d ", TEST_SOURCES, TEST_LEN);

	mk_gf_field();


	// Allocate the arrays
	for(i=0; i<TEST_SOURCES; i++){
		if (posix_memalign(&buf, 64, TEST_LEN)) {
			printf("alloc error: Fail");
			return -1;
		}
		buffs[i] = buf;
	}

	if (posix_memalign(&buf, 64, TEST_LEN)) {
		printf("alloc error: Fail");
		return -1;
	}
	dest = buf;

	if (posix_memalign(&buf, 64, TEST_LEN)) {
		printf("alloc error: Fail");
		return -1;
	}
	dest_ref = buf;
	
	if (posix_memalign(&buf, 64, TEST_LEN)) {
		printf("alloc error: Fail");
		return -1;
	}
	temp_buff = buf;


	// Test of all zeros
	for(i=0; i<TEST_SOURCES; i++)
		memset(buffs[i], 0, TEST_LEN);

	memset(dest, 0, TEST_LEN);
	memset(temp_buff, 0, TEST_LEN);
	memset(dest_ref, 0, TEST_LEN);
	memset(g, 0, TEST_SOURCES);


	for(i=0; i<TEST_SOURCES; i++)
		gf_vect_mul_init(g[i], &g_tbls[i*32]);

	gf_vect_dot_prod_base(TEST_LEN, TEST_SOURCES, &g_tbls[0], buffs, dest_ref);

	gf_vect_dot_prod_sse(TEST_LEN, TEST_SOURCES, g_tbls, buffs, dest);

	if (0 != memcmp(dest_ref, dest, TEST_LEN)){
		printf("Fail zero vect_dot_prod_sse test\n");
		dump_matrix(buffs, 5, TEST_SOURCES);
		printf("dprod_base:"); 
		dump(dest_ref, 25);
		printf("dprod_sse:"); 
		dump(dest, 25);;
		return -1;
	}
	else
		putchar('.');

	// Rand data test
	for(rtest=0; rtest<RANDOMS; rtest++){
		for(i=0; i<TEST_SOURCES; i++)
			for(j=0; j<TEST_LEN; j++)
				buffs[i][j] = rand();

		for (i=0; i<TEST_SOURCES; i++)
			g[i] = rand();

		for(i=0; i<TEST_SOURCES; i++)
			gf_vect_mul_init(g[i], &g_tbls[i*32]);

		gf_vect_dot_prod_base(TEST_LEN, TEST_SOURCES, &g_tbls[0], buffs, dest_ref);
		gf_vect_dot_prod_sse(TEST_LEN, TEST_SOURCES, g_tbls, buffs, dest);

		if (0 != memcmp(dest_ref, dest, TEST_LEN)){
			printf("Fail rand vect_dot_prod_sse test 1\n");
			dump_matrix(buffs, 5, TEST_SOURCES);
			printf("dprod_base:"); 
			dump(dest_ref, 25);
			printf("dprod_sse:"); 
			dump(dest, 25);
			return -1;
		}

		putchar('.');
	}

	// Rand data test with varied parameters
	for(rtest=0; rtest < RANDOMS; rtest++){
		for (srcs = TEST_SOURCES; srcs > 0; srcs--){
			for(i=0; i<srcs; i++)
				for(j=0; j<TEST_LEN; j++)
					buffs[i][j] = rand();

			for (i=0; i<srcs; i++)
				g[i] = rand();

			for(i=0; i<srcs; i++)
				gf_vect_mul_init(g[i], &g_tbls[i*32]);

			gf_vect_dot_prod_base(TEST_LEN, srcs, &g_tbls[0], buffs, dest_ref);
			gf_vect_dot_prod_sse(TEST_LEN, srcs, g_tbls, buffs, dest);

			if (0 != memcmp(dest_ref, dest, TEST_LEN)){
				printf("Fail rand vect_dot_prod_sse test 2\n");
				dump_matrix(buffs, 5, srcs);
				printf("dprod_base:"); 
				dump(dest_ref, 5);
				printf("dprod_sse:"); 
				dump(dest, 5);
				return -1;
			}

			putchar('.');
		}
	}




	// Test erasure code using gf_vect_dot_prod

	// Pick a first test
	m = 9;
	k = 5;
	if (m > MMAX || k > KMAX)
		return -1;

	gf_gen_rs_matrix(a, m, k);

	// Make random data
	for(i=0; i<k; i++)
		for(j=0; j<TEST_LEN; j++)
			buffs[i][j] = rand();

	// Make parity vects
	for (i=k; i<m; i++) {
		for (j=0; j<k; j++)
			gf_vect_mul_init(a[k*i+j], &g_tbls[j*32]);
#ifndef USEREF
		gf_vect_dot_prod_sse(TEST_LEN,
				k, g_tbls, buffs, buffs[i]);
#else
		gf_vect_dot_prod_base(TEST_LEN,
				k, &g_tbls[0], buffs, buffs[i]);
#endif
	}


	// Random buffers in erasure
	memset(src_in_err, 0, TEST_SOURCES);
	for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
		err = 1 & rand();
		src_in_err[i] = err;
		if (err)
			src_err_list[nerrs++] = i;
	}

	// construct b by removing error rows
	for(i=0, r=0; i<k; i++, r++){
		while (src_in_err[r]) {
			r++; 
			continue;
		}
		for(j=0; j<k; j++)
			b[k*i+j] = a[k*r+j];
	}

	if (gf_invert_matrix((u8*)b, (u8*)d, k) < 0)
		printf("BAD MATRIX\n");
	

	for(i=0, r=0; i<k; i++, r++){
		while (src_in_err[r]) {
			r++; 
			continue;
		}
		recov[i] = buffs[r];
	}

	// Recover data
	for(i=0; i<nerrs; i++){
		for (j=0; j<k; j++)
			gf_vect_mul_init(d[k*src_err_list[i]+j], &g_tbls[j*32]);
#ifndef USEREF
		gf_vect_dot_prod_sse(TEST_LEN,
				k, g_tbls, recov, temp_buff);
#else
		gf_vect_dot_prod_base(TEST_LEN,
				k, &g_tbls[0], recov, temp_buff);
#endif

		if (0 != memcmp(temp_buff, buffs[src_err_list[i]],
					TEST_LEN)){
			printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
			printf("recov %d:",src_err_list[i]); 
			dump(temp_buff, 25);
			printf("orig   :");     
			dump(buffs[src_err_list[i]],25);
			return -1;
		}
	}


	// Do more random tests

	for (rtest = 0; rtest < RANDOMS; rtest++){
		while ((m = (rand() % MMAX)) < 2);
		while ((k = (rand() % KMAX)) >= m || k < 1);

		if (m>MMAX || k>KMAX)
			continue;

		gf_gen_rs_matrix(a, m, k);

		// Make random data
		for(i=0; i<k; i++)
			for(j=0; j<TEST_LEN; j++)
				buffs[i][j] = rand();

		// Make parity vects
		for (i=k; i<m; i++) {
			for (j=0; j<k; j++)
				gf_vect_mul_init(a[k*i+j], &g_tbls[j*32]);
#ifndef USEREF
			gf_vect_dot_prod_sse(TEST_LEN, k, g_tbls, buffs, buffs[i]);
#else
			gf_vect_dot_prod_base(TEST_LEN, k, &g_tbls[0], buffs, buffs[i]);
#endif
		}

		// Random errors
		memset(src_in_err, 0, TEST_SOURCES);
		for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
			err = 1 & rand();
			src_in_err[i] = err;
			if (err)
				src_err_list[nerrs++] = i;
		}
		if (nerrs == 0){  // should have at least one error
			while ((err = (rand() % KMAX)) >= k) ;
			src_err_list[nerrs++] = err;
			src_in_err[err] = 1;
		}

		// construct b by removing error rows
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) {
				r++; 
				continue;
			}
			for(j=0; j<k; j++)
				b[k*i+j] = a[k*r+j];
		}

		if (gf_invert_matrix((u8*)b, (u8*)d, k) < 0)
			printf("BAD MATRIX\n");
	
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) {
				r++; 
				continue;
			}
			recov[i] = buffs[r];
		}

		// Recover data
		for(i=0; i<nerrs; i++){
			for (j=0; j<k; j++)
				gf_vect_mul_init(d[k*src_err_list[i]+j], &g_tbls[j*32]);
#ifndef USEREF
			gf_vect_dot_prod_sse(TEST_LEN, k, g_tbls, recov, temp_buff);
#else
			gf_vect_dot_prod_base(TEST_LEN, k, &g_tbls[0], recov, temp_buff);
#endif
			if (0 != memcmp(temp_buff, buffs[src_err_list[i]],
						TEST_LEN)){
				printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
				printf(" - erase list = ");
				for (i=0; i<nerrs; i++)
					printf(" %d", src_err_list[i]);
				printf("\na:\n"); 
				dump_u8xu8((u8*)a, m, k);
				printf("inv b:\n");   
				dump_u8xu8((u8*)d, k, k);
				printf("orig data:\n"); 
				dump_matrix(buffs, m, 25);
				printf("orig   :");     
				dump(buffs[src_err_list[i]],25);
				printf("recov %d:",src_err_list[i]); 
				dump(temp_buff, 25);
				return -1;
			}
		}
		putchar('.');
	}

	// Run tests at end of buffer for Electric Fence
	align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
	for(size=EFENCE_TEST_MIN_SIZE; size<=TEST_SIZE; size+=align){
		for(i=0; i<TEST_SOURCES; i++)
			for(j=0; j<TEST_LEN; j++)
				buffs[i][j] = rand();

		for(i=0; i<TEST_SOURCES; i++) // Line up TEST_SIZE from end
			efence_buffs[i] = buffs[i] + TEST_LEN - size;

		for (i=0; i<TEST_SOURCES; i++)
			g[i] = rand();

		for(i=0; i<TEST_SOURCES; i++)
			gf_vect_mul_init(g[i], &g_tbls[i*32]);

		gf_vect_dot_prod_base(size, TEST_SOURCES, &g_tbls[0], efence_buffs, dest_ref);
		gf_vect_dot_prod_sse(size, TEST_SOURCES, g_tbls, efence_buffs, dest);

		if (0 != memcmp(dest_ref, dest, size)){
			printf("Fail rand vect_dot_prod_sse test 3\n");
			dump_matrix(efence_buffs, 5, TEST_SOURCES);
			printf("dprod_base:");
			dump(dest_ref, align);
			printf("dprod_sse:");
			dump(dest, align);
			return -1;
		}

		putchar('.');
	}

	// Test rand ptr alignment if available

	for(rtest=0; rtest<RANDOMS; rtest++){
		size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;
		srcs = rand() % TEST_SOURCES;
		if (srcs == 0)
			continue;

		offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
		// Add random offsets
		for(i=0; i<srcs; i++)
			ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));

		udest_ptr = dest + (rand() & (PTR_ALIGN_CHK_B - offset));

		memset(dest, 0, TEST_LEN);  // zero pad to check write-over

		for(i=0; i<srcs; i++)
			for(j=0; j<size; j++)
				ubuffs[i][j] = rand();

		for (i=0; i<srcs; i++)
			g[i] = rand();

		for(i=0; i<srcs; i++)
			gf_vect_mul_init(g[i], &g_tbls[i*32]);

		gf_vect_dot_prod_base(size, srcs, &g_tbls[0], ubuffs, dest_ref);

		gf_vect_dot_prod_sse(size, srcs, g_tbls, ubuffs, udest_ptr);

		if (memcmp(dest_ref, udest_ptr, size)){
			printf("Fail rand vect_dot_prod_sse test ualign srcs=%d\n", srcs);
			dump_matrix(ubuffs, 5, TEST_SOURCES);
			printf("dprod_base:"); 
			dump(dest_ref, 25);
			printf("dprod_sse:"); 
			dump(udest_ptr, 25);
			return -1;
		}

		// Confirm that padding around dests is unchanged
		memset(dest_ref, 0, PTR_ALIGN_CHK_B);  // Make reference zero buff
		offset = udest_ptr - dest;

		if (memcmp(dest, dest_ref, offset)){
			printf("Fail rand ualign pad start\n");
			return -1;
		}
		if (memcmp(dest + offset + size, dest_ref, PTR_ALIGN_CHK_B - offset)){
			printf("Fail rand ualign pad end\n");
			return -1;
		}

		putchar('.');
	}


	// Test all size alignment
	align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;

	for(size=TEST_LEN; size>15; size-=align){
		srcs = TEST_SOURCES;

		for(i=0; i<srcs; i++)
			for(j=0; j<size; j++)
				buffs[i][j] = rand();

		for (i=0; i<srcs; i++)
			g[i] = rand();

		for(i=0; i<srcs; i++)
			gf_vect_mul_init(g[i], &g_tbls[i*32]);

		gf_vect_dot_prod_base(size, srcs, &g_tbls[0], buffs, dest_ref);

		gf_vect_dot_prod_sse(size, srcs, g_tbls, buffs, dest);

		if (memcmp(dest_ref, dest, size)){
			printf("Fail rand vect_dot_prod_sse test ualign len=%d\n", size);
			dump_matrix(buffs, 5, TEST_SOURCES);
			printf("dprod_base:"); 
			dump(dest_ref, 25);
			printf("dprod_sse:"); 
			dump(dest, 25);
			return -1;
		}
	}

	printf("done all: Pass\n");
	return 0;
}
コード例 #5
0
main(int argc, char **argv)
{
  int i, j, k, *vdm, *inv, *prod, cache_size;
  int rows, cols, blocksize, orig_size;
  int n, m, sz, *factors, tmp, factor, *exists, *map;
  char *stem, *filename,*file_name; 
  char **buffer, *buf_file, *block;
  struct stat buf;
  Condensed_Matrix *cm;
  int *mat, *id;
  FILE *f;

  if (argc != 3) /*modified by supriya*/{ 
    fprintf(stderr, "usage: rs_decode_file stem file_name\n");
    exit(1);
  }
  
  file_name=argv[2];/*added by supriya*/

  stem = argv[1];
  buf_file = (char *) malloc(sizeof(char)*(strlen(stem)+30));
  if (buf_file == NULL) { perror("malloc - buf_file"); exit(1); }
  sprintf(buf_file, "%s-info.txt", stem, i);
  f = fopen(buf_file, "r");
  if (f == NULL) { perror(buf_file); exit(1); }
  if (fscanf(f, "%d\n", &orig_size) != 1) { fprintf(stderr, "Error reading info file 1\n"); exit(1); }
  if (fscanf(f, "%d\n", &sz) != 1) { fprintf(stderr, "Error reading info file 2\n"); exit(1); }
  if (fscanf(f, "%d\n", &blocksize) != 1) { fprintf(stderr, "Error reading info file 3\n"); exit(1); }
  if (fscanf(f, "%d\n", &n) != 1) { fprintf(stderr, "Error reading info file 4\n"); exit(1); }
  if (fscanf(f, "%d\n", &m) != 1) { fprintf(stderr, "Error reading info file 5\n"); exit(1); }
  vdm = gf_read_matrix(f, &rows, &cols);
  if (vdm == NULL) { fprintf(stderr, "Error reading info file matrix\n"); exit(1); }
  fclose(f);
  
  if (rows != n+m) {
    fprintf(stderr, "Error in %s - rows != n+m\n", buf_file);
    exit(1);
  }
  if (cols != n) {
    fprintf(stderr, "Error in %s - cols != n\n", buf_file);
    exit(1);
  }

  exists = (int *) malloc(sizeof(int) * rows);
  if (exists == NULL) { perror("malloc - exists"); exit(1); }
  factors = (int *) malloc(sizeof(int) * rows);
  if (factors == NULL) { perror("malloc - factors"); exit(1); }
  map = (int *) malloc(sizeof(int) * rows);
  if (map == NULL) { perror("malloc - map"); exit(1); }

  buffer = (char **) malloc(sizeof(char *)*n);
  for (i = 0; i < n; i++) {
    buffer[i] = (char *) malloc(blocksize);
    if (buffer[i] == NULL) {
      perror("Allocating buffer to store the whole file");
      exit(1);
    }
  }

	/* Added by : Supriya */
	/*Socket program in order to fetch all the encoded files from different nodes*/

	
	int clientsocket;/* Socket descriptor for client */
	int enable=1,num=n,flag=0,p=0,pos,port=7855; 
	char file[40]="",stem_filename[40]="",ip_addr[40]="",ack[10];
	char sup[1024]="",c,line[100]="",search_string[30]="";
	int bytes_received,y;
	strcpy(search_string,file_name);
	//printf("\n%s***",search_string);
	struct sockaddr_in serverAddr;/* client address */
	socklen_t addr_size;
	
	
	/*opening the master to file*/
	FILE *fp;
	fp=fopen("Master_file.txt","r");
	
	if(!fp)
            {
                    perror("\ncould not find the file");
                    exit(0);
            }
	while ( c!= EOF )/* read a line */
           {
		c=fscanf( fp, "%s",line ) ;
		//printf("%s\n",line);

                    if(strstr(line,search_string)){
			p=1;
			break;	
			
			}
            }
		//printf("ssss");
		pos=ftell(fp);
		//printf("%d\n",n);
		pos=pos-strlen(search_string);
		fseek( fp, pos, SEEK_SET );
		if(p!=1){
			perror("\nNo such File exists");
			exit(0);
		}
		//printf("%d\n",fseek( fp, n, SEEK_SET ));
		//c=fscanf( fp, "%s",line ) ;
		//printf("%s\n",line);

	

	printf("%d\n",num);
	while(num>0)
	{
	strcpy(sup,"");
	/*reading the master file to get the ip address of different nodes*/
	fscanf(fp,"%s",file);
	fscanf(fp,"%s",stem_filename);
	fscanf(fp,"%s",ip_addr);
	//fscanf(fp,"%s",port);
	//printf("%s...%s\n",file,stem_filename);

	/* Create socket for connections */
	clientsocket=socket(PF_INET,SOCK_STREAM,0);
	printf("\n\nSocket creation: %s\n",strerror(errno));

	//printf("%d\n",clientsocket);
	port=port+1;
	num=num-1;

	
	/* Construct local address structure */
	serverAddr.sin_family=AF_INET; /* Internet address family */
	serverAddr.sin_port = htons(port);/*server port*/
	serverAddr.sin_addr.s_addr=inet_addr(ip_addr);/*server address*/
	memset(serverAddr.sin_zero,'\0',sizeof serverAddr.sin_zero);/* Zero out structure */ 

	
	
	printf("Before connect\n");
	/*reuse the same port*/
	if (setsockopt(clientsocket, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0)
    		error("setsockopt(SO_REUSEADDR) failed");

	/*connect to the nodes to receive the encoded files*/
	connect(clientsocket,(struct sockaddr*)&serverAddr,sizeof(serverAddr));
		printf("Connection: %s\n",strerror(errno));
	    //printf("Connected to the node with ip address : %s\n",ip_addr);

	
	printf("After connect\n");

	send(clientsocket,"Decode",7,0);
	bytes_received=recv(clientsocket,ack,sizeof(ack),0);

	send(clientsocket,stem_filename,strlen(stem_filename),0);
	bytes_received=recv(clientsocket,sup,sizeof(sup),0);

	sup[bytes_received]='\0';
	printf("**************");

	/*checking if the file has been received or not */
	if(strcmp(sup,"File does not exist")!=0){

		/*write the received content into file*/
	
		FILE *ft;
		ft=fopen(stem_filename,"w");
		
		//flag = fwrite(sup, sizeof(char), bytes_received, f);
		printf("content: %s\n",sup);
		fprintf(ft,"%s",sup);
		flag++;
		fclose(ft);

		
	}
	else{
		printf("%s \n",sup);		
         }

		//printf("%d\t%d\n",num,port);
	}	
	fclose(fp);
	

//fflush(stdin);
	if(flag<n){
	   fprintf(stderr, "Only %d fragments -- need %d.  Sorry\n", flag, n);
   	   exit(1);
	}
/*end of socket program*/


  j = 0;
  for (i = 0; i < rows && j < cols; i++) {
    sprintf(buf_file, "%s-%04d.rs", stem, i);
    if (stat(buf_file, &buf) != 0) {
      map[i] = -1;
    } else {
      if (buf.st_size != blocksize) {
        map[i] = -1;
      } else {
        map[i] = j++;
        f = fopen(buf_file, "r");
        if (f == NULL) { perror(buf_file); exit(1); }
        k = fread(buffer[map[i]], 1, blocksize, f);
        if (k != blocksize) {
          fprintf(stderr, "%s -- stat says %d bytes, but only read %d\n", 
             buf_file, buf.st_size, k);
          exit(1);
        }
      }
    }
  }
 
printf("\n********************");

  /*if (j < cols) {
    fprintf(stderr, "Only %d fragments -- need %d.  Sorry\n", j, cols);
    exit(1);
  }*/
  
  j = 0;
  for (i = 0; i < cols; i++) if (map[i] == -1) j++;
  fprintf(stderr, "Blocks to decode: %d\n", j);
  if (j == 0) {
    cache_size = orig_size;
    for (i = 0; i < cols; i++) {
      if (cache_size > 0) {
        fwrite(buffer[i], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
        cache_size -= blocksize;
      }
    }
    exit(0);
  } 

  block = (char *) malloc(sizeof(char)*blocksize);
  if (block == NULL) { perror("malloc - block"); exit(1); }
  
  for (i = 0; i < rows; i++) exists[i] = (map[i] != -1);
  cm = gf_condense_dispersal_matrix(vdm, exists, rows, cols);
  mat = cm->condensed_matrix;
  id = cm->row_identities;
  /* Fix it so that map[i] for i = 0 to cols-1 is defined correctly.
     map[i] is the index of buffer[] that holds the blocks for row i in 
     the condensed matrix */

  for (i = 0; i < cols; i++) {
    if (map[i] == -1) map[i] = map[id[i]];
  }

  fprintf(stderr, "Inverting condensed dispersal matrix ... "); fflush(stderr);
  inv = gf_invert_matrix(mat, cols);
  if (inv == NULL) {
    fprintf(stderr, "\n\nError -- matrix unvertible\n");
    exit(1);
  }
  fprintf(stderr, "Done\n"); fflush(stderr);
  
  fprintf(stderr, "\nCondensed matrix:\n\n");
  gf_fprint_matrix(stderr, mat, cols, cols);

  fprintf(stderr, "\nInverted matrix:\n\n");
  gf_fprint_matrix(stderr, inv, cols, cols);

  for(i = 0; i < rows; i++) factors[i] = 1;

  cache_size = orig_size;
  for (i = 0; i < cols && cache_size > 0; i++) {
    if (id[i] < cols) {
      fprintf(stderr, "Writing block %d from memory ... ", i); fflush(stderr);
      if (factors[i] != 1) {
        tmp = gf_single_divide(1, factors[i]);
/*        fprintf(stderr, "Factor = %3d.  Tmp = %3d.  Before[0] = %3d.  ",
                factors[i], tmp, (unsigned char) buffer[map[i]][0]); */
        factors[i] = 1;
        gf_mult_region(buffer[map[i]], blocksize, tmp);
/*        fprintf(stderr, "After[0] = %3d.\n", (unsigned char) buffer[map[i]][0]); */
      } else {
/*        fprintf(stderr, "Factor = %3d.  Buffer[0] = %3d.\b", factors[i], 
             (unsigned char) buffer[map[i]][0]); */
      }
      fwrite(buffer[map[i]], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
      cache_size -= blocksize;
      fprintf(stderr, "Done\n"); fflush(stderr);
    } else {
      fprintf(stderr, "Decoding block %d ... ", i); fflush(stderr);
      memset(block, 0, blocksize);
      for (j = 0; j < cols; j++) {
        tmp = inv[i*cols+j];
        factor = gf_single_divide(tmp, factors[j]);
/*        fprintf(stderr, "Factors[%d] = %3d.  Tmp = %3d.  Factor = %3d\n    Before[j][0] = %3d.  ", 
                j, factors[j], tmp, factor, (unsigned char) buffer[map[j]][0]); */
        factors[j] = tmp;
        gf_mult_region(buffer[map[j]], blocksize, factor);
/*        fprintf(stderr, "After[j][0] = %3d.  ", (unsigned char) buffer[map[j]][0]);
        fprintf(stderr, "Before-block[0] = %3d.  ", (unsigned char) block[0]); */
        gf_add_parity(buffer[map[j]], block, blocksize);
/*        fprintf(stderr, "After-block[0] = %3d.\n", (unsigned char) block[0]); */
      }
      fprintf(stderr, "writing ... "); fflush(stderr);
      fwrite(block, 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
      cache_size -= blocksize;
      fprintf(stderr, "Done\n"); fflush(stderr);
    }
  }
}
コード例 #6
0
int main(int argc, char *argv[])
{
    int i, j, rtest, m, k, nerrs, r, err;
    void *buf;
    u8 *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
    u8 a[MMAX*KMAX], b[MMAX*KMAX], c[MMAX*KMAX], d[MMAX*KMAX];
    u8 g_tbls[KMAX*TEST_SOURCES*32], src_in_err[TEST_SOURCES];
    u8 src_err_list[TEST_SOURCES], *recov[TEST_SOURCES];
    struct perf start, stop;

    m = 32;
    k = 28;
    printf("erasure_code_sse_perf: %dx%d ",
           m, (TEST_LEN(m)));


    // Allocate the arrays
    for(i=0; i<TEST_SOURCES; i++) {
        if (posix_memalign(&buf, 64, TEST_LEN(m))) {
            printf("alloc error: Fail");
            return -1;
        }
        buffs[i] = buf;
    }

    for (i=0; i<TEST_SOURCES; i++) {
        if (posix_memalign(&buf, 64, TEST_LEN(m))) {
            printf("alloc error: Fail");
            return -1;
        }
        temp_buffs[i] = buf;
    }

    // Test erasure code by encode and recovery

    // Pick a first test
    if (m > MMAX || k > KMAX)
        return -1;


    // Make random data
    for(i=0; i<k; i++)
        for(j=0; j<(TEST_LEN(m)); j++)
            buffs[i][j] = rand();


    memset(src_in_err, 0, TEST_SOURCES);

    srand(1);
    for (i=0, nerrs=0; i<k && nerrs<m-k; i++) {
        err = 1 & rand();
        src_in_err[i] = err;
        if (err)
            src_err_list[nerrs++] = i;
    }
    if (nerrs == 0) { // should have at least one error
        while ((err = (rand() % KMAX)) >= k) ;
        src_err_list[nerrs++] = err;
        src_in_err[err] = 1;
    }
    printf("Test erase list = ");
    for (i=0; i<nerrs; i++)
        printf(" %d", src_err_list[i]);
    printf("\n");

    perf_start(&start);

    for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
        gf_gen_rs_matrix(a, m, k);

        // Make parity vects
        ec_init_tables(k, m-k, &a[k*k], g_tbls);
        ec_encode_data_sse((TEST_LEN(m)),
                           k, m-k, g_tbls, buffs, &buffs[k]);
    }

    perf_stop(&stop);
    printf("erasure_code_sse_encode" TEST_TYPE_STR ": ");
    perf_print(stop,start,
               (long long)(TEST_LEN(m))*(m)*rtest);

    perf_start(&start);

    for (rtest = 0; rtest < TEST_LOOPS(m); rtest++) {
        // Construct b by removing error rows
        for(i=0, r=0; i<k; i++, r++) {
            while (src_in_err[r])
                r++;
            for(j=0; j<k; j++)
                b[k*i+j] = a[k*r+j];
        }

        if (gf_invert_matrix(b, d, k) < 0) {
            printf("BAD MATRIX\n");
            return -1;
        }

        for(i=0, r=0; i<k; i++, r++) {
            while (src_in_err[r])
                r++;
            recov[i] = buffs[r];
        }

        for(i=0; i<nerrs; i++) {
            for(j=0; j<k; j++) {
                c[k*i+j]=d[k*src_err_list[i]+j];
            }
        }

        // Recover data
        ec_init_tables(k, nerrs, c, g_tbls);
        ec_encode_data_sse((TEST_LEN(m)),
                           k, nerrs, g_tbls, recov, &temp_buffs[k]);

    }

    perf_stop(&stop);
    for(i=0; i<nerrs; i++) {
        if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]],
                        (TEST_LEN(m)))) {
            printf("Fail error recovery (%d, %d, %d) - ",
                   m, k, nerrs);
            printf(" - erase list = ");
            for (j=0; j<nerrs; j++)
                printf(" %d", src_err_list[j]);
            printf("\na:\n");
            dump_u8xu8((u8*)a, m, k);
            printf("inv b:\n");
            dump_u8xu8((u8*)d, k, k);
            printf("orig data:\n");
            dump_matrix(buffs, m, 25);
            printf("orig   :");
            dump(buffs[src_err_list[i]],25);
            printf("recov %d:",src_err_list[i]);
            dump(temp_buffs[k+i], 25);
            return -1;
        }
    }

    printf("erasure_code_sse_decode" TEST_TYPE_STR ": ");
    perf_print(stop,start,
               (long long)(TEST_LEN(m))*(k+nerrs)*rtest);

    printf("done all: Pass\n");
    return 0;
}
コード例 #7
0
ファイル: rs_decode_file.c プロジェクト: funglee/RSCoding
void RsDecodeFile(int argc, char **argv)
{
  int i, j, k, *vdm, *inv, *prod, cache_size;
  int rows, cols, blocksize, orig_size;
  int n, m, sz, *factors, tmp, factor, *exists, *map;
  char *stem, *filename; 
  char **buffer, *buf_file, *block;
  struct stat buf;
  Condensed_Matrix *cm;
  int *mat, *id;
  FILE *f;

  if (argc != 2) {
    fprintf(stderr, "usage: rs_decode_file stem\n");
    exit(1);
  }
  
  stem = argv[1];
  buf_file = (char *) malloc(sizeof(char)*(strlen(stem)+30));
  if (buf_file == NULL) { perror("malloc - buf_file"); exit(1); }
  sprintf(buf_file, "%s-info.txt", stem, i);
  f = fopen(buf_file, "r");
  if (f == NULL) { perror(buf_file); exit(1); }
  if (fscanf(f, "%d\n", &orig_size) != 1) { fprintf(stderr, "Error reading info file 1\n"); exit(1); }
  if (fscanf(f, "%d\n", &sz) != 1) { fprintf(stderr, "Error reading info file 2\n"); exit(1); }
  if (fscanf(f, "%d\n", &blocksize) != 1) { fprintf(stderr, "Error reading info file 3\n"); exit(1); }
  if (fscanf(f, "%d\n", &n) != 1) { fprintf(stderr, "Error reading info file 4\n"); exit(1); }
  if (fscanf(f, "%d\n", &m) != 1) { fprintf(stderr, "Error reading info file 5\n"); exit(1); }
  vdm = gf_read_matrix(f, &rows, &cols);
  if (vdm == NULL) { fprintf(stderr, "Error reading info file matrix\n"); exit(1); }
  fclose(f);
  
  if (rows != n+m) {
    fprintf(stderr, "Error in %s - rows != n+m\n", buf_file);
    exit(1);
  }
  if (cols != n) {
    fprintf(stderr, "Error in %s - cols != n\n", buf_file);
    exit(1);
  }

  exists = (int *) malloc(sizeof(int) * rows);
  if (exists == NULL) { perror("malloc - exists"); exit(1); }
  factors = (int *) malloc(sizeof(int) * rows);
  if (factors == NULL) { perror("malloc - factors"); exit(1); }
  map = (int *) malloc(sizeof(int) * rows);
  if (map == NULL) { perror("malloc - map"); exit(1); }

  buffer = (char **) malloc(sizeof(char *)*n);
  for (i = 0; i < n; i++) {
    buffer[i] = (char *) malloc(blocksize);
    if (buffer[i] == NULL) {
      perror("Allocating buffer to store the whole file");
      exit(1);
    }
  }

  j = 0;
  for (i = 0; i < rows && j < cols; i++) {
    sprintf(buf_file, "%s-%04d.rs", stem, i);
    if (stat(buf_file, &buf) != 0) {
      map[i] = -1;
    } else {
      if (buf.st_size != blocksize) {
        map[i] = -1;
      } else {
        map[i] = j++;
        f = fopen(buf_file, "r");
        if (f == NULL) { perror(buf_file); exit(1); }
        k = fread(buffer[map[i]], 1, blocksize, f);
        if (k != blocksize) {
          fprintf(stderr, "%s -- stat says %d bytes, but only read %d\n", 
             buf_file, buf.st_size, k);
          exit(1);
        }
      }
    }
  }

  if (j < cols) {
    fprintf(stderr, "Only %d fragments -- need %d.  Sorry\n", j, cols);
    exit(1);
  }
  
  j = 0;
  for (i = 0; i < cols; i++) if (map[i] == -1) j++;
  fprintf(stderr, "Blocks to decode: %d\n", j);
  if (j == 0) {
    cache_size = orig_size;
    for (i = 0; i < cols; i++) {
      if (cache_size > 0) {
        fwrite(buffer[i], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
        cache_size -= blocksize;
      }
    }
    exit(0);
  } 

  block = (char *) malloc(sizeof(char)*blocksize);
  if (block == NULL) { perror("malloc - block"); exit(1); }
  
  for (i = 0; i < rows; i++) exists[i] = (map[i] != -1);
  cm = gf_condense_dispersal_matrix(vdm, exists, rows, cols);
  mat = cm->condensed_matrix;
  id = cm->row_identities;
  /* Fix it so that map[i] for i = 0 to cols-1 is defined correctly.
     map[i] is the index of buffer[] that holds the blocks for row i in 
     the condensed matrix */

  for (i = 0; i < cols; i++) {
    if (map[i] == -1) map[i] = map[id[i]];
  }

  fprintf(stderr, "Inverting condensed dispersal matrix ... "); fflush(stderr);
  inv = gf_invert_matrix(mat, cols);
  if (inv == NULL) {
    fprintf(stderr, "\n\nError -- matrix unvertible\n");
    exit(1);
  }
  fprintf(stderr, "Done\n"); fflush(stderr);
  
  fprintf(stderr, "\nCondensed matrix:\n\n");
  gf_fprint_matrix(stderr, mat, cols, cols);

  fprintf(stderr, "\nInverted matrix:\n\n");
  gf_fprint_matrix(stderr, inv, cols, cols);

  for(i = 0; i < rows; i++) factors[i] = 1;

  cache_size = orig_size;
  for (i = 0; i < cols && cache_size > 0; i++)
  {
    if (id[i] < cols)
    {
      fprintf(stderr, "Writing block %d from memory ... ", i); fflush(stderr);
      if (factors[i] != 1)
      {
        tmp = gf_single_divide(1, factors[i]);
/*        fprintf(stderr, "Factor = %3d.  Tmp = %3d.  Before[0] = %3d.  ",
                factors[i], tmp, (unsigned char) buffer[map[i]][0]); */
        factors[i] = 1;
        gf_mult_region(buffer[map[i]], blocksize, tmp);
/*        fprintf(stderr, "After[0] = %3d.\n", (unsigned char) buffer[map[i]][0]); */
      }
      else
      {
/*        fprintf(stderr, "Factor = %3d.  Buffer[0] = %3d.\b", factors[i], 
             (unsigned char) buffer[map[i]][0]); */
      }
      fwrite(buffer[map[i]], 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
      cache_size -= blocksize;
      fprintf(stderr, "Done\n"); fflush(stderr);
    }
    else
    {
      fprintf(stderr, "Decoding block %d ... ", i); fflush(stderr);
      memset(block, 0, blocksize);
      for (j = 0; j < cols; j++)
      {
        tmp = inv[i*cols+j];
        factor = gf_single_divide(tmp, factors[j]);
/*        fprintf(stderr, "Factors[%d] = %3d.  Tmp = %3d.  Factor = %3d\n    Before[j][0] = %3d.  ", 
                j, factors[j], tmp, factor, (unsigned char) buffer[map[j]][0]); */
        factors[j] = tmp;
        gf_mult_region(buffer[map[j]], blocksize, factor);
/*        fprintf(stderr, "After[j][0] = %3d.  ", (unsigned char) buffer[map[j]][0]);
        fprintf(stderr, "Before-block[0] = %3d.  ", (unsigned char) block[0]); */
        gf_add_parity(buffer[map[j]], block, blocksize);
/*        fprintf(stderr, "After-block[0] = %3d.\n", (unsigned char) block[0]); */
      }
      fprintf(stderr, "writing ... "); fflush(stderr);
      fwrite(block, 1, (cache_size > blocksize) ? blocksize : cache_size, stdout);
      cache_size -= blocksize;
      fprintf(stderr, "Done\n"); fflush(stderr);
    }
  }
}
コード例 #8
0
ファイル: gf_inverse_test.c プロジェクト: gbtucker/isa-l
int main(int argc, char *argv[])
{
	int i, k, t;
	u8 *test_mat, *save_mat, *invr_mat;

	u8 test1[] = { 1, 1, 6,
		1, 1, 1,
		7, 1, 9
	};

	u8 test2[] = { 0, 1, 6,
		1, 0, 1,
		0, 1, 9
	};

	u8 test3[] = { 0, 0, 1,
		1, 0, 0,
		0, 1, 1
	};

	u8 test4[] = { 0, 1, 6, 7,
		1, 1, 0, 0,
		0, 1, 2, 3,
		3, 2, 2, 3
	};			// = row3+3*row2

	printf("gf_inverse_test: max=%d ", KMAX);

	test_mat = malloc(KMAX * KMAX);
	save_mat = malloc(KMAX * KMAX);
	invr_mat = malloc(KMAX * KMAX);

	if (NULL == test_mat || NULL == save_mat || NULL == invr_mat)
		return -1;

	// Test with lots of leading 1's
	k = 3;
	memcpy(test_mat, test1, k * k);
	if (inv_test(test_mat, invr_mat, save_mat, k))
		return -1;

	// Test with leading zeros
	k = 3;
	memcpy(test_mat, test2, k * k);
	if (inv_test(test_mat, invr_mat, save_mat, k))
		return -1;

	// Test 3
	k = 3;
	memcpy(test_mat, test3, k * k);
	if (inv_test(test_mat, invr_mat, save_mat, k))
		return -1;

	// Test 4 - try a singular matrix
	k = 4;
	memcpy(test_mat, test4, k * k);
	if (!gf_invert_matrix(test_mat, invr_mat, k)) {
		printf("Fail: didn't catch singular matrix\n");
		print_matrix(test4, 4);
		return -1;
	}
	// Do random test of size KMAX
	k = KMAX;

	for (i = 0; i < k * k; i++)
		test_mat[i] = save_mat[i] = rand();

	if (gf_invert_matrix(test_mat, invr_mat, k)) {
		printf("rand picked a singular matrix, try again\n");
		return -1;
	}

	matrix_mult(invr_mat, save_mat, test_mat, k);

	if (is_ident(test_mat, k)) {
		printf("fail\n");
		print_matrix(save_mat, k);
		print_matrix(invr_mat, k);
		print_matrix(test_mat, k);
		return -1;
	}
	// Do Randoms.  Random size and coefficients
	for (t = 0; t < RANDOMS; t++) {
		k = rand() % KMAX;

		for (i = 0; i < k * k; i++)
			test_mat[i] = save_mat[i] = rand();

		if (gf_invert_matrix(test_mat, invr_mat, k))
			continue;

		matrix_mult(invr_mat, save_mat, test_mat, k);

		if (is_ident(test_mat, k)) {
			printf("fail rand k=%d\n", k);
			print_matrix(save_mat, k);
			print_matrix(invr_mat, k);
			print_matrix(test_mat, k);
			return -1;
		}
		if (0 == (t % 8))
			putchar('.');
	}

	printf(" Pass\n");
	return 0;
}
コード例 #9
0
JNIEXPORT jint JNICALL Java_org_apache_hadoop_raid_ReedSolomonDecoder_isaDecode
  (JNIEnv *env, jclass myclass, jobjectArray alldata, jintArray erasures, jint blocksize){
      Codec_Parameter_de * pCodecParameter = NULL;
      pthread_once(&key_once, make_key);
      jboolean isCopy;

      int * erasured = NULL;
      int i, j, p, r, k, errorlocation;
      int alldataLen = env->GetArrayLength(alldata);
      int erasureLen = env->GetArrayLength(erasures);
      int src_error_list[erasureLen];
      u8 s;

// Check all the parameters.

      if(NULL == (pCodecParameter = (Codec_Parameter_de *)pthread_getspecific(keyDe))){
          printf("ReedSolomonDecoder DE not initilized!\n");
          return -3;
      }

      if(erasureLen > pCodecParameter->paritySize){
          printf("Too many erasured data!\n");
          return -4;
      }

      if(alldataLen != pCodecParameter->stripeSize + pCodecParameter->paritySize){
          printf("Wrong data and parity data size.\n");
          return -5;
      }

      for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize; j++){
          pCodecParameter->erasured[j] = -1;
      }

      int * tmp = (int *)env->GetIntArrayElements(erasures, &isCopy);

      int parityErrors = 0; 


      for(j = 0; j < erasureLen; j++){
          if (tmp[j] >= pCodecParameter->paritySize) {  // errors in parity will not be fixed
              errorlocation = tmp[j] - pCodecParameter->paritySize;
              pCodecParameter->erasured[errorlocation] = 1;
          }
          else if (tmp[j] >= 0){ // put error parity postion
              pCodecParameter->erasured[tmp[j] + pCodecParameter->stripeSize] = 1;
              parityErrors++;
          }

      }
      
      // make the src_error_list in the right order
      for(j = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++ ) {
          if(pCodecParameter->erasured[j] == 1)    src_error_list[r++] = j ;
      }

      for(j = pCodecParameter->paritySize, i = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++){
          pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
          pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
          if(pCodecParameter->erasured[j - pCodecParameter->paritySize] == -1){
               pCodecParameter->recov[r++] = pCodecParameter->data[j];
          }
          else{
               pCodecParameter->code[i++] = pCodecParameter->data[j];
          }
      }
//first parity length elements in alldata are saving parity data 
      for (j = 0; j < pCodecParameter->paritySize ; j++){
          pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
          pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
          if(pCodecParameter->erasured[j + pCodecParameter->stripeSize] == -1) {
              pCodecParameter->recov[r++] = pCodecParameter->data[j];
          } else {
              pCodecParameter->code[i++] = pCodecParameter->data[j];
          }
      }

      for(i = 0, r = 0; i < pCodecParameter->stripeSize; i++, r++){
          while(pCodecParameter->erasured[r] == 1) r++;
            for(j = 0; j < pCodecParameter->stripeSize; j++){
                 pCodecParameter->b[pCodecParameter->stripeSize * i + j] = 
                            pCodecParameter->a[pCodecParameter->stripeSize * r + j];
            }
      }

      //Construct d, the inverted matrix.

      if(gf_invert_matrix(pCodecParameter->b, pCodecParameter->d, pCodecParameter->stripeSize) < 0){
          printf("BAD MATRIX!\n");
          return -6;
      }
      int srcErrors = erasureLen - parityErrors;

      for(i = 0; i < srcErrors; i++){
          for(j = 0; j < pCodecParameter->stripeSize; j++){

              //store all the erasured line numbers's to the c. 
              pCodecParameter->c[pCodecParameter->stripeSize * i + j] = 
                    pCodecParameter->d[pCodecParameter->stripeSize * src_error_list[i] + j];
          }
      }

      // recover data
      for(i = srcErrors, p = 0; i < erasureLen; i++, p++) {
         for(j = 0; j < pCodecParameter->stripeSize; j++)  {
            pCodecParameter->e[pCodecParameter->stripeSize * p + j] = pCodecParameter->a[pCodecParameter->stripeSize * src_error_list[i] + j];
         }
      }
      
      // e * invert - b
      for(p = 0; p < parityErrors; p++) {
         for(i = 0; i < pCodecParameter->stripeSize; i++) {
            s = 0; 
            for(j = 0; j < pCodecParameter->stripeSize; j++)
              s ^= gf_mul(pCodecParameter->d[j * pCodecParameter->stripeSize + i], pCodecParameter->e[pCodecParameter->stripeSize * p + j]);
            pCodecParameter->c[pCodecParameter->stripeSize * (srcErrors + p) + i] = s;
         }
      }
      ec_init_tables(pCodecParameter->stripeSize, erasureLen, pCodecParameter->c, pCodecParameter->g_tbls);

    // Get all the repaired data into pCodecParameter->data, in the first erasuredLen rows.
      ec_encode_data(blocksize, pCodecParameter->stripeSize, erasureLen, pCodecParameter->g_tbls, 
                        pCodecParameter->recov, pCodecParameter->code);

    // Set the repaired data to alldata. 
      for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize ; j++){
          if(pCodecParameter->erasured[j - pCodecParameter->paritySize] != -1){
              env->SetObjectArrayElement(alldata, j, pCodecParameter->datajbuf[j]);
          }
      }
      
      if(isCopy){
          env->ReleaseIntArrayElements(erasures, (jint *)tmp, JNI_ABORT);
      }
      return 0;
 }
コード例 #10
0
int main(int argc, char *argv[])
{
	int i, j, rtest, m, k, nerrs, r, err;
	void *buf;
	unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES], *a, *b, *c, *d, *g_tbls;
	unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
	unsigned char *recov[TEST_SOURCES];

	int rows, align, size;
	unsigned char *efence_buffs[TEST_SOURCES];
	unsigned int offset;
	u8 *ubuffs[TEST_SOURCES];
	u8 *temp_ubuffs[TEST_SOURCES];

	printf("erasure_code_sse_test: %dx%d ", TEST_SOURCES, TEST_LEN);


	// Allocate the arrays
	for(i=0; i<TEST_SOURCES; i++){
		if (posix_memalign(&buf, 64, TEST_LEN)) {
			printf("alloc error: Fail");
			return -1;
		}
		buffs[i] = buf;
	}

	for(i=0; i<TEST_SOURCES; i++){
		if (posix_memalign(&buf, 64, TEST_LEN)) {
			printf("alloc error: Fail");
			return -1;
		}
		temp_buffs[i] = buf;
	}

	// Test erasure code by encode and recovery

	a = malloc(MMAX*KMAX);
	b = malloc(MMAX*KMAX);
	c = malloc(MMAX*KMAX);
	d = malloc(MMAX*KMAX);
	g_tbls = malloc(KMAX*TEST_SOURCES*32);

	if (a == NULL || b == NULL || c == NULL || d == NULL || g_tbls == NULL) {
		printf("Test failure! Error with malloc\n");
		return -1;
	}

	// Pick a first test
	m = 9;
	k = 5;
	if (m > MMAX || k > KMAX)
		return -1;


	// Make random data
	for(i=0; i<k; i++)
		for(j=0; j<TEST_LEN; j++)
			buffs[i][j] = rand();


	gf_gen_rs_matrix(a, m, k);
	ec_init_tables(k, m-k, &a[k*k], g_tbls);
	ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);


	// Choose random buffers to be in erasure
	memset(src_in_err, 0, TEST_SOURCES);
	for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
		err = 1 & rand();
		src_in_err[i] = err;
		if (err)
			src_err_list[nerrs++] = i;
	}

	// Construct matrix b by removing error rows
	for(i=0, r=0; i<k; i++, r++){
		while (src_in_err[r]) 
			r++; 
		for(j=0; j<k; j++)
			b[k*i+j] = a[k*r+j];
	}


	// Generate decode matrix d as matrix inverse of b
	if (gf_invert_matrix(b, d, k) < 0){
		printf("BAD MATRIX\n");
		return -1;
	}
	
	// Pack recovery array as list of valid sources
	for(i=0, r=0; i<k; i++, r++){
		while (src_in_err[r]) 
			r++;
		recov[i] = buffs[r];
	}

	for(i=0; i<nerrs; i++){
		for(j=0; j<k; j++){
			c[k*i+j]=d[k*src_err_list[i]+j];
		}
	}

	// Recover data
	ec_init_tables(k, nerrs, c, g_tbls);
	ec_encode_data(TEST_LEN,
			k, nerrs, g_tbls, recov, &temp_buffs[k]);

	for(i=0; i<nerrs; i++){

		if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
			printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
			printf("recov %d:",src_err_list[i]); 
			dump(temp_buffs[k+i], 25);
			printf("orig   :");     
			dump(buffs[src_err_list[i]],25);
			return -1;
		}
	}
	
	// Do more random tests

	for (rtest = 0; rtest < RANDOMS; rtest++){
		while ((m = (rand() % MMAX)) < 2);
		while ((k = (rand() % KMAX)) >= m || k < 1);

		if (m>MMAX || k>KMAX) 
			continue;


		// Make random data
		for(i=0; i<k; i++)
			for(j=0; j<TEST_LEN; j++)
				buffs[i][j] = rand();


		gf_gen_rs_matrix(a, m, k);

		// Make parity vects
		ec_init_tables(k, m-k, &a[k*k], g_tbls);
		ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);



		// Random errors
		memset(src_in_err, 0, TEST_SOURCES);
		for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
			err = 1 & rand();
			src_in_err[i] = err;
			if (err)
				src_err_list[nerrs++] = i;
		}
		if (nerrs == 0){  // should have at least one error
			while ((err = (rand() % KMAX)) >= k) ;
			src_err_list[nerrs++] = err;
			src_in_err[err] = 1;
		}

		// Construct b by removing error rows
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			for(j=0; j<k; j++)
				b[k*i+j] = a[k*r+j];
		}

		if (gf_invert_matrix(b, d, k) < 0){
			printf("BAD MATRIX\n");
			return -1;
		}
	
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			recov[i] = buffs[r];
		}
		for(i=0; i<nerrs; i++){
			for(j=0; j<k; j++){
				c[k*i+j]=d[k*src_err_list[i]+j];
			}
		}

		// Recover data
		ec_init_tables(k, nerrs, c, g_tbls);
		ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);

		for(i=0; i<nerrs; i++){

			if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
				printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
				printf(" - erase list = ");
				for (i=0; i<nerrs; i++)
					printf(" %d", src_err_list[i]);
				printf("\na:\n"); 
				dump_u8xu8((unsigned char*)a, m, k);
				printf("inv b:\n");   
				dump_u8xu8((unsigned char*)d, k, k);
				printf("orig data:\n"); 
				dump_matrix(buffs, m, 25);
				printf("orig   :");     
				dump(buffs[src_err_list[i]],25);
				printf("recov %d:",src_err_list[i]); 
				dump(temp_buffs[k+i], 25);
				return -1;
			}
		}
		putchar('.');
	}

	// Run tests at end of buffer for Electric Fence
	k = 16;
	align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
	if (k > KMAX)
		return -1;

	for(rows=1; rows<=16; rows++){
		m = k+rows;
		if (m > MMAX)
			return -1;

		// Make random data
		for(i=0; i<k; i++)
			for(j=0; j<TEST_LEN; j++)
				buffs[i][j] = rand();


		for(size=EFENCE_TEST_MIN_SIZE; size<=TEST_SIZE; size+=align){
			for(i=0; i<m; i++) // Line up TEST_SIZE from end
				efence_buffs[i] = buffs[i] + TEST_LEN - size;

			gf_gen_rs_matrix(a, m, k);
			ec_init_tables(k, m-k, &a[k*k], g_tbls);
			ec_encode_data_sse(size, k, m-k, g_tbls, efence_buffs, &efence_buffs[k]);

			// Random errors
			memset(src_in_err, 0, TEST_SOURCES);
			for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
				err = 1 & rand();
				src_in_err[i] = err;
				if (err)
					src_err_list[nerrs++] = i;
			}
			if (nerrs == 0){  // should have at least one error
				while ((err = (rand() % KMAX)) >= k) ;
				src_err_list[nerrs++] = err;
				src_in_err[err] = 1;
			}

			// Construct b by removing error rows
			for(i=0, r=0; i<k; i++, r++){
				while (src_in_err[r])
					r++;
				for(j=0; j<k; j++)
					b[k*i+j] = a[k*r+j];
			}

			// Generate decode matrix d as matrix inverse of b
			if (gf_invert_matrix(b, d, k) < 0){
				printf("BAD MATRIX\n");
				return -1;
			}

			// Pack recovery array as list of valid sources
			for(i=0, r=0; i<k; i++, r++){
				while (src_in_err[r])
					r++;
				recov[i] = efence_buffs[r];
			}
			for(i=0; i<nerrs; i++){
				for(j=0; j<k; j++){
					c[k*i+j]=d[k*src_err_list[i]+j];
				}
			}

			// Recover data
			ec_init_tables(k, nerrs, c, g_tbls);
			ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);

			for(i=0; i<nerrs; i++){

				if (0 != memcmp(temp_buffs[k+i], efence_buffs[src_err_list[i]], size)){
					printf("Efence: Fail error recovery (%d, %d, %d)\n", m, k, nerrs);

					printf("Test erase list = ");
					for (i=0; i<nerrs; i++)
						printf(" %d", src_err_list[i]);
					printf("\n");

					printf("recov %d:",src_err_list[i]);
					dump(temp_buffs[k+i], align);
					printf("orig   :");
					dump(efence_buffs[src_err_list[i]],align);
					return -1;
				}
			}
		}

	}

	// Test rand ptr alignment if available

	for(rtest=0; rtest<RANDOMS; rtest++){
		while ((m = (rand() % MMAX)) < 2);
		while ((k = (rand() % KMAX)) >= m || k < 1);

		if (m>MMAX || k>KMAX)
			continue;

		size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;

		offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
		// Add random offsets
		for(i=0; i<m; i++) {
			memset(buffs[i], 0, TEST_LEN);  // zero pad to check write-over
			memset(temp_buffs[i], 0, TEST_LEN);  // zero pad to check write-over
			ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
			temp_ubuffs[i] = temp_buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
		}

		for(i=0; i<k; i++)
			for(j=0; j<size; j++)
				ubuffs[i][j] = rand();

		gf_gen_rs_matrix(a, m, k);

		// Make parity vects
		ec_init_tables(k, m-k, &a[k*k], g_tbls);
		ec_encode_data_sse(size, k, m-k, g_tbls, ubuffs, &ubuffs[k]);

		// Random errors
		memset(src_in_err, 0, TEST_SOURCES);
		for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
			err = 1 & rand();
			src_in_err[i] = err;
			if (err)
				src_err_list[nerrs++] = i;
		}
		if (nerrs == 0){  // should have at least one error
			while ((err = (rand() % KMAX)) >= k) ;
			src_err_list[nerrs++] = err;
			src_in_err[err] = 1;
		}

		// Construct b by removing error rows
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			for(j=0; j<k; j++)
				b[k*i+j] = a[k*r+j];
		}

		if (gf_invert_matrix(b, d, k) < 0){
			printf("BAD MATRIX\n");
			return -1;
		}

		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			recov[i] = ubuffs[r];
		}
		for(i=0; i<nerrs; i++){
			for(j=0; j<k; j++){
				c[k*i+j]=d[k*src_err_list[i]+j];
			}
		}

		// Recover data
		ec_init_tables(k, nerrs, c, g_tbls);
		ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_ubuffs[k]);

		for(i=0; i<nerrs; i++){

			if (0 != memcmp(temp_ubuffs[k+i], ubuffs[src_err_list[i]], size)){
				printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
				printf(" - erase list = ");
				for (i=0; i<nerrs; i++)
					printf(" %d", src_err_list[i]);
				printf("\na:\n"); 
				dump_u8xu8((unsigned char*)a, m, k);
				printf("inv b:\n");   
				dump_u8xu8((unsigned char*)d, k, k);
				printf("orig data:\n"); 
				dump_matrix(ubuffs, m, 25);
				printf("orig   :");     
				dump(ubuffs[src_err_list[i]],25);
				printf("recov %d:",src_err_list[i]); 
				dump(temp_ubuffs[k+i], 25);
				return -1;
			}
		}

		// Confirm that padding around dests is unchanged
		memset(temp_buffs[0], 0, PTR_ALIGN_CHK_B);  // Make reference zero buff

		for(i=0; i<m; i++){

			offset = ubuffs[i] - buffs[i];

			if (memcmp(buffs[i], temp_buffs[0], offset)){
				printf("Fail rand ualign encode pad start\n");
				return -1;
			}
			if (memcmp(buffs[i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
				printf("Fail rand ualign encode pad end\n");
				return -1;
			}
		}

		for(i=0; i<nerrs; i++){

			offset = temp_ubuffs[k+i] - temp_buffs[k+i];
			if (memcmp(temp_buffs[k+i], temp_buffs[0], offset)){
				printf("Fail rand ualign decode pad start\n");
				return -1;
			}
			if (memcmp(temp_buffs[k+i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
				printf("Fail rand ualign decode pad end\n");
				return -1;
			}
		}

		putchar('.');
	}

	// Test size alignment
	align = (LEN_ALIGN_CHK_B != 0) ? 13 : 16;

	for(size=TEST_LEN; size>0; size-=align){
		while ((m = (rand() % MMAX)) < 2);
		while ((k = (rand() % KMAX)) >= m || k < 1);

		if (m>MMAX || k>KMAX)
			continue;

		for(i=0; i<k; i++)
			for(j=0; j<size; j++)
				buffs[i][j] = rand();

		gf_gen_rs_matrix(a, m, k);

		// Make parity vects
		ec_init_tables(k, m-k, &a[k*k], g_tbls);
		ec_encode_data_sse(size, k, m-k, g_tbls, buffs, &buffs[k]);

		// Random errors
		memset(src_in_err, 0, TEST_SOURCES);
		for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
			err = 1 & rand();
			src_in_err[i] = err;
			if (err)
				src_err_list[nerrs++] = i;
		}
		if (nerrs == 0){  // should have at least one error
			while ((err = (rand() % KMAX)) >= k) ;
			src_err_list[nerrs++] = err;
			src_in_err[err] = 1;
		}

		// Construct b by removing error rows
		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			for(j=0; j<k; j++)
				b[k*i+j] = a[k*r+j];
		}

		if (gf_invert_matrix(b, d, k) < 0){
			printf("BAD MATRIX\n");
			return -1;
		}

		for(i=0, r=0; i<k; i++, r++){
			while (src_in_err[r]) 
				r++;
			recov[i] = buffs[r];
		}
		for(i=0; i<nerrs; i++){
			for(j=0; j<k; j++){
				c[k*i+j]=d[k*src_err_list[i]+j];
			}
		}

		// Recover data
		ec_init_tables(k, nerrs, c, g_tbls);
		ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);

		for(i=0; i<nerrs; i++){

			if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], size)){
				printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
				printf(" - erase list = ");
				for (i=0; i<nerrs; i++)
					printf(" %d", src_err_list[i]);
				printf("\na:\n"); 
				dump_u8xu8((unsigned char*)a, m, k);
				printf("inv b:\n");   
				dump_u8xu8((unsigned char*)d, k, k);
				printf("orig data:\n"); 
				dump_matrix(buffs, m, 25);
				printf("orig   :");     
				dump(buffs[src_err_list[i]],25);
				printf("recov %d:",src_err_list[i]); 
				dump(temp_buffs[k+i], 25);
				return -1;
			}
		}
	}

	printf("done EC tests: Pass\n");
	return 0;
}
コード例 #11
0
ファイル: ec_minio_decode.c プロジェクト: technosophos/minio
int minio_init_decoder (int32_t *error_index,
                        int k, int n, int errs,
                        unsigned char *encode_matrix,
                        unsigned char **decode_matrix,
                        unsigned char **decode_tbls,
                        uint32_t **decode_index)
{
        int i, j, r, l;

        uint32_t      *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
        unsigned char *input_matrix;
        unsigned char *inverse_matrix;
        unsigned char *tmp_decode_matrix;
        unsigned char *tmp_decode_tbls;

        input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
        inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
        tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
        tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);

        for (i = 0, r = 0; i < k; i++, r++) {
                while (_minio_src_index_in_error(r, error_index, errs))
                        r++;
                for (j = 0; j < k; j++) {
                        input_matrix[k * i + j] = encode_matrix[k * r + j];
                }
                tmp_decode_index[i] = r;
        }

        // Not all vandermonde matrix can be inverted
        if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
                free(tmp_decode_matrix);
                free(tmp_decode_tbls);
                free(tmp_decode_index);
                return -1;
        }

        for (l = 0; l < errs; l++) {
                if (error_index[l] < k) {
                        // decoding matrix elements for data chunks
                        for (j = 0; j < k; j++) {
                                tmp_decode_matrix[k * l + j] =
                                        inverse_matrix[k *
                                                       error_index[l] + j];
                        }
                } else {
                        // decoding matrix element for coding chunks
                        for (i = 0; i < k; i++) {
                                unsigned char s = 0;
                                for (j = 0; j < k; j++) {
                                        s ^= gf_mul(inverse_matrix[j * k + i],
                                                    encode_matrix[k *
                                                                  error_index[l] + j]);
                                }
                                tmp_decode_matrix[k * l + i] = s;
                        }
                }
        }

        ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);

        *decode_matrix = tmp_decode_matrix;
        *decode_tbls = tmp_decode_tbls;
        *decode_index = tmp_decode_index;

        return 0;
}