JNIEXPORT jint JNICALL Java_org_apache_hadoop_raid_ReedSolomonDecoder_isaDecode
(JNIEnv *env, jclass myclass, jobjectArray alldata, jintArray erasures, jint blocksize){
Codec_Parameter_de * pCodecParameter = NULL;
pthread_once(&key_once, make_key);
jboolean isCopy;
int * erasured = NULL;
int i, j, p, r, k, errorlocation;
int alldataLen = env->GetArrayLength(alldata);
int erasureLen = env->GetArrayLength(erasures);
int src_error_list[erasureLen];
u8 s;
// Check all the parameters.
if(NULL == (pCodecParameter = (Codec_Parameter_de *)pthread_getspecific(keyDe))){
printf("ReedSolomonDecoder DE not initilized!\n");
return -3;
}
if(erasureLen > pCodecParameter->paritySize){
printf("Too many erasured data!\n");
return -4;
}
if(alldataLen != pCodecParameter->stripeSize + pCodecParameter->paritySize){
printf("Wrong data and parity data size.\n");
return -5;
}
for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize; j++){
pCodecParameter->erasured[j] = -1;
}
int * tmp = (int *)env->GetIntArrayElements(erasures, &isCopy);
int parityErrors = 0;
for(j = 0; j < erasureLen; j++){
if (tmp[j] >= pCodecParameter->paritySize) { // errors in parity will not be fixed
errorlocation = tmp[j] - pCodecParameter->paritySize;
pCodecParameter->erasured[errorlocation] = 1;
}
else if (tmp[j] >= 0){ // put error parity postion
pCodecParameter->erasured[tmp[j] + pCodecParameter->stripeSize] = 1;
parityErrors++;
}
}
// make the src_error_list in the right order
for(j = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++ ) {
if(pCodecParameter->erasured[j] == 1) src_error_list[r++] = j ;
}
for(j = pCodecParameter->paritySize, i = 0, r = 0; j < pCodecParameter->paritySize + pCodecParameter->stripeSize; j++){
pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
if(pCodecParameter->erasured[j - pCodecParameter->paritySize] == -1){
pCodecParameter->recov[r++] = pCodecParameter->data[j];
}
else{
pCodecParameter->code[i++] = pCodecParameter->data[j];
}
}
//first parity length elements in alldata are saving parity data
for (j = 0; j < pCodecParameter->paritySize ; j++){
pCodecParameter->datajbuf[j] = env->GetObjectArrayElement(alldata, j);
pCodecParameter->data[j] = (u8 *)env->GetDirectBufferAddress(pCodecParameter->datajbuf[j]);
if(pCodecParameter->erasured[j + pCodecParameter->stripeSize] == -1) {
pCodecParameter->recov[r++] = pCodecParameter->data[j];
} else {
pCodecParameter->code[i++] = pCodecParameter->data[j];
}
}
for(i = 0, r = 0; i < pCodecParameter->stripeSize; i++, r++){
while(pCodecParameter->erasured[r] == 1) r++;
for(j = 0; j < pCodecParameter->stripeSize; j++){
pCodecParameter->b[pCodecParameter->stripeSize * i + j] =
pCodecParameter->a[pCodecParameter->stripeSize * r + j];
}
}
//Construct d, the inverted matrix.
if(gf_invert_matrix(pCodecParameter->b, pCodecParameter->d, pCodecParameter->stripeSize) < 0){
printf("BAD MATRIX!\n");
return -6;
}
int srcErrors = erasureLen - parityErrors;
for(i = 0; i < srcErrors; i++){
for(j = 0; j < pCodecParameter->stripeSize; j++){
//store all the erasured line numbers's to the c.
pCodecParameter->c[pCodecParameter->stripeSize * i + j] =
pCodecParameter->d[pCodecParameter->stripeSize * src_error_list[i] + j];
}
}
// recover data
for(i = srcErrors, p = 0; i < erasureLen; i++, p++) {
for(j = 0; j < pCodecParameter->stripeSize; j++) {
pCodecParameter->e[pCodecParameter->stripeSize * p + j] = pCodecParameter->a[pCodecParameter->stripeSize * src_error_list[i] + j];
}
}
// e * invert - b
for(p = 0; p < parityErrors; p++) {
for(i = 0; i < pCodecParameter->stripeSize; i++) {
s = 0;
for(j = 0; j < pCodecParameter->stripeSize; j++)
s ^= gf_mul(pCodecParameter->d[j * pCodecParameter->stripeSize + i], pCodecParameter->e[pCodecParameter->stripeSize * p + j]);
pCodecParameter->c[pCodecParameter->stripeSize * (srcErrors + p) + i] = s;
}
}
ec_init_tables(pCodecParameter->stripeSize, erasureLen, pCodecParameter->c, pCodecParameter->g_tbls);
// Get all the repaired data into pCodecParameter->data, in the first erasuredLen rows.
ec_encode_data(blocksize, pCodecParameter->stripeSize, erasureLen, pCodecParameter->g_tbls,
pCodecParameter->recov, pCodecParameter->code);
// Set the repaired data to alldata.
for(j = 0; j < pCodecParameter->stripeSize + pCodecParameter->paritySize ; j++){
if(pCodecParameter->erasured[j - pCodecParameter->paritySize] != -1){
env->SetObjectArrayElement(alldata, j, pCodecParameter->datajbuf[j]);
}
}
if(isCopy){
env->ReleaseIntArrayElements(erasures, (jint *)tmp, JNI_ABORT);
}
return 0;
}
int main(int argc, char *argv[])
{
int re = 0;
int i, j, p, rtest, m, k;
int nerrs, nsrcerrs;
void *buf;
unsigned int decode_index[MMAX];
unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
unsigned char *encode_matrix, *decode_matrix, *invert_matrix, *g_tbls;
unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
unsigned char *recov[TEST_SOURCES];
int rows, align, size;
unsigned char *efence_buffs[TEST_SOURCES];
unsigned int offset;
u8 *ubuffs[TEST_SOURCES];
u8 *temp_ubuffs[TEST_SOURCES];
printf("erasure_code_test: %dx%d ", TEST_SOURCES, TEST_LEN);
srand(TEST_SEED);
// Allocate the arrays
for (i = 0; i < TEST_SOURCES; i++) {
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
buffs[i] = buf;
}
for (i = 0; i < TEST_SOURCES; i++) {
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
temp_buffs[i] = buf;
}
// Test erasure code by encode and recovery
encode_matrix = malloc(MMAX * KMAX);
decode_matrix = malloc(MMAX * KMAX);
invert_matrix = malloc(MMAX * KMAX);
g_tbls = malloc(KMAX * TEST_SOURCES * 32);
if (encode_matrix == NULL || decode_matrix == NULL
|| invert_matrix == NULL || g_tbls == NULL) {
printf("Test failure! Error with malloc\n");
return -1;
}
// Pick a first test
m = 9;
k = 5;
if (m > MMAX || k > KMAX)
return -1;
// Make random data
for (i = 0; i < k; i++)
for (j = 0; j < TEST_LEN; j++)
buffs[i][j] = rand();
// Generate encode matrix encode_matrix
// The matrix generated by gf_gen_rs_matrix
// is not always invertable.
gf_gen_rs_matrix(encode_matrix, m, k);
// Generate g_tbls from encode matrix encode_matrix
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix encode_matrix
ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]);
// Choose random buffers to be in erasure
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list, src_in_err,
nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = buffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) {
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf(" - erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((u8 *) encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((u8 *) invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((u8 *) decode_matrix, m, k);
printf("recov %d:", src_err_list[i]);
dump(temp_buffs[k + i], 25);
printf("orig :");
dump(buffs[src_err_list[i]], 25);
return -1;
}
}
// Pick a first test
m = 9;
k = 5;
if (m > MMAX || k > KMAX)
return -1;
// Make random data
for (i = 0; i < k; i++)
for (j = 0; j < TEST_LEN; j++)
buffs[i][j] = rand();
// The matrix generated by gf_gen_cauchy1_matrix
// is always invertable.
gf_gen_cauchy1_matrix(encode_matrix, m, k);
// Generate g_tbls from encode matrix encode_matrix
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix encode_matrix
ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]);
// Choose random buffers to be in erasure
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list, src_in_err,
nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = buffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) {
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf(" - erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((u8 *) encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((u8 *) invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((u8 *) decode_matrix, m, k);
printf("recov %d:", src_err_list[i]);
dump(temp_buffs[k + i], 25);
printf("orig :");
dump(buffs[src_err_list[i]], 25);
return -1;
}
}
// Do more random tests
for (rtest = 0; rtest < RANDOMS; rtest++) {
while ((m = (rand() % MMAX)) < 2) ;
while ((k = (rand() % KMAX)) >= m || k < 1) ;
if (m > MMAX || k > KMAX)
continue;
// Make random data
for (i = 0; i < k; i++)
for (j = 0; j < TEST_LEN; j++)
buffs[i][j] = rand();
// The matrix generated by gf_gen_cauchy1_matrix
// is always invertable.
gf_gen_cauchy1_matrix(encode_matrix, m, k);
// Make parity vects
// Generate g_tbls from encode matrix a
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix a
ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list,
src_in_err, nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = buffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) {
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((u8 *) encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((u8 *) invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((u8 *) decode_matrix, m, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]], 25);
printf("recov %d:", src_err_list[i]);
dump(temp_buffs[k + i], 25);
return -1;
}
}
putchar('.');
}
// Run tests at end of buffer for Electric Fence
k = 16;
align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
if (k > KMAX)
return -1;
for (rows = 1; rows <= 16; rows++) {
m = k + rows;
if (m > MMAX)
return -1;
// Make random data
for (i = 0; i < k; i++)
for (j = 0; j < TEST_LEN; j++)
buffs[i][j] = rand();
for (size = EFENCE_TEST_MIN_SIZE; size <= TEST_SIZE; size += align) {
for (i = 0; i < m; i++) { // Line up TEST_SIZE from end
efence_buffs[i] = buffs[i] + TEST_LEN - size;
}
// The matrix generated by gf_gen_cauchy1_matrix
// is always invertable.
gf_gen_cauchy1_matrix(encode_matrix, m, k);
// Make parity vects
// Generate g_tbls from encode matrix a
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix a
ec_encode_data(size, k, m - k, g_tbls, efence_buffs, &efence_buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list,
src_in_err, nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = efence_buffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 !=
memcmp(temp_buffs[k + i], efence_buffs[src_err_list[i]],
size)) {
printf("Efence: Fail error recovery (%d, %d, %d)\n", m,
k, nerrs);
printf("size = %d\n", size);
printf("Test erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((u8 *) encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((u8 *) invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((u8 *) decode_matrix, m, k);
printf("recov %d:", src_err_list[i]);
dump(temp_buffs[k + i], align);
printf("orig :");
dump(efence_buffs[src_err_list[i]], align);
return -1;
}
}
}
}
// Test rand ptr alignment if available
for (rtest = 0; rtest < RANDOMS; rtest++) {
while ((m = (rand() % MMAX)) < 2) ;
while ((k = (rand() % KMAX)) >= m || k < 1) ;
if (m > MMAX || k > KMAX)
continue;
size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;
offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
// Add random offsets
for (i = 0; i < m; i++) {
memset(buffs[i], 0, TEST_LEN); // zero pad to check write-over
memset(temp_buffs[i], 0, TEST_LEN); // zero pad to check write-over
ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
temp_ubuffs[i] = temp_buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
}
for (i = 0; i < k; i++)
for (j = 0; j < size; j++)
ubuffs[i][j] = rand();
// The matrix generated by gf_gen_cauchy1_matrix
// is always invertable.
gf_gen_cauchy1_matrix(encode_matrix, m, k);
// Make parity vects
// Generate g_tbls from encode matrix a
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix a
ec_encode_data(size, k, m - k, g_tbls, ubuffs, &ubuffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list,
src_in_err, nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = ubuffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_ubuffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_ubuffs[k + i], ubuffs[src_err_list[i]], size)) {
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((unsigned char *)encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char *)invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((unsigned char *)decode_matrix, m, k);
printf("orig data:\n");
dump_matrix(ubuffs, m, 25);
printf("orig :");
dump(ubuffs[src_err_list[i]], 25);
printf("recov %d:", src_err_list[i]);
dump(temp_ubuffs[k + i], 25);
return -1;
}
}
// Confirm that padding around dests is unchanged
memset(temp_buffs[0], 0, PTR_ALIGN_CHK_B); // Make reference zero buff
for (i = 0; i < m; i++) {
offset = ubuffs[i] - buffs[i];
if (memcmp(buffs[i], temp_buffs[0], offset)) {
printf("Fail rand ualign encode pad start\n");
return -1;
}
if (memcmp
(buffs[i] + offset + size, temp_buffs[0],
PTR_ALIGN_CHK_B - offset)) {
printf("Fail rand ualign encode pad end\n");
return -1;
}
}
for (i = 0; i < nerrs; i++) {
offset = temp_ubuffs[k + i] - temp_buffs[k + i];
if (memcmp(temp_buffs[k + i], temp_buffs[0], offset)) {
printf("Fail rand ualign decode pad start\n");
return -1;
}
if (memcmp
(temp_buffs[k + i] + offset + size, temp_buffs[0],
PTR_ALIGN_CHK_B - offset)) {
printf("Fail rand ualign decode pad end\n");
return -1;
}
}
putchar('.');
}
// Test size alignment
align = (LEN_ALIGN_CHK_B != 0) ? 13 : 16;
for (size = TEST_LEN; size > 0; size -= align) {
while ((m = (rand() % MMAX)) < 2) ;
while ((k = (rand() % KMAX)) >= m || k < 1) ;
if (m > MMAX || k > KMAX)
continue;
for (i = 0; i < k; i++)
for (j = 0; j < size; j++)
buffs[i][j] = rand();
// The matrix generated by gf_gen_cauchy1_matrix
// is always invertable.
gf_gen_cauchy1_matrix(encode_matrix, m, k);
// Make parity vects
// Generate g_tbls from encode matrix a
ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
// Perform matrix dot_prod for EC encoding
// using g_tbls from encode matrix a
ec_encode_data(size, k, m - k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
// Generate decode matrix
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
invert_matrix, decode_index, src_err_list,
src_in_err, nerrs, nsrcerrs, k, m);
if (re != 0) {
printf("Fail to gf_gen_decode_matrix\n");
return -1;
}
// Pack recovery array as list of valid sources
// Its order must be the same as the order
// to generate matrix b in gf_gen_decode_matrix
for (i = 0; i < k; i++) {
recov[i] = buffs[decode_index[i]];
}
// Recover data
ec_init_tables(k, nerrs, decode_matrix, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for (i = 0; i < nerrs; i++) {
if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], size)) {
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (j = 0; j < nerrs; j++)
printf(" %d", src_err_list[j]);
printf(" - Index = ");
for (p = 0; p < k; p++)
printf(" %d", decode_index[p]);
printf("\nencode_matrix:\n");
dump_u8xu8((unsigned char *)encode_matrix, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char *)invert_matrix, k, k);
printf("\ndecode_matrix:\n");
dump_u8xu8((unsigned char *)decode_matrix, m, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]], 25);
printf("recov %d:", src_err_list[i]);
dump(temp_buffs[k + i], 25);
return -1;
}
}
}
printf("done EC tests: Pass\n");
return 0;
}
int main(int argc, char *argv[])
{
int i, j, rtest, m, k, nerrs, r, err;
void *buf;
unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES], *a, *b, *c, *d, *g_tbls;
unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
unsigned char *recov[TEST_SOURCES];
int rows, align, size;
unsigned char *efence_buffs[TEST_SOURCES];
unsigned int offset;
u8 *ubuffs[TEST_SOURCES];
u8 *temp_ubuffs[TEST_SOURCES];
printf("erasure_code_sse_test: %dx%d ", TEST_SOURCES, TEST_LEN);
// Allocate the arrays
for(i=0; i<TEST_SOURCES; i++){
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
buffs[i] = buf;
}
for(i=0; i<TEST_SOURCES; i++){
if (posix_memalign(&buf, 64, TEST_LEN)) {
printf("alloc error: Fail");
return -1;
}
temp_buffs[i] = buf;
}
// Test erasure code by encode and recovery
a = malloc(MMAX*KMAX);
b = malloc(MMAX*KMAX);
c = malloc(MMAX*KMAX);
d = malloc(MMAX*KMAX);
g_tbls = malloc(KMAX*TEST_SOURCES*32);
if (a == NULL || b == NULL || c == NULL || d == NULL || g_tbls == NULL) {
printf("Test failure! Error with malloc\n");
return -1;
}
// Pick a first test
m = 9;
k = 5;
if (m > MMAX || k > KMAX)
return -1;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);
// Choose random buffers to be in erasure
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
// Construct matrix b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
// Generate decode matrix d as matrix inverse of b
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
// Pack recovery array as list of valid sources
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(TEST_LEN,
k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
return -1;
}
}
// Do more random tests
for (rtest = 0; rtest < RANDOMS; rtest++){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(TEST_LEN, k, m-k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], TEST_LEN)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
return -1;
}
}
putchar('.');
}
// Run tests at end of buffer for Electric Fence
k = 16;
align = (LEN_ALIGN_CHK_B != 0) ? 1 : 16;
if (k > KMAX)
return -1;
for(rows=1; rows<=16; rows++){
m = k+rows;
if (m > MMAX)
return -1;
// Make random data
for(i=0; i<k; i++)
for(j=0; j<TEST_LEN; j++)
buffs[i][j] = rand();
for(size=EFENCE_TEST_MIN_SIZE; size<=TEST_SIZE; size+=align){
for(i=0; i<m; i++) // Line up TEST_SIZE from end
efence_buffs[i] = buffs[i] + TEST_LEN - size;
gf_gen_rs_matrix(a, m, k);
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, efence_buffs, &efence_buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
// Generate decode matrix d as matrix inverse of b
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
// Pack recovery array as list of valid sources
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = efence_buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], efence_buffs[src_err_list[i]], size)){
printf("Efence: Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
printf("Test erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\n");
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], align);
printf("orig :");
dump(efence_buffs[src_err_list[i]],align);
return -1;
}
}
}
}
// Test rand ptr alignment if available
for(rtest=0; rtest<RANDOMS; rtest++){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
size = (TEST_LEN - PTR_ALIGN_CHK_B) & ~15;
offset = (PTR_ALIGN_CHK_B != 0) ? 1 : PTR_ALIGN_CHK_B;
// Add random offsets
for(i=0; i<m; i++) {
memset(buffs[i], 0, TEST_LEN); // zero pad to check write-over
memset(temp_buffs[i], 0, TEST_LEN); // zero pad to check write-over
ubuffs[i] = buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
temp_ubuffs[i] = temp_buffs[i] + (rand() & (PTR_ALIGN_CHK_B - offset));
}
for(i=0; i<k; i++)
for(j=0; j<size; j++)
ubuffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, ubuffs, &ubuffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = ubuffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_ubuffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_ubuffs[k+i], ubuffs[src_err_list[i]], size)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(ubuffs, m, 25);
printf("orig :");
dump(ubuffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_ubuffs[k+i], 25);
return -1;
}
}
// Confirm that padding around dests is unchanged
memset(temp_buffs[0], 0, PTR_ALIGN_CHK_B); // Make reference zero buff
for(i=0; i<m; i++){
offset = ubuffs[i] - buffs[i];
if (memcmp(buffs[i], temp_buffs[0], offset)){
printf("Fail rand ualign encode pad start\n");
return -1;
}
if (memcmp(buffs[i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
printf("Fail rand ualign encode pad end\n");
return -1;
}
}
for(i=0; i<nerrs; i++){
offset = temp_ubuffs[k+i] - temp_buffs[k+i];
if (memcmp(temp_buffs[k+i], temp_buffs[0], offset)){
printf("Fail rand ualign decode pad start\n");
return -1;
}
if (memcmp(temp_buffs[k+i] + offset + size, temp_buffs[0], PTR_ALIGN_CHK_B - offset)){
printf("Fail rand ualign decode pad end\n");
return -1;
}
}
putchar('.');
}
// Test size alignment
align = (LEN_ALIGN_CHK_B != 0) ? 13 : 16;
for(size=TEST_LEN; size>0; size-=align){
while ((m = (rand() % MMAX)) < 2);
while ((k = (rand() % KMAX)) >= m || k < 1);
if (m>MMAX || k>KMAX)
continue;
for(i=0; i<k; i++)
for(j=0; j<size; j++)
buffs[i][j] = rand();
gf_gen_rs_matrix(a, m, k);
// Make parity vects
ec_init_tables(k, m-k, &a[k*k], g_tbls);
ec_encode_data_sse(size, k, m-k, g_tbls, buffs, &buffs[k]);
// Random errors
memset(src_in_err, 0, TEST_SOURCES);
for (i=0, nerrs=0; i<k && nerrs<m-k; i++){
err = 1 & rand();
src_in_err[i] = err;
if (err)
src_err_list[nerrs++] = i;
}
if (nerrs == 0){ // should have at least one error
while ((err = (rand() % KMAX)) >= k) ;
src_err_list[nerrs++] = err;
src_in_err[err] = 1;
}
// Construct b by removing error rows
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
for(j=0; j<k; j++)
b[k*i+j] = a[k*r+j];
}
if (gf_invert_matrix(b, d, k) < 0){
printf("BAD MATRIX\n");
return -1;
}
for(i=0, r=0; i<k; i++, r++){
while (src_in_err[r])
r++;
recov[i] = buffs[r];
}
for(i=0; i<nerrs; i++){
for(j=0; j<k; j++){
c[k*i+j]=d[k*src_err_list[i]+j];
}
}
// Recover data
ec_init_tables(k, nerrs, c, g_tbls);
ec_encode_data(size, k, nerrs, g_tbls, recov, &temp_buffs[k]);
for(i=0; i<nerrs; i++){
if (0 != memcmp(temp_buffs[k+i], buffs[src_err_list[i]], size)){
printf("Fail error recovery (%d, %d, %d) - ", m, k, nerrs);
printf(" - erase list = ");
for (i=0; i<nerrs; i++)
printf(" %d", src_err_list[i]);
printf("\na:\n");
dump_u8xu8((unsigned char*)a, m, k);
printf("inv b:\n");
dump_u8xu8((unsigned char*)d, k, k);
printf("orig data:\n");
dump_matrix(buffs, m, 25);
printf("orig :");
dump(buffs[src_err_list[i]],25);
printf("recov %d:",src_err_list[i]);
dump(temp_buffs[k+i], 25);
return -1;
}
}
}
printf("done EC tests: Pass\n");
return 0;
}
