int isa_l_min_fragments(void *desc, int *missing_idxs,
int *fragments_to_exclude, int *fragments_needed)
{
isa_l_descriptor *isa_l_desc = (isa_l_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (isa_l_desc->k + isa_l_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
fragments_needed[j] = i;
j++;
}
if (j == isa_l_desc->k) {
ret = 0;
fragments_needed[j] = -1;
break;
}
}
return ret;
}
static int liberasurecode_rs_vand_min_fragments(void *desc, int *missing_idxs,
int *fragments_to_exclude, int *fragments_needed)
{
struct liberasurecode_rs_vand_descriptor *rs_vand_desc =
(struct liberasurecode_rs_vand_descriptor*)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (rs_vand_desc->k + rs_vand_desc->m); i++) {
if (!(missing_bm & (1 << i))) {
fragments_needed[j] = i;
j++;
}
if (j == rs_vand_desc->k) {
ret = 0;
fragments_needed[j] = -1;
break;
}
}
return ret;
}
static unsigned char* isa_l_get_decode_matrix(int k, int m, unsigned char *encode_matrix, int *missing_idxs)
{
int i = 0, j = 0, l = 0;
int n = k + m;
unsigned char *decode_matrix = malloc(sizeof(unsigned char) * k * k);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
while (i < k && l < n) {
if (((1 << l) & missing_bm) == 0) {
for (j = 0; j < k; j++) {
decode_matrix[(k * i) + j] = encode_matrix[(k * l) + j];
}
i++;
}
l++;
}
if (i != k) {
free(decode_matrix);
decode_matrix = NULL;
}
return decode_matrix;
}
int isa_l_reconstruct(void *desc, char **data, char **parity,
int *missing_idxs, int destination_idx, int blocksize)
{
isa_l_descriptor *isa_l_desc = (isa_l_descriptor*) desc;
unsigned char *g_tbls = NULL;
unsigned char *decode_matrix = NULL;
unsigned char *decode_inverse = NULL;
unsigned char *inverse_rows = NULL;
unsigned char *reconstruct_buf = NULL;
unsigned char **available_fragments = NULL;
int k = isa_l_desc->k;
int m = isa_l_desc->m;
int n = k + m;
int ret = -1;
int i, j;
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
int inverse_row = -1;
/**
* Get available elements and compute the inverse of their
* corresponding rows.
*/
decode_matrix = isa_l_get_decode_matrix(k, m, isa_l_desc->matrix, missing_idxs);
if (NULL == decode_matrix) {
goto out;
}
decode_inverse = (unsigned char*)malloc(sizeof(unsigned char) * k * k);
if (NULL == decode_inverse) {
goto out;
}
int im_ret = isa_l_desc->gf_invert_matrix(decode_matrix, decode_inverse, k);
if (im_ret < 0) {
goto out;
}
/**
* Get the row needed to reconstruct
*/
inverse_rows = get_inverse_rows(k, m, decode_inverse, isa_l_desc->matrix, missing_idxs, isa_l_desc->gf_mul);
// Generate g_tbls from computed decode matrix (k x k) matrix
g_tbls = malloc(sizeof(unsigned char) * (k * m * 32));
if (NULL == g_tbls) {
goto out;
}
/**
* Fill in the available elements
*/
available_fragments = (unsigned char**)malloc(sizeof(unsigned char*)*k);
if (NULL == available_fragments) {
goto out;
}
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
continue;
}
if (j == k) {
break;
}
if (i < k) {
available_fragments[j] = (unsigned char*)data[i];
} else {
available_fragments[j] = (unsigned char*)parity[i-k];
}
j++;
}
/**
* Copy pointer of buffer to reconstruct
*/
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
if (i == destination_idx) {
if (i < k) {
reconstruct_buf = (unsigned char*)data[i];
} else {
reconstruct_buf = (unsigned char*)parity[i-k];
}
inverse_row = j;
break;
}
j++;
}
}
/**
* Do the reconstruction
*/
isa_l_desc->ec_init_tables(k, 1, &inverse_rows[inverse_row * k], g_tbls);
isa_l_desc->ec_encode_data(blocksize, k, 1, g_tbls, (unsigned char**)available_fragments,
(unsigned char**)&reconstruct_buf);
ret = 0;
out:
free(g_tbls);
free(decode_matrix);
free(decode_inverse);
free(inverse_rows);
free(available_fragments);
return ret;
}
int isa_l_decode(void *desc, char **data, char **parity,
int *missing_idxs, int blocksize)
{
isa_l_descriptor *isa_l_desc = (isa_l_descriptor*)desc;
unsigned char *g_tbls = NULL;
unsigned char *decode_matrix = NULL;
unsigned char *decode_inverse = NULL;
unsigned char *inverse_rows = NULL;
unsigned char **decoded_elements = NULL;
unsigned char **available_fragments = NULL;
int k = isa_l_desc->k;
int m = isa_l_desc->m;
int n = k + m;
int ret = -1;
int i, j;
int num_missing_elements = get_num_missing_elements(missing_idxs);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
decode_matrix = isa_l_get_decode_matrix(k, m, isa_l_desc->matrix, missing_idxs);
if (NULL == decode_matrix) {
goto out;
}
decode_inverse = (unsigned char*)malloc(sizeof(unsigned char) * k * k);
if (NULL == decode_inverse) {
goto out;
}
int im_ret = isa_l_desc->gf_invert_matrix(decode_matrix, decode_inverse, k);
if (im_ret < 0) {
goto out;
}
// Generate g_tbls from computed decode matrix (k x k) matrix
g_tbls = malloc(sizeof(unsigned char) * (k * m * 32));
if (NULL == g_tbls) {
goto out;
}
inverse_rows = get_inverse_rows(k, m, decode_inverse, isa_l_desc->matrix, missing_idxs, isa_l_desc->gf_mul);
decoded_elements = (unsigned char**)malloc(sizeof(unsigned char*)*num_missing_elements);
if (NULL == decoded_elements) {
goto out;
}
available_fragments = (unsigned char**)malloc(sizeof(unsigned char*)*k);
if (NULL == available_fragments) {
goto out;
}
j = 0;
for (i = 0; i < n; i++) {
if (missing_bm & (1 << i)) {
continue;
}
if (j == k) {
break;
}
if (i < k) {
available_fragments[j] = (unsigned char*)data[i];
} else {
available_fragments[j] = (unsigned char*)parity[i-k];
}
j++;
}
// Grab pointers to memory needed for missing data fragments
j = 0;
for (i = 0; i < k; i++) {
if (missing_bm & (1 << i)) {
decoded_elements[j] = (unsigned char*)data[i];
j++;
}
}
for (i = k; i < n; i++) {
if (missing_bm & (1 << i)) {
decoded_elements[j] = (unsigned char*)parity[i - k];
j++;
}
}
isa_l_desc->ec_init_tables(k, num_missing_elements, inverse_rows, g_tbls);
isa_l_desc->ec_encode_data(blocksize, k, num_missing_elements, g_tbls, (unsigned char**)available_fragments,
(unsigned char**)decoded_elements);
ret = 0;
out:
free(g_tbls);
free(decode_matrix);
free(decode_inverse);
free(inverse_rows);
free(decoded_elements);
free(available_fragments);
return ret;
}
/*
* TODO: Add in missing parity rows and adjust the inverse_rows to
* be used for parity.
*/
static unsigned char* get_inverse_rows(int k,
int m,
unsigned char *decode_inverse,
unsigned char* encode_matrix,
int *missing_idxs,
gf_mul_func gf_mul)
{
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs);
int num_missing_elements = get_num_missing_elements(missing_idxs);
unsigned char *inverse_rows = (unsigned char*)malloc(sizeof(unsigned
char*) * k * num_missing_elements);
int i, j, l = 0;
int n = k + m;
if (NULL == inverse_rows) {
return NULL;
}
memset(inverse_rows, 0, sizeof(unsigned
char*) * k * num_missing_elements);
/*
* Fill in rows for missing data
*/
for (i = 0; i < k; i++) {
if ((1 << i) & missing_bm) {
for (j = 0; j < k; j++) {
inverse_rows[(l * k) + j] = decode_inverse[(i * k) + j];
}
l++;
}
}
/*
* Process missing parity.
*
* Start with an all-zero row.
*
* For each data element, if the data element is:
*
* Available: XOR the corresponding coefficient from the
* encoding matrix.
*
* Unavailable: multiply corresponding coefficient with
* the row that corresponds to the missing data in inverse_rows
* and XOR the resulting row with this row.
*/
for (i = k; i < n; i++) {
// Parity is missing
if ((1 << i) & missing_bm) {
int d_idx_avail = 0;
int d_idx_unavail = 0;
for (j = 0; j < k; j++) {
// This data is available, so we can use the encode matrix
if (((1 << j) & missing_bm) == 0) {
inverse_rows[(l * k) + d_idx_avail] ^= encode_matrix[(i * k) + j];
d_idx_avail++;
} else {
mult_and_xor_row(&inverse_rows[l * k],
&inverse_rows[d_idx_unavail * k],
encode_matrix[(i * k) + j],
k,
gf_mul);
d_idx_unavail++;
}
}
l++;
}
}
return inverse_rows;
}
/*
* Note that the caller should always check realloc_bm during success or
* failure to free buffers allocated here. We could free up in this function,
* but it is internal to this library and only used in a few places. In any
* case, the caller has to free up in the success case, so it may as well do
* so in the failure case.
*/
int prepare_fragments_for_decode(
int k, int m,
char **data, char **parity,
int *missing_idxs,
int *orig_size, int *fragment_payload_size, int fragment_size,
uint64_t *realloc_bm)
{
int i; /* a counter */
unsigned long long missing_bm; /* bitmap form of missing indexes list */
int orig_data_size = -1;
int payload_size = -1;
missing_bm = convert_list_to_bitmap(missing_idxs);
/*
* Determine if each data fragment is:
* 1.) Alloc'd: if not, alloc new buffer (for missing fragments)
* 2.) Aligned to 16-byte boundaries: if not, alloc a new buffer
* memcpy the contents and free the old buffer
*/
for (i = 0; i < k; i++) {
/*
* Allocate or replace with aligned buffer if the buffer was not
* aligned.
* DO NOT FREE: the python GC should free the original when cleaning up
* 'data_list'
*/
if (NULL == data[i]) {
data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == data[i]) {
log_error("Could not allocate data buffer!");
return -ENOMEM;
}
*realloc_bm = *realloc_bm | (1 << i);
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -ENOMEM;
}
memcpy(tmp_buf, data[i], fragment_size);
data[i] = tmp_buf;
*realloc_bm = *realloc_bm | (1 << i);
}
/* Need to determine the size of the original data */
if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(data[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
return -EBADHEADER;
}
payload_size = get_fragment_payload_size(data[i]);
if (orig_data_size < 0) {
log_error("Invalid fragment_size in fragment header!");
return -EBADHEADER;
}
}
}
/* Perform the same allocation, alignment checks on the parity fragments */
for (i = 0; i < m; i++) {
/*
* Allocate or replace with aligned buffer, if the buffer was not aligned.
* DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
*/
if (NULL == parity[i]) {
parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
if (NULL == parity[i]) {
log_error("Could not allocate parity buffer!");
return -ENOMEM;
}
*realloc_bm = *realloc_bm | (1 << (k + i));
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -ENOMEM;
}
memcpy(tmp_buf, parity[i], fragment_size);
parity[i] = tmp_buf;
*realloc_bm = *realloc_bm | (1 << (k + i));
}
/* Need to determine the size of the original data */
if (((missing_bm & (1 << (k + i))) == 0) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
return -EBADHEADER;
}
payload_size = get_fragment_payload_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid fragment_size in fragment header!");
return -EBADHEADER;
}
}
}
*orig_size = orig_data_size;
*fragment_payload_size = payload_size;
return 0;
}
