mrb_irep*
mrb_read_irep_file(mrb_state *mrb, FILE* fp)
{
mrb_irep *irep = NULL;
uint8_t *buf;
const size_t header_size = sizeof(struct rite_binary_header);
size_t buf_size = 0;
uint8_t flags = 0;
int result;
if ((mrb == NULL) || (fp == NULL)) {
return NULL;
}
buf = (uint8_t*)mrb_malloc(mrb, header_size);
if (fread(buf, header_size, 1, fp) == 0) {
goto irep_exit;
}
result = read_binary_header(buf, &buf_size, NULL, &flags);
if (result != MRB_DUMP_OK || buf_size <= header_size) {
goto irep_exit;
}
buf = (uint8_t*)mrb_realloc(mrb, buf, buf_size);
if (fread(buf+header_size, buf_size-header_size, 1, fp) == 0) {
goto irep_exit;
}
irep = read_irep(mrb, buf, FLAG_SRC_MALLOC);
irep_exit:
mrb_free(mrb, buf);
return irep;
}
static mrb_irep*
read_irep(mrb_state *mrb, const uint8_t *bin, uint8_t flags)
{
int result;
mrb_irep *irep = NULL;
const struct rite_section_header *section_header;
uint16_t crc;
size_t bin_size = 0;
size_t n;
if ((mrb == NULL) || (bin == NULL)) {
return NULL;
}
result = read_binary_header(bin, &bin_size, &crc, &flags);
if (result != MRB_DUMP_OK) {
return NULL;
}
n = offset_crc_body();
if (crc != calc_crc_16_ccitt(bin + n, bin_size - n, 0)) {
return NULL;
}
bin += sizeof(struct rite_binary_header);
do {
section_header = (const struct rite_section_header *)bin;
if (memcmp(section_header->section_ident, RITE_SECTION_IREP_IDENT, sizeof(section_header->section_ident)) == 0) {
irep = read_section_irep(mrb, bin, flags);
if (!irep) return NULL;
}
else if (memcmp(section_header->section_ident, RITE_SECTION_LINENO_IDENT, sizeof(section_header->section_ident)) == 0) {
if (!irep) return NULL; /* corrupted data */
result = read_section_lineno(mrb, bin, irep);
if (result < MRB_DUMP_OK) {
return NULL;
}
}
else if (memcmp(section_header->section_ident, RITE_SECTION_DEBUG_IDENT, sizeof(section_header->section_ident)) == 0) {
if (!irep) return NULL; /* corrupted data */
result = read_section_debug(mrb, bin, irep, flags);
if (result < MRB_DUMP_OK) {
return NULL;
}
}
else if (memcmp(section_header->section_ident, RITE_SECTION_LV_IDENT, sizeof(section_header->section_ident)) == 0) {
if (!irep) return NULL;
result = read_section_lv(mrb, bin, irep, flags);
if (result < MRB_DUMP_OK) {
return NULL;
}
}
bin += bin_to_uint32(section_header->section_size);
} while (memcmp(section_header->section_ident, RITE_BINARY_EOF, sizeof(section_header->section_ident)) != 0);
return irep;
}
/**
* @brief Read a sparse tensor in coordinate form from a binary file and
* distribute among MPI ranks.
*
* @param fin The file to read from.
* @param comm The MPI communicator to distribute among.
*
* @return The sparse tensor.
*/
static sptensor_t * p_tt_mpi_read_binary_file(
FILE * fin,
MPI_Comm comm)
{
sptensor_t * tt = NULL;
int rank, npes;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &npes);
idx_t global_nnz;
idx_t nmodes;
idx_t dims[MAX_NMODES];
/* get header and tensor stats */
bin_header header;
if(rank == 0) {
read_binary_header(fin, &header);
fill_binary_idx(&nmodes, 1, &header, fin);
fill_binary_idx(dims, nmodes, &header, fin);
fill_binary_idx(&global_nnz, 1, &header, fin);
}
/* send dimension info */
if(rank == 0) {
MPI_Bcast(&nmodes, 1, SPLATT_MPI_IDX, 0, comm);
MPI_Bcast(&global_nnz, 1, SPLATT_MPI_IDX, 0, comm);
} else {
MPI_Bcast(&nmodes, 1, SPLATT_MPI_IDX, 0, comm);
MPI_Bcast(&global_nnz, 1, SPLATT_MPI_IDX, 0, comm);
}
/* sanity check */
if(nmodes > MAX_NMODES) {
if(rank == 0) {
fprintf(stderr, "SPLATT ERROR: maximum %"SPLATT_PF_IDX" modes supported. "
"Found %"SPLATT_PF_IDX". Please recompile with "
"MAX_NMODES=%"SPLATT_PF_IDX".\n",
MAX_NMODES, nmodes, nmodes);
}
return NULL;
}
/* compute my even chunk of nonzeros -- root rank gets the extra amount */
idx_t const target_nnz = global_nnz / npes;
idx_t my_nnz = target_nnz;
if(rank == 0) {
my_nnz = global_nnz - ((npes-1)* target_nnz);
}
tt = tt_alloc(my_nnz, nmodes);
/* read/send all chunks */
if(rank == 0) {
/* handle inds */
idx_t * ibuf = splatt_malloc(target_nnz * sizeof(idx_t));
for(idx_t m=0; m < nmodes; ++m) {
for(int p=1; p < npes; ++p) {
fill_binary_idx(ibuf, target_nnz, &header, fin);
MPI_Send(ibuf, target_nnz, SPLATT_MPI_IDX, p, m, comm);
}
/* load my own */
fill_binary_idx(tt->ind[m], my_nnz, &header, fin);
}
splatt_free(ibuf);
/* now vals */
val_t * vbuf = splatt_malloc(target_nnz * sizeof(val_t));
for(int p=1; p < npes; ++p) {
fill_binary_val(vbuf, target_nnz, &header, fin);
MPI_Send(vbuf, target_nnz, SPLATT_MPI_VAL, p, nmodes, comm);
}
splatt_free(vbuf);
/* finally, load my own vals */
fill_binary_val(tt->vals, my_nnz, &header, fin);
} else {
/* non-root ranks just recv */
MPI_Status status;
/* receive my chunk */
for(idx_t m=0; m < tt->nmodes; ++m) {
MPI_Recv(tt->ind[m], my_nnz, SPLATT_MPI_IDX, 0, m, comm, &status);
}
MPI_Recv(tt->vals, my_nnz, SPLATT_MPI_VAL, 0, nmodes, comm, &status);
}
return tt;
}
mrb_irep*
mrb_read_irep_file(mrb_state *mrb, FILE* fp)
{
mrb_irep *irep = NULL;
int result;
uint8_t *buf;
uint16_t crc, crcwk = 0;
size_t section_size = 0;
size_t nbytes;
struct rite_section_header section_header;
long fpos;
size_t block_size = 1 << 14;
const uint8_t block_fallback_count = 4;
int i;
const size_t buf_size = sizeof(struct rite_binary_header);
if ((mrb == NULL) || (fp == NULL)) {
return NULL;
}
/* You don't need use SIZE_ERROR as buf_size is enough small. */
buf = (uint8_t*)mrb_malloc(mrb, buf_size);
if (fread(buf, buf_size, 1, fp) == 0) {
mrb_free(mrb, buf);
return NULL;
}
result = read_binary_header(buf, NULL, &crc);
mrb_free(mrb, buf);
if (result != MRB_DUMP_OK) {
return NULL;
}
/* verify CRC */
fpos = ftell(fp);
/* You don't need use SIZE_ERROR as block_size is enough small. */
for (i = 0; i < block_fallback_count; i++,block_size >>= 1) {
buf = (uint8_t*)mrb_malloc_simple(mrb, block_size);
if (buf) break;
}
if (!buf) {
return NULL;
}
fseek(fp, offset_crc_body(), SEEK_SET);
while ((nbytes = fread(buf, 1, block_size, fp)) > 0) {
crcwk = calc_crc_16_ccitt(buf, nbytes, crcwk);
}
mrb_free(mrb, buf);
if (nbytes == 0 && ferror(fp)) {
return NULL;
}
if (crcwk != crc) {
return NULL;
}
fseek(fp, fpos + section_size, SEEK_SET);
/* read sections */
do {
fpos = ftell(fp);
if (fread(§ion_header, sizeof(struct rite_section_header), 1, fp) == 0) {
return NULL;
}
section_size = (size_t)bin_to_uint32(section_header.section_size);
if (memcmp(section_header.section_identify, RITE_SECTION_IREP_IDENTIFIER, sizeof(section_header.section_identify)) == 0) {
fseek(fp, fpos, SEEK_SET);
irep = read_section_irep_file(mrb, fp);
if (!irep) return NULL;
}
else if (memcmp(section_header.section_identify, RITE_SECTION_LINENO_IDENTIFIER, sizeof(section_header.section_identify)) == 0) {
if (!irep) return NULL; /* corrupted data */
fseek(fp, fpos, SEEK_SET);
result = read_section_lineno_file(mrb, fp, irep);
if (result < MRB_DUMP_OK) return NULL;
}
else if (memcmp(section_header.section_identify, RITE_SECTION_DEBUG_IDENTIFIER, sizeof(section_header.section_identify)) == 0) {
if (!irep) return NULL; /* corrupted data */
else {
uint8_t* const bin = (uint8_t*)mrb_malloc(mrb, section_size);
fseek(fp, fpos, SEEK_SET);
if (fread((char*)bin, section_size, 1, fp) != 1) {
mrb_free(mrb, bin);
return NULL;
}
result = read_section_debug(mrb, bin, irep, TRUE);
mrb_free(mrb, bin);
}
if (result < MRB_DUMP_OK) return NULL;
}
else if (memcmp(section_header.section_identify, RITE_SECTION_LV_IDENTIFIER, sizeof(section_header.section_identify)) == 0) {
if (!irep) return NULL;
else {
uint8_t* const bin = (uint8_t*)mrb_malloc(mrb, section_size);
fseek(fp, fpos, SEEK_SET);
if (fread((char*)bin, section_size, 1, fp) != 1) {
mrb_free(mrb, bin);
return NULL;
}
result = read_section_lv(mrb, bin, irep, TRUE);
mrb_free(mrb, bin);
}
if (result < MRB_DUMP_OK) return NULL;
}
fseek(fp, fpos + section_size, SEEK_SET);
} while (memcmp(section_header.section_identify, RITE_BINARY_EOF, sizeof(section_header.section_identify)) != 0);
return irep;
}
