adios_datablock * adios_transform_alacrity_pg_reqgroup_completed(adios_transform_read_request *reqgroup, adios_transform_pg_read_request *completed_pg_reqgroup) { //uint64_t raw_size = (uint64_t)completed_pg_reqgroup->raw_var_length; void* raw_buff = completed_pg_reqgroup->subreqs->data; uint64_t orig_size = adios_get_type_size(reqgroup->transinfo->orig_type, ""); int d = 0; for(d = 0; d < reqgroup->transinfo->orig_ndim; d++) orig_size *= (uint64_t)(completed_pg_reqgroup->orig_varblock->count[d]); void* orig_buff = malloc(orig_size); ALPartitionData output_partition; uint64_t numElements = 0; // Decompress into output_partition from compressed_buf memstream_t ms = memstreamInitReturn (raw_buff); // Deserialize the ALPartitionData from memstream ALDeserializePartitionData (&output_partition, &ms); // Decode ALPartitionData into decompresed buffer int rtn = ALDecode (&output_partition, orig_buff, &numElements); if (ALErrorNone != rtn) return NULL; ALPartitionDataDestroy(&output_partition); return adios_datablock_new(reqgroup->transinfo->orig_type, completed_pg_reqgroup->timestep, completed_pg_reqgroup->pg_bounds_sel, orig_buff); }
// Encodes the data from infile into the ALFileStore outfile, closing both afterward int decodeCommandDecode(ALStore *infile, FILE *outfile) { ALPartitionData partdata; // Output buffer void *outbuf; uint64_t outbufCount; uint64_t totalElemsWritten = 0; ALError err; size_t writelen; while (!ALStoreEOF(infile)) { err = ALStoreReadPartition(infile, &partdata); if (err != ALErrorNone) { fprintf(stderr, "Error reading the next partition from the input files\n"); return 1; } outbuf = malloc(ALGetDecodeLength(&partdata)); if (!outbuf) { fprintf(stderr, "Error allocating %llu btyes of temporary memory for decoding\n", ALGetDecodeLength(&partdata)); return 1; } err = ALDecode(&partdata, outbuf, &outbufCount); if (err != ALErrorNone) { fprintf(stderr, "Error decoding partition from input files\n"); return 1; } writelen = fwrite(outbuf, partdata.metadata.elementSize, outbufCount, outfile); if (writelen != outbufCount) { fprintf(stderr, "Error writing decoded elements to file (wrote %llu, expected to write %llu)\n", (uint64_t)writelen, (uint64_t)partdata.metadata.elementSize); return 1; } totalElemsWritten += outbufCount; free(outbuf); ALPartitionDataDestroy(&partdata); } // Cleanup fclose(outfile); printf("Decoding complete, wrote out %llu elements\n", totalElemsWritten); dbprintf("Closing ALACRITY input files...\n"); if (ALStoreClose(infile) != ALErrorNone) { fprintf(stderr, "Error closing ALACRITY input file, aborting (output should still be correct)\n"); return 1; } return 0; }
// Encodes the data from infile into the ALFileStore outfile, closing both afterward int encodeCommandEncode(FILE *infile, uint64_t infile_len, ALStore *outfile) { const int datatypeLen = ALDatatypeGetSize(OPTIONS.datatype); // Allocate the input buffer uint64_t buflen; if (infile_len > (OPTIONS.part_size_in_elem * datatypeLen)) buflen = (OPTIONS.part_size_in_elem * datatypeLen); else buflen = infile_len; void *inbuf = malloc(buflen); // Input buffer // Prepare the output buffer (encoder) ALIndexForm index_form = OPTIONS.index_form; if (index_form >= ALCompressedInvertedIndex) // all other compressed index based on inverted index index_form = ALInvertedIndex; ALEncoderConfig econfig; ALEncoderConfigure(&econfig, OPTIONS.significant_bits, OPTIONS.datatype, index_form); ALPartitionData partdata; // Output buffer size_t bytesread; int i = 0; double s ; encode_time = 0; compress_time = 0; write_time = 0; total_time = 0; double ss = dclock(); while (!feof(infile) && !ferror(infile) && (bytesread = fread(inbuf, 1, buflen, infile)) != 0) { i++; dbprintf("Encoding partition %d with size %llu...\n", i, bytesread); s = dclock(); ALEncode(&econfig, inbuf, bytesread / datatypeLen, &partdata); encode_time = encode_time + (dclock() - s); s = dclock(); if (OPTIONS.index_form >= ALCompressedInvertedIndex) // all other compressed index based on inverted index ALConvertIndexForm(&partdata.metadata, &partdata.index, OPTIONS.index_form); compress_time = compress_time + (dclock() - s); s = dclock(); if (ALStoreWritePartition(outfile, &partdata) != ALErrorNone) { fprintf(stderr, "Error appending ALACRITY partition to file, aborting\n"); abort(); return 1; } write_time = write_time + (dclock() - s); ALPartitionDataDestroy(&partdata); // TODO: make this automatic when encoding over an existing partition data dbprintf("Partition %d done!\n", i); } total_time = dclock() - ss; printf("[read: %9.3lf] [encode: %9.3lf] [compress: %9.3lf] [write: %9.3lf] [total: %9.3lf]\n", total_time - (encode_time + compress_time + write_time ), encode_time, compress_time, write_time, total_time); if (ferror(infile)) { fprintf(stderr, "Error reading from input file, aborting\n"); return 1; } // Cleanup free(inbuf); fclose(infile); printf("Encoding complete, %llu bytes of input data successfully encoded into %llu partitions\n", infile_len, outfile->cur_partition); dbprintf("Closing ALACRITY output file...\n"); if (ALStoreClose(outfile) != ALErrorNone) { fprintf(stderr, "Error closing ALACRITY output file, aborting\n"); return 1; } return 0; }