uint8_t* Sparse::get (const uint64_t voxel_offset, const size_t index) const { assert (index < get_numel (voxel_offset)); const uint64_t offset = sizeof(uint32_t) + (index * class_size); assert (voxel_offset + offset + class_size <= data_end); uint8_t* const ptr = off2mem(voxel_offset) + offset; return ptr; }
void SparseLegacy::load (const Header& header, size_t) { Default::load (header,0); std::fstream stream (file.name.c_str(), std::ios_base::in | std::ios_base::binary); stream.seekg (0, std::ios::end); const uint64_t file_size = stream.tellg(); stream.close(); const uint64_t current_sparse_data_size = file_size - file.start; if (current_sparse_data_size) { mmap.reset (new File::MMap (file, is_image_readwrite(), true, current_sparse_data_size)); data_end = current_sparse_data_size; } else if (is_image_readwrite()) { //CONF option: SparseDataInitialSize //CONF default: 16777216 //CONF Initial buffer size for data in MRtrix sparse image format file (in bytes). // Default = initialise 16MB, this is enough to store whole-brain fixel data at 2.5mm resolution const uint64_t init_sparse_data_size = File::Config::get_int ("SparseDataInitialSize", 16777216); const size_t new_file_size = file.start + init_sparse_data_size; DEBUG ("Initialising output sparse data file " + file.name + ": new file size " + str(new_file_size) + " (" + str(init_sparse_data_size) + " of which is initial sparse data buffer)"); File::resize (file.name, new_file_size); mmap.reset (new File::MMap (file, is_image_readwrite(), false, init_sparse_data_size)); // Writes a single uint32_t(0) to the start of the sparse data region // Any voxel that has its value initialised to 0 will point here, and therefore dereferencing of any // such voxel will yield a Fixel::Value with zero elements memset (off2mem (0), 0x00, sizeof (uint32_t)); data_end = sizeof(uint32_t); } // If this is the formation of a new image, want to explicitly zero all of the // raw image data - otherwise any random data could be misinterpreted as a large // pointer offset from the start of the sparse image data if (is_image_new()) { for (auto& i : mmaps) memset (i->address(), 0x00, i->size()); } }
void Sparse::unload() { Default::unload(); const uint64_t truncate_file_size = (data_end == size()) ? 0 : file.start + data_end; // Null the excess data before closing the memory map to prevent std::ofstream from giving an error // (writing uninitialised values) memset (off2mem(data_end), 0x00, size() - data_end); mmap = NULL; if (truncate_file_size) { DEBUG ("truncating sparse image data file " + file.name + " to " + str(truncate_file_size) + " bytes"); File::resize (file.name, truncate_file_size); } }
uint64_t Sparse::set_numel (const uint64_t old_offset, const uint32_t numel) { assert (Base::writable); // Before allocating new memory, verify that the current offset does not point to // a voxel that has more elements than is required if (old_offset) { assert (old_offset < data_end); const uint32_t existing_numel = get_numel (old_offset); if (existing_numel >= numel) { // Set the new number of elements, clear the unwanted data, and return memcpy (off2mem(old_offset), &numel, sizeof(uint32_t)); memset (off2mem(old_offset) + sizeof(uint32_t) + (numel * class_size), 0x00, ((existing_numel - numel) * class_size)); // If this voxel is being cleared (i.e. no elements), rather than returning a pointer to a voxel with // zero elements, instead return a pointer to the start of the sparse data, where there's a single // uint32_t(0) which can be used for any and all voxels with zero elements return numel ? old_offset : 0; } else { // Existing memory allocation for this voxel is not sufficient; erase it // Note that the handler makes no attempt at re-using this memory later; new data is just concatenated // to the end of the buffer memset (off2mem(old_offset), 0x00, sizeof(uint32_t) + (existing_numel * class_size)); } } // Don't allocate memory for voxels with no sparse data; just point them all to the start of the // sparse data where theres a single uint32_t(0) if (!numel) return 0; const int64_t requested_size = sizeof (uint32_t) + (numel * class_size); if (data_end + requested_size > size()) { // size() should never be empty if data is being written... assert (size()); uint64_t new_sparse_data_size = 2 * size(); // Cover rare case where a huge memory request occurs early while (new_sparse_data_size < data_end + requested_size) new_sparse_data_size *= 2; // Memory error arises due to the uninitialised data between the end of the sparse data and the end // of the file at its current size being written using std::ofstream // Therefore, explicitly null all data that hasn't already been written // (this does not prohibit the use of this memory for subsequent sparse data though) memset (off2mem(data_end), 0x00, size() - data_end); mmap = NULL; const size_t new_file_size = file.start + new_sparse_data_size; DEBUG ("Resizing sparse data file " + file.name + ": new file size " + str(new_file_size) + " (" + str(new_sparse_data_size) + " of which is for sparse data)"); File::resize (file.name, new_file_size); mmap = new File::MMap (file, Base::writable, true, new_sparse_data_size); } // Write the uint32_t indicating the number of elements in this voxel memcpy (off2mem(data_end), &numel, sizeof(uint32_t)); // The return value is the offset from the beginning of the sparse data const uint64_t ret = data_end; data_end += requested_size; return ret; }
uint32_t Sparse::get_numel (const uint64_t offset) const { return *(reinterpret_cast<uint32_t*>(off2mem(offset))); }