void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Are we using the file back end?
bool fileBackEnd = getProperty("FileBackEnd");
if (fileBackEnd && m_BoxStructureAndMethadata)
throw std::invalid_argument("Combination of BoxStructureOnly or "
"MetaDataOnly were set to TRUE with "
"fileBackEnd "
": this is not possible.");
CPUTimer tim;
auto prog = new Progress(this, 0.0, 1.0, 100);
prog->report("Opening file.");
std::string title;
try {
m_file->getAttr("title", title);
} catch (std::exception &) {
// Leave the title blank if error on loading
}
ws->setTitle(title);
// Load the WorkspaceHistory "process"
if (this->getProperty("LoadHistory")) {
ws->history().loadNexus(m_file.get());
}
this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws));
m_file->closeGroup();
m_file->close();
// Add each of the dimension
for (size_t d = 0; d < nd; d++)
ws->addDimension(m_dims[d]);
// Coordinate system
ws->setCoordinateSystem(m_coordSystem);
// ----------------------------------------- Box Structure
// ------------------------------
prog->report("Reading box structure from HDD.");
MDBoxFlatTree FlatBoxTree;
int nDims = static_cast<int>(nd); // should be safe
FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName());
BoxController_sptr bc = ws->getBoxController();
bc->fromXMLString(FlatBoxTree.getBCXMLdescr());
prog->report("Restoring box structure and connectivity");
std::vector<API::IMDNode *> boxTree;
FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd,
m_BoxStructureAndMethadata);
size_t numBoxes = boxTree.size();
// ---------------------------------------- DEAL WITH BOXES
// ------------------------------------
if (fileBackEnd) { // TODO:: call to the file format factory
auto loader = boost::shared_ptr<API::IBoxControllerIO>(
new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
bc->setFileBacked(loader, m_filename);
// boxes have been already made file-backed when restoring the boxTree;
// How much memory for the cache?
{
// TODO: Clean up, only a write buffer now
double mb = getProperty("Memory");
// Defaults have changed, default disk buffer size should be 10 data
// chunks TODO: find optimal, 100 may be better.
if (mb <= 0)
mb = double(10 * loader->getDataChunk() * sizeof(MDE)) /
double(1024 * 1024);
// Express the cache memory in units of number of events.
uint64_t cacheMemory =
static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1;
// Set these values in the diskMRU
bc->getFileIO()->setWriteBufferSize(cacheMemory);
g_log.information() << "Setting a DiskBuffer cache size of " << mb
<< " MB, or " << cacheMemory << " events.\n";
}
} // Not file back end
else if (!m_BoxStructureAndMethadata) {
// ---------------------------------------- READ IN THE BOXES
// ------------------------------------
// TODO:: call to the file format factory
auto loader =
file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
loader->openFile(m_filename, "r");
const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex();
prog->setNumSteps(numBoxes);
for (size_t i = 0; i < numBoxes; i++) {
prog->report();
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]);
if (!box)
continue;
if (BoxEventIndex[2 * i + 1] >
0) // Load in memory NOT using the file as the back-end,
{
boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]);
boxTree[i]->loadAndAddFrom(
loader.get(), BoxEventIndex[2 * i],
static_cast<size_t>(BoxEventIndex[2 * i + 1]));
}
}
loader->closeFile();
} else // box structure and metadata only
{
}
g_log.debug() << tim
<< " to create all the boxes and fill them with events.\n";
// Box of ID 0 is the head box.
ws->setBox(boxTree[0]);
// Make sure the max ID is ok for later ID generation
bc->setMaxId(numBoxes);
// end-of bMetaDataOnly
// Refresh cache
// TODO:if(!fileBackEnd)ws->refreshCache();
ws->refreshCache();
g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints()
<< " points after refresh.\n";
g_log.debug() << tim << " to finish up.\n";
delete prog;
}
/** Perform the merging, but clone the initial workspace and use the same splitting
* as its structure is equivalent to the partial box structures.
*
* @param ws :: first MDEventWorkspace in the list to merge to.
* @param outputFile :: the name of the output file where file-based workspace should be saved
*/
void MergeMDFiles::doExecByCloning(Mantid::API::IMDEventWorkspace_sptr ws,const std::string &outputFile)
{
m_OutIWS = ws;
m_MDEventType = ws->getEventTypeName();
// Run the tasks in parallel? TODO: enable
//bool Parallel = this->getProperty("Parallel");
// Fix the box controller settings in the output workspace so that it splits normally
BoxController_sptr bc = ws->getBoxController();
// set up internal variables characterizing the workspace.
m_nDims = static_cast<int>(bc->getNDims());
// Fix the max depth to something bigger.
bc->setMaxDepth(20);
bc->setSplitThreshold(5000);
auto saver = boost::shared_ptr<API::IBoxControllerIO>(new MDEvents::BoxControllerNeXusIO(bc.get()));
saver->setDataType(sizeof(coord_t),m_MDEventType);
if(m_fileBasedTargetWS)
{
bc->setFileBacked(saver,outputFile);
// Complete the file-back-end creation.
g_log.notice() << "Setting cache to 400 MB write." << std::endl;
bc->getFileIO()->setWriteBufferSize(400000000/m_OutIWS->sizeofEvent());
}
/* else
{
saver->openFile(outputFile,"w");
}*/
// Init box structure used for memory/file space calculations
m_BoxStruct.initFlatStructure(ws,outputFile);
// First, load all the box data and experiment info and calculate file positions of the target workspace
this->loadBoxData();
size_t numBoxes = m_BoxStruct.getNBoxes();
// Progress report based on events processed.
this->prog = new Progress(this, 0.1, 0.9, size_t(numBoxes));
prog->setNotifyStep(0.1);
// For tracking progress
//uint64_t totalEventsInTasks = 0;
// Prepare thread pool
CPUTimer overallTime;
ThreadSchedulerFIFO * ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
Kernel::DiskBuffer *DiskBuf(NULL);
if(m_fileBasedTargetWS)
{
DiskBuf = bc->getFileIO();
}
this->totalLoaded = 0;
std::vector<API::IMDNode *> &boxes = m_BoxStruct.getBoxes();
for(size_t ib=0;ib<numBoxes;ib++)
{
auto box = boxes[ib];
if(!box->isBox())continue;
// load all contributed events into current box;
this->loadEventsFromSubBoxes(boxes[ib]);
if(DiskBuf)
{
if(box->getDataInMemorySize()>0)
{ // data position has been already precalculated
box->getISaveable()->save();
box->clearDataFromMemory();
//Kernel::ISaveable *Saver = box->getISaveable();
//DiskBuf->toWrite(Saver);
}
}
//else
//{ size_t ID = box->getID();
// uint64_t filePosition = targetEventIndexes[2*ID];
// box->saveAt(saver.get(), filePosition);
//}
//
//if (!Parallel)
//{
// // Run the task serially only
// task->run();
// delete task;
//}
//else
//{
// // Enqueue to run in parallel (at the joinAll() call below).
// ts->push(task);
//}
prog->reportIncrement(ib,"Loading and merging box data");
}
if(DiskBuf)
{
DiskBuf->flushCache();
bc->getFileIO()->flushData();
}
//// Run any final tasks
//tp.joinAll();
g_log.information() << overallTime << " to do all the adding." << std::endl;
// Close any open file handle
clearEventLoaders();
// Finish things up
this->finalizeOutput(outputFile);
}
void SaveMD::doSaveEvents(typename MDEventWorkspace<MDE, nd>::sptr ws) {
std::string filename = getPropertyValue("Filename");
bool update = getProperty("UpdateFileBackEnd");
bool MakeFileBacked = getProperty("MakeFileBacked");
bool wsIsFileBacked = ws->isFileBacked();
if (update && MakeFileBacked)
throw std::invalid_argument(
"Please choose either UpdateFileBackEnd or MakeFileBacked, not both.");
if (MakeFileBacked && wsIsFileBacked)
throw std::invalid_argument(
"You picked MakeFileBacked but the workspace is already file-backed!");
BoxController_sptr bc = ws->getBoxController();
if (!wsIsFileBacked) { // Erase the file if it exists
Poco::File oldFile(filename);
if (oldFile.exists())
oldFile.remove();
}
auto prog = new Progress(this, 0.0, 0.05, 1);
if (update) // workspace has its own file and ignores any changes to the
// algorithm parameters
{
if (!ws->isFileBacked())
throw std::runtime_error(" attempt to update non-file backed workspace");
filename = bc->getFileIO()->getFileName();
}
//-----------------------------------------------------------------------------------------------------
// create or open WS group and put there additional information about WS and
// its dimensions
int nDims = static_cast<int>(nd);
bool data_exist;
auto file = file_holder_type(MDBoxFlatTree::createOrOpenMDWSgroup(
filename, nDims, MDE::getTypeName(), false, data_exist));
// Save each NEW ExperimentInfo to a spot in the file
MDBoxFlatTree::saveExperimentInfos(file.get(), ws);
if (!update || !data_exist) {
MDBoxFlatTree::saveWSGenericInfo(file.get(), ws);
}
file->closeGroup();
file->close();
MDBoxFlatTree BoxFlatStruct;
//-----------------------------------------------------------------------------------------------------
if (update) // the workspace is already file backed;
{
// remove all boxes from the DiskBuffer. DB will calculate boxes positions
// on HDD.
bc->getFileIO()->flushCache();
// flatten the box structure; this will remember boxes file positions in the
// box structure
BoxFlatStruct.initFlatStructure(ws, filename);
} else // not file backed;
{
// the boxes file positions are unknown and we need to calculate it.
BoxFlatStruct.initFlatStructure(ws, filename);
// create saver class
auto Saver = boost::shared_ptr<API::IBoxControllerIO>(
new DataObjects::BoxControllerNeXusIO(bc.get()));
Saver->setDataType(sizeof(coord_t), MDE::getTypeName());
if (MakeFileBacked) {
// store saver with box controller
bc->setFileBacked(Saver, filename);
// get access to boxes array
std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes();
// calculate the position of the boxes on file, indicating to make them
// saveable and that the boxes were not saved.
BoxFlatStruct.setBoxesFilePositions(true);
prog->resetNumSteps(boxes.size(), 0.06, 0.90);
for (auto &boxe : boxes) {
auto saveableTag = boxe->getISaveable();
if (saveableTag) // only boxes can be saveable
{
// do not spend time on empty boxes
if (boxe->getDataInMemorySize() == 0)
continue;
// save boxes directly using the boxes file postion, precalculated in
// boxFlatStructure.
saveableTag->save();
// remove boxes data from memory. This will actually correctly set the
// tag indicatin that data were not loaded.
saveableTag->clearDataFromMemory();
// put boxes into write buffer wich will save them when necessary
// Saver->toWrite(saveTag);
prog->report("Saving Box");
}
}
// remove everything from diskBuffer; (not sure if it really necessary
// but just in case , should not make any harm)
Saver->flushCache();
// drop NeXus on HDD (not sure if it really necessary but just in case )
Saver->flushData();
} else // just save data, and finish with it
{
Saver->openFile(filename, "w");
BoxFlatStruct.setBoxesFilePositions(false);
std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes();
std::vector<uint64_t> &eventIndex = BoxFlatStruct.getEventIndex();
prog->resetNumSteps(boxes.size(), 0.06, 0.90);
for (size_t i = 0; i < boxes.size(); i++) {
if (eventIndex[2 * i + 1] == 0)
continue;
boxes[i]->saveAt(Saver.get(), eventIndex[2 * i]);
prog->report("Saving Box");
}
Saver->closeFile();
}
}
// -------------- Save Box Structure -------------------------------------
// OK, we've filled these big arrays of data representing flat box structrre.
// Save them.
progress(0.91, "Writing Box Data");
prog->resetNumSteps(8, 0.92, 1.00);
// Save box structure;
BoxFlatStruct.saveBoxStructure(filename);
delete prog;
ws->setFileNeedsUpdating(false);
}
