void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) { // Are we using the file back end? bool fileBackEnd = getProperty("FileBackEnd"); if (fileBackEnd && m_BoxStructureAndMethadata) throw std::invalid_argument("Combination of BoxStructureOnly or " "MetaDataOnly were set to TRUE with " "fileBackEnd " ": this is not possible."); CPUTimer tim; auto prog = new Progress(this, 0.0, 1.0, 100); prog->report("Opening file."); std::string title; try { m_file->getAttr("title", title); } catch (std::exception &) { // Leave the title blank if error on loading } ws->setTitle(title); // Load the WorkspaceHistory "process" if (this->getProperty("LoadHistory")) { ws->history().loadNexus(m_file.get()); } this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws)); m_file->closeGroup(); m_file->close(); // Add each of the dimension for (size_t d = 0; d < nd; d++) ws->addDimension(m_dims[d]); // Coordinate system ws->setCoordinateSystem(m_coordSystem); // ----------------------------------------- Box Structure // ------------------------------ prog->report("Reading box structure from HDD."); MDBoxFlatTree FlatBoxTree; int nDims = static_cast<int>(nd); // should be safe FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName()); BoxController_sptr bc = ws->getBoxController(); bc->fromXMLString(FlatBoxTree.getBCXMLdescr()); prog->report("Restoring box structure and connectivity"); std::vector<API::IMDNode *> boxTree; FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd, m_BoxStructureAndMethadata); size_t numBoxes = boxTree.size(); // ---------------------------------------- DEAL WITH BOXES // ------------------------------------ if (fileBackEnd) { // TODO:: call to the file format factory auto loader = boost::shared_ptr<API::IBoxControllerIO>( new DataObjects::BoxControllerNeXusIO(bc.get())); loader->setDataType(sizeof(coord_t), MDE::getTypeName()); bc->setFileBacked(loader, m_filename); // boxes have been already made file-backed when restoring the boxTree; // How much memory for the cache? { // TODO: Clean up, only a write buffer now double mb = getProperty("Memory"); // Defaults have changed, default disk buffer size should be 10 data // chunks TODO: find optimal, 100 may be better. if (mb <= 0) mb = double(10 * loader->getDataChunk() * sizeof(MDE)) / double(1024 * 1024); // Express the cache memory in units of number of events. uint64_t cacheMemory = static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1; // Set these values in the diskMRU bc->getFileIO()->setWriteBufferSize(cacheMemory); g_log.information() << "Setting a DiskBuffer cache size of " << mb << " MB, or " << cacheMemory << " events.\n"; } } // Not file back end else if (!m_BoxStructureAndMethadata) { // ---------------------------------------- READ IN THE BOXES // ------------------------------------ // TODO:: call to the file format factory auto loader = file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get())); loader->setDataType(sizeof(coord_t), MDE::getTypeName()); loader->openFile(m_filename, "r"); const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex(); prog->setNumSteps(numBoxes); for (size_t i = 0; i < numBoxes; i++) { prog->report(); MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]); if (!box) continue; if (BoxEventIndex[2 * i + 1] > 0) // Load in memory NOT using the file as the back-end, { boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]); boxTree[i]->loadAndAddFrom( loader.get(), BoxEventIndex[2 * i], static_cast<size_t>(BoxEventIndex[2 * i + 1])); } } loader->closeFile(); } else // box structure and metadata only { } g_log.debug() << tim << " to create all the boxes and fill them with events.\n"; // Box of ID 0 is the head box. ws->setBox(boxTree[0]); // Make sure the max ID is ok for later ID generation bc->setMaxId(numBoxes); // end-of bMetaDataOnly // Refresh cache // TODO:if(!fileBackEnd)ws->refreshCache(); ws->refreshCache(); g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints() << " points after refresh.\n"; g_log.debug() << tim << " to finish up.\n"; delete prog; }
/** Perform the merging, but clone the initial workspace and use the same splitting * as its structure is equivalent to the partial box structures. * * @param ws :: first MDEventWorkspace in the list to merge to. * @param outputFile :: the name of the output file where file-based workspace should be saved */ void MergeMDFiles::doExecByCloning(Mantid::API::IMDEventWorkspace_sptr ws,const std::string &outputFile) { m_OutIWS = ws; m_MDEventType = ws->getEventTypeName(); // Run the tasks in parallel? TODO: enable //bool Parallel = this->getProperty("Parallel"); // Fix the box controller settings in the output workspace so that it splits normally BoxController_sptr bc = ws->getBoxController(); // set up internal variables characterizing the workspace. m_nDims = static_cast<int>(bc->getNDims()); // Fix the max depth to something bigger. bc->setMaxDepth(20); bc->setSplitThreshold(5000); auto saver = boost::shared_ptr<API::IBoxControllerIO>(new MDEvents::BoxControllerNeXusIO(bc.get())); saver->setDataType(sizeof(coord_t),m_MDEventType); if(m_fileBasedTargetWS) { bc->setFileBacked(saver,outputFile); // Complete the file-back-end creation. g_log.notice() << "Setting cache to 400 MB write." << std::endl; bc->getFileIO()->setWriteBufferSize(400000000/m_OutIWS->sizeofEvent()); } /* else { saver->openFile(outputFile,"w"); }*/ // Init box structure used for memory/file space calculations m_BoxStruct.initFlatStructure(ws,outputFile); // First, load all the box data and experiment info and calculate file positions of the target workspace this->loadBoxData(); size_t numBoxes = m_BoxStruct.getNBoxes(); // Progress report based on events processed. this->prog = new Progress(this, 0.1, 0.9, size_t(numBoxes)); prog->setNotifyStep(0.1); // For tracking progress //uint64_t totalEventsInTasks = 0; // Prepare thread pool CPUTimer overallTime; ThreadSchedulerFIFO * ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts); Kernel::DiskBuffer *DiskBuf(NULL); if(m_fileBasedTargetWS) { DiskBuf = bc->getFileIO(); } this->totalLoaded = 0; std::vector<API::IMDNode *> &boxes = m_BoxStruct.getBoxes(); for(size_t ib=0;ib<numBoxes;ib++) { auto box = boxes[ib]; if(!box->isBox())continue; // load all contributed events into current box; this->loadEventsFromSubBoxes(boxes[ib]); if(DiskBuf) { if(box->getDataInMemorySize()>0) { // data position has been already precalculated box->getISaveable()->save(); box->clearDataFromMemory(); //Kernel::ISaveable *Saver = box->getISaveable(); //DiskBuf->toWrite(Saver); } } //else //{ size_t ID = box->getID(); // uint64_t filePosition = targetEventIndexes[2*ID]; // box->saveAt(saver.get(), filePosition); //} // //if (!Parallel) //{ // // Run the task serially only // task->run(); // delete task; //} //else //{ // // Enqueue to run in parallel (at the joinAll() call below). // ts->push(task); //} prog->reportIncrement(ib,"Loading and merging box data"); } if(DiskBuf) { DiskBuf->flushCache(); bc->getFileIO()->flushData(); } //// Run any final tasks //tp.joinAll(); g_log.information() << overallTime << " to do all the adding." << std::endl; // Close any open file handle clearEventLoaders(); // Finish things up this->finalizeOutput(outputFile); }
void SaveMD::doSaveEvents(typename MDEventWorkspace<MDE, nd>::sptr ws) { std::string filename = getPropertyValue("Filename"); bool update = getProperty("UpdateFileBackEnd"); bool MakeFileBacked = getProperty("MakeFileBacked"); bool wsIsFileBacked = ws->isFileBacked(); if (update && MakeFileBacked) throw std::invalid_argument( "Please choose either UpdateFileBackEnd or MakeFileBacked, not both."); if (MakeFileBacked && wsIsFileBacked) throw std::invalid_argument( "You picked MakeFileBacked but the workspace is already file-backed!"); BoxController_sptr bc = ws->getBoxController(); if (!wsIsFileBacked) { // Erase the file if it exists Poco::File oldFile(filename); if (oldFile.exists()) oldFile.remove(); } auto prog = new Progress(this, 0.0, 0.05, 1); if (update) // workspace has its own file and ignores any changes to the // algorithm parameters { if (!ws->isFileBacked()) throw std::runtime_error(" attempt to update non-file backed workspace"); filename = bc->getFileIO()->getFileName(); } //----------------------------------------------------------------------------------------------------- // create or open WS group and put there additional information about WS and // its dimensions int nDims = static_cast<int>(nd); bool data_exist; auto file = file_holder_type(MDBoxFlatTree::createOrOpenMDWSgroup( filename, nDims, MDE::getTypeName(), false, data_exist)); // Save each NEW ExperimentInfo to a spot in the file MDBoxFlatTree::saveExperimentInfos(file.get(), ws); if (!update || !data_exist) { MDBoxFlatTree::saveWSGenericInfo(file.get(), ws); } file->closeGroup(); file->close(); MDBoxFlatTree BoxFlatStruct; //----------------------------------------------------------------------------------------------------- if (update) // the workspace is already file backed; { // remove all boxes from the DiskBuffer. DB will calculate boxes positions // on HDD. bc->getFileIO()->flushCache(); // flatten the box structure; this will remember boxes file positions in the // box structure BoxFlatStruct.initFlatStructure(ws, filename); } else // not file backed; { // the boxes file positions are unknown and we need to calculate it. BoxFlatStruct.initFlatStructure(ws, filename); // create saver class auto Saver = boost::shared_ptr<API::IBoxControllerIO>( new DataObjects::BoxControllerNeXusIO(bc.get())); Saver->setDataType(sizeof(coord_t), MDE::getTypeName()); if (MakeFileBacked) { // store saver with box controller bc->setFileBacked(Saver, filename); // get access to boxes array std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes(); // calculate the position of the boxes on file, indicating to make them // saveable and that the boxes were not saved. BoxFlatStruct.setBoxesFilePositions(true); prog->resetNumSteps(boxes.size(), 0.06, 0.90); for (auto &boxe : boxes) { auto saveableTag = boxe->getISaveable(); if (saveableTag) // only boxes can be saveable { // do not spend time on empty boxes if (boxe->getDataInMemorySize() == 0) continue; // save boxes directly using the boxes file postion, precalculated in // boxFlatStructure. saveableTag->save(); // remove boxes data from memory. This will actually correctly set the // tag indicatin that data were not loaded. saveableTag->clearDataFromMemory(); // put boxes into write buffer wich will save them when necessary // Saver->toWrite(saveTag); prog->report("Saving Box"); } } // remove everything from diskBuffer; (not sure if it really necessary // but just in case , should not make any harm) Saver->flushCache(); // drop NeXus on HDD (not sure if it really necessary but just in case ) Saver->flushData(); } else // just save data, and finish with it { Saver->openFile(filename, "w"); BoxFlatStruct.setBoxesFilePositions(false); std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes(); std::vector<uint64_t> &eventIndex = BoxFlatStruct.getEventIndex(); prog->resetNumSteps(boxes.size(), 0.06, 0.90); for (size_t i = 0; i < boxes.size(); i++) { if (eventIndex[2 * i + 1] == 0) continue; boxes[i]->saveAt(Saver.get(), eventIndex[2 * i]); prog->report("Saving Box"); } Saver->closeFile(); } } // -------------- Save Box Structure ------------------------------------- // OK, we've filled these big arrays of data representing flat box structrre. // Save them. progress(0.91, "Writing Box Data"); prog->resetNumSteps(8, 0.92, 1.00); // Save box structure; BoxFlatStruct.saveBoxStructure(filename); delete prog; ws->setFileNeedsUpdating(false); }