void CloneMDWorkspace::doClone(const typename MDEventWorkspace<MDE, nd>::sptr ws) { std::string outWSName = getPropertyValue("OutputWorkspace"); Progress prog(this, 0.0, 10.0, 100); BoxController_sptr bc = ws->getBoxController(); if (!bc) throw std::runtime_error("Error with InputWorkspace: no BoxController!"); if (bc->isFileBacked()) { // Generate a new filename to copy to prog.report("Copying File"); std::string originalFile = bc->getFilename(); std::string outFilename = getPropertyValue("Filename"); if (outFilename.empty()) { // Auto-generated name Poco::Path path = Poco::Path(originalFile).absolute(); std::string newName = path.getBaseName() + "_clone." + path.getExtension(); path.setFileName(newName); outFilename = path.toString(); } // Perform the copying g_log.notice() << "Cloned workspace file being copied to: " << outFilename << std::endl; Poco::File(originalFile).copyTo(outFilename); g_log.information() << "File copied successfully." << std::endl; // Now load it back IAlgorithm_sptr alg = createSubAlgorithm("LoadMD", 0.5, 1.0, false); alg->setPropertyValue("Filename", outFilename); alg->setPropertyValue("FileBackEnd", "1"); alg->setPropertyValue("Memory", "0"); //TODO: How much memory? alg->setPropertyValue("OutputWorkspace", outWSName); alg->executeAsSubAlg(); // Set the output workspace to this IMDEventWorkspace_sptr outWS = alg->getProperty("OutputWorkspace"); this->setProperty("OutputWorkspace", outWS); } else { // Perform the clone in memory. boost::shared_ptr<MDEventWorkspace<MDE,nd> > outWS(new MDEventWorkspace<MDE,nd>(*ws)); this->setProperty("OutputWorkspace", boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS) ); } }
void SliceMD::slice(typename MDEventWorkspace<MDE, nd>::sptr ws) { // Create the ouput workspace typename MDEventWorkspace<OMDE, ond>::sptr outWS( new MDEventWorkspace<OMDE, ond>()); for (size_t od = 0; od < m_binDimensions.size(); od++) { outWS->addDimension(m_binDimensions[od]); } outWS->setCoordinateSystem(ws->getSpecialCoordinateSystem()); outWS->initialize(); // Copy settings from the original box controller BoxController_sptr bc = ws->getBoxController(); // store wrute buffer size for the future // uint64_t writeBufSize = // bc->getFileIO()getDiskBuffer().getWriteBufferSize(); // and disable write buffer (if any) for input MD Events for this algorithm // purposes; // bc->setCacheParameters(1,0); BoxController_sptr obc = outWS->getBoxController(); // Use the "number of bins" as the "split into" parameter for (size_t od = 0; od < m_binDimensions.size(); od++) obc->setSplitInto(od, m_binDimensions[od]->getNBins()); obc->setSplitThreshold(bc->getSplitThreshold()); bool bTakeDepthFromInputWorkspace = getProperty("TakeMaxRecursionDepthFromInput"); int tempDepth = getProperty("MaxRecursionDepth"); size_t maxDepth = bTakeDepthFromInputWorkspace ? bc->getMaxDepth() : size_t(tempDepth); obc->setMaxDepth(maxDepth); // size_t outputSize = writeBufSize; // obc->setCacheParameters(sizeof(OMDE),outputSize); obc->resetNumBoxes(); // Perform the first box splitting outWS->splitBox(); size_t lastNumBoxes = obc->getTotalNumMDBoxes(); // --- File back end ? ---------------- std::string filename = getProperty("OutputFilename"); if (!filename.empty()) { // First save to the NXS file g_log.notice() << "Running SaveMD to create file back-end" << std::endl; IAlgorithm_sptr alg = createChildAlgorithm("SaveMD"); alg->setPropertyValue("Filename", filename); alg->setProperty("InputWorkspace", outWS); alg->setProperty("MakeFileBacked", true); alg->executeAsChildAlg(); if (!obc->isFileBacked()) throw std::runtime_error("SliceMD with file-backed output: Can not set " "up file-backed output workspace "); auto IOptr = obc->getFileIO(); size_t outBufSize = IOptr->getWriteBufferSize(); // the buffer size for resulting workspace; reasonable size is at least 10 // data chunk sizes (nice to verify) if (outBufSize < 10 * IOptr->getDataChunk()) { outBufSize = 10 * IOptr->getDataChunk(); IOptr->setWriteBufferSize(outBufSize); } } // Function defining which events (in the input dimensions) to place in the // output MDImplicitFunction *function = this->getImplicitFunctionForChunk(NULL, NULL); std::vector<API::IMDNode *> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, // hopefully. bool fileBackedWS = bc->isFileBacked(); if (fileBackedWS) API::IMDNode::sortObjByID(boxes); Progress *prog = new Progress(this, 0.0, 1.0, boxes.size()); // The root of the output workspace MDBoxBase<OMDE, ond> *outRootBox = outWS->getBox(); // if target workspace has events, we should count them as added uint64_t totalAdded = outWS->getNEvents(); uint64_t numSinceSplit = 0; // Go through every box for this chunk. // PARALLEL_FOR_IF( !bc->isFileBacked() ) for (int i = 0; i < int(boxes.size()); i++) { MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxes[i]); // Perform the binning in this separate method. if (box) { // An array to hold the rotated/transformed coordinates coord_t outCenter[ond]; const std::vector<MDE> &events = box->getConstEvents(); typename std::vector<MDE>::const_iterator it = events.begin(); typename std::vector<MDE>::const_iterator it_end = events.end(); for (; it != it_end; it++) { // Cache the center of the event (again for speed) const coord_t *inCenter = it->getCenter(); if (function->isPointContained(inCenter)) { // Now transform to the output dimensions m_transformFromOriginal->apply(inCenter, outCenter); // Create the event OMDE newEvent(it->getSignal(), it->getErrorSquared(), outCenter); // Copy extra data, if any copyEvent(*it, newEvent); // Add it to the workspace outRootBox->addEvent(newEvent); numSinceSplit++; } } box->releaseEvents(); // Ask BC if one needs to split boxes if (obc->shouldSplitBoxes(totalAdded, numSinceSplit, lastNumBoxes)) // if (numSinceSplit > 20000000 || (i == int(boxes.size()-1))) { // This splits up all the boxes according to split thresholds and sizes. Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts); outWS->splitAllIfNeeded(ts); tp.joinAll(); // Accumulate stats totalAdded += numSinceSplit; numSinceSplit = 0; lastNumBoxes = obc->getTotalNumMDBoxes(); // Progress reporting if (!fileBackedWS) prog->report(i); } if (fileBackedWS) { if (!(i % 10)) prog->report(i); } } // is box } // for each box in the vector prog->report(); outWS->splitAllIfNeeded(NULL); // Refresh all cache. outWS->refreshCache(); g_log.notice() << totalAdded << " " << OMDE::getTypeName() << "'s added to the output workspace." << std::endl; if (outWS->isFileBacked()) { // Update the file-back-end g_log.notice() << "Running SaveMD" << std::endl; IAlgorithm_sptr alg = createChildAlgorithm("SaveMD"); alg->setProperty("UpdateFileBackEnd", true); alg->setProperty("InputWorkspace", outWS); alg->executeAsChildAlg(); } // return the size of the input workspace write buffer to its initial value // bc->setCacheParameters(sizeof(MDE),writeBufSize); this->setProperty("OutputWorkspace", boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS)); delete prog; }
void BinMD::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) { BoxController_sptr bc = ws->getBoxController(); // store exisiting write buffer size for the future // uint64_t writeBufSize =bc->getDiskBuffer().getWriteBufferSize(); // and disable write buffer (if any) for input MD Events for this algorithm // purposes; // bc->setCacheParameters(1,0); // Cache some data to speed up accessing them a bit indexMultiplier = new size_t[m_outD]; for (size_t d = 0; d < m_outD; d++) { if (d > 0) indexMultiplier[d] = outWS->getIndexMultiplier()[d - 1]; else indexMultiplier[d] = 1; } signals = outWS->getSignalArray(); errors = outWS->getErrorSquaredArray(); numEvents = outWS->getNumEventsArray(); // Start with signal/error/numEvents at 0.0 outWS->setTo(0.0, 0.0, 0.0); // The dimension (in the output workspace) along which we chunk for parallel // processing // TODO: Find the smartest dimension to chunk against size_t chunkDimension = 0; // How many bins (in that dimension) per chunk. // Try to split it so each core will get 2 tasks: int chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins() / (PARALLEL_GET_MAX_THREADS * 2)); if (chunkNumBins < 1) chunkNumBins = 1; // Do we actually do it in parallel? bool doParallel = getProperty("Parallel"); // Not if file-backed! if (bc->isFileBacked()) doParallel = false; if (!doParallel) chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins()); // Total number of steps size_t progNumSteps = 0; if (prog) prog->setNotifyStep(0.1); if (prog) prog->resetNumSteps(100, 0.00, 1.0); // Run the chunks in parallel. There is no overlap in the output workspace so // it is thread safe to write to it.. // cppcheck-suppress syntaxError PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) ) for (int chunk = 0; chunk < int(m_binDimensions[chunkDimension]->getNBins()); chunk += chunkNumBins) { PARALLEL_START_INTERUPT_REGION // Region of interest for this chunk. std::vector<size_t> chunkMin(m_outD); std::vector<size_t> chunkMax(m_outD); for (size_t bd = 0; bd < m_outD; bd++) { // Same limits in the other dimensions chunkMin[bd] = 0; chunkMax[bd] = m_binDimensions[bd]->getNBins(); } // Parcel out a chunk in that single dimension dimension chunkMin[chunkDimension] = size_t(chunk); if (size_t(chunk + chunkNumBins) > m_binDimensions[chunkDimension]->getNBins()) chunkMax[chunkDimension] = m_binDimensions[chunkDimension]->getNBins(); else chunkMax[chunkDimension] = size_t(chunk + chunkNumBins); // Build an implicit function (it needs to be in the space of the // MDEventWorkspace) MDImplicitFunction *function = this->getImplicitFunctionForChunk(chunkMin.data(), chunkMax.data()); // Use getBoxes() to get an array with a pointer to each box std::vector<API::IMDNode *> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, // hopefully. if (bc->isFileBacked()) API::IMDNode::sortObjByID(boxes); // For progress reporting, the # of boxes if (prog) { PARALLEL_CRITICAL(BinMD_progress) { g_log.debug() << "Chunk " << chunk << ": found " << boxes.size() << " boxes within the implicit function.\n"; progNumSteps += boxes.size(); prog->setNumSteps(progNumSteps); } } // Go through every box for this chunk. for (auto &boxe : boxes) { MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxe); // Perform the binning in this separate method. if (box && !box->getIsMasked()) this->binMDBox(box, chunkMin.data(), chunkMax.data()); // Progress reporting if (prog) prog->report(); // For early cancelling of the loop if (this->m_cancel) break; } // for each box in the vector PARALLEL_END_INTERUPT_REGION } // for each chunk in parallel PARALLEL_CHECK_INTERUPT_REGION // Now the implicit function if (implicitFunction) { if (prog) prog->report("Applying implicit function."); signal_t nan = std::numeric_limits<signal_t>::quiet_NaN(); outWS->applyImplicitFunction(implicitFunction, nan, nan); } // return the size of the input workspace write buffer to its initial value // bc->setCacheParameters(sizeof(MDE),writeBufSize); }
void BinToMDHistoWorkspace::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) { BoxController_sptr bc = ws->getBoxController(); // Start with signal at 0.0 outWS->setTo(0.0, 0.0); // Cache some data to speed up accessing them a bit indexMultiplier = new size_t[outD]; for (size_t d=0; d<outD; d++) { if (d > 0) indexMultiplier[d] = outWS->getIndexMultiplier()[d-1]; else indexMultiplier[d] = 1; } signals = outWS->getSignalArray(); errors = outWS->getErrorSquaredArray(); // The dimension (in the output workspace) along which we chunk for parallel processing // TODO: Find the smartest dimension to chunk against size_t chunkDimension = 0; // How many bins (in that dimension) per chunk. // Try to split it so each core will get 2 tasks: int chunkNumBins = int(binDimensions[chunkDimension]->getNBins() / (Mantid::Kernel::ThreadPool::getNumPhysicalCores() * 2)); if (chunkNumBins < 1) chunkNumBins = 1; // Do we actually do it in parallel? bool doParallel = getProperty("Parallel"); // Not if file-backed! if (bc->isFileBacked()) doParallel = false; if (!doParallel) chunkNumBins = int(binDimensions[chunkDimension]->getNBins()); // Total number of steps size_t progNumSteps = 0; if (prog) prog->setNotifyStep(0.1); if (prog) prog->resetNumSteps(100, 0.00, 1.0); // Run the chunks in parallel. There is no overlap in the output workspace so it is // thread safe to write to it.. PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) ) for(int chunk=0; chunk < int(binDimensions[chunkDimension]->getNBins()); chunk += chunkNumBins) { PARALLEL_START_INTERUPT_REGION // Region of interest for this chunk. size_t * chunkMin = new size_t[outD]; size_t * chunkMax = new size_t[outD]; for (size_t bd=0; bd<outD; bd++) { // Same limits in the other dimensions chunkMin[bd] = 0; chunkMax[bd] = binDimensions[bd]->getNBins(); } // Parcel out a chunk in that single dimension dimension chunkMin[chunkDimension] = size_t(chunk); if (size_t(chunk+chunkNumBins) > binDimensions[chunkDimension]->getNBins()) chunkMax[chunkDimension] = binDimensions[chunkDimension]->getNBins(); else chunkMax[chunkDimension] = size_t(chunk+chunkNumBins); // Build an implicit function (it needs to be in the space of the MDEventWorkspace) MDImplicitFunction * function = this->getImplicitFunctionForChunk(chunkMin, chunkMax); // Use getBoxes() to get an array with a pointer to each box std::vector<IMDBox<MDE,nd>*> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, hopefully. if (bc->isFileBacked()) IMDBox<MDE, nd>::sortBoxesByFilePos(boxes); // For progress reporting, the # of boxes if (prog) { PARALLEL_CRITICAL(BinToMDHistoWorkspace_progress) { std::cout << "Chunk " << chunk << ": found " << boxes.size() << " boxes within the implicit function." << std::endl; progNumSteps += boxes.size(); prog->setNumSteps( progNumSteps ); } } // Go through every box for this chunk. for (size_t i=0; i<boxes.size(); i++) { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]); // Perform the binning in this separate method. if (box) this->binMDBox(box, chunkMin, chunkMax); // Progress reporting if (prog) prog->report(); }// for each box in the vector PARALLEL_END_INTERUPT_REGION } // for each chunk in parallel PARALLEL_CHECK_INTERUPT_REGION // Now the implicit function if (implicitFunction) { prog->report("Applying implicit function."); signal_t nan = std::numeric_limits<signal_t>::quiet_NaN(); outWS->applyImplicitFunction(implicitFunction, nan, nan); } }