void CloneMDWorkspace::doClone(const typename MDEventWorkspace<MDE, nd>::sptr ws)
{
std::string outWSName = getPropertyValue("OutputWorkspace");
Progress prog(this, 0.0, 10.0, 100);
BoxController_sptr bc = ws->getBoxController();
if (!bc) throw std::runtime_error("Error with InputWorkspace: no BoxController!");
if (bc->isFileBacked())
{
// Generate a new filename to copy to
prog.report("Copying File");
std::string originalFile = bc->getFilename();
std::string outFilename = getPropertyValue("Filename");
if (outFilename.empty())
{
// Auto-generated name
Poco::Path path = Poco::Path(originalFile).absolute();
std::string newName = path.getBaseName() + "_clone." + path.getExtension();
path.setFileName(newName);
outFilename = path.toString();
}
// Perform the copying
g_log.notice() << "Cloned workspace file being copied to: " << outFilename << std::endl;
Poco::File(originalFile).copyTo(outFilename);
g_log.information() << "File copied successfully." << std::endl;
// Now load it back
IAlgorithm_sptr alg = createSubAlgorithm("LoadMD", 0.5, 1.0, false);
alg->setPropertyValue("Filename", outFilename);
alg->setPropertyValue("FileBackEnd", "1");
alg->setPropertyValue("Memory", "0"); //TODO: How much memory?
alg->setPropertyValue("OutputWorkspace", outWSName);
alg->executeAsSubAlg();
// Set the output workspace to this
IMDEventWorkspace_sptr outWS = alg->getProperty("OutputWorkspace");
this->setProperty("OutputWorkspace", outWS);
}
else
{
// Perform the clone in memory.
boost::shared_ptr<MDEventWorkspace<MDE,nd> > outWS(new MDEventWorkspace<MDE,nd>(*ws));
this->setProperty("OutputWorkspace", boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS) );
}
}
void SliceMD::slice(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Create the ouput workspace
typename MDEventWorkspace<OMDE, ond>::sptr outWS(
new MDEventWorkspace<OMDE, ond>());
for (size_t od = 0; od < m_binDimensions.size(); od++) {
outWS->addDimension(m_binDimensions[od]);
}
outWS->setCoordinateSystem(ws->getSpecialCoordinateSystem());
outWS->initialize();
// Copy settings from the original box controller
BoxController_sptr bc = ws->getBoxController();
// store wrute buffer size for the future
// uint64_t writeBufSize =
// bc->getFileIO()getDiskBuffer().getWriteBufferSize();
// and disable write buffer (if any) for input MD Events for this algorithm
// purposes;
// bc->setCacheParameters(1,0);
BoxController_sptr obc = outWS->getBoxController();
// Use the "number of bins" as the "split into" parameter
for (size_t od = 0; od < m_binDimensions.size(); od++)
obc->setSplitInto(od, m_binDimensions[od]->getNBins());
obc->setSplitThreshold(bc->getSplitThreshold());
bool bTakeDepthFromInputWorkspace =
getProperty("TakeMaxRecursionDepthFromInput");
int tempDepth = getProperty("MaxRecursionDepth");
size_t maxDepth =
bTakeDepthFromInputWorkspace ? bc->getMaxDepth() : size_t(tempDepth);
obc->setMaxDepth(maxDepth);
// size_t outputSize = writeBufSize;
// obc->setCacheParameters(sizeof(OMDE),outputSize);
obc->resetNumBoxes();
// Perform the first box splitting
outWS->splitBox();
size_t lastNumBoxes = obc->getTotalNumMDBoxes();
// --- File back end ? ----------------
std::string filename = getProperty("OutputFilename");
if (!filename.empty()) {
// First save to the NXS file
g_log.notice() << "Running SaveMD to create file back-end" << std::endl;
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setPropertyValue("Filename", filename);
alg->setProperty("InputWorkspace", outWS);
alg->setProperty("MakeFileBacked", true);
alg->executeAsChildAlg();
if (!obc->isFileBacked())
throw std::runtime_error("SliceMD with file-backed output: Can not set "
"up file-backed output workspace ");
auto IOptr = obc->getFileIO();
size_t outBufSize = IOptr->getWriteBufferSize();
// the buffer size for resulting workspace; reasonable size is at least 10
// data chunk sizes (nice to verify)
if (outBufSize < 10 * IOptr->getDataChunk()) {
outBufSize = 10 * IOptr->getDataChunk();
IOptr->setWriteBufferSize(outBufSize);
}
}
// Function defining which events (in the input dimensions) to place in the
// output
MDImplicitFunction *function = this->getImplicitFunctionForChunk(NULL, NULL);
std::vector<API::IMDNode *> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time,
// hopefully.
bool fileBackedWS = bc->isFileBacked();
if (fileBackedWS)
API::IMDNode::sortObjByID(boxes);
Progress *prog = new Progress(this, 0.0, 1.0, boxes.size());
// The root of the output workspace
MDBoxBase<OMDE, ond> *outRootBox = outWS->getBox();
// if target workspace has events, we should count them as added
uint64_t totalAdded = outWS->getNEvents();
uint64_t numSinceSplit = 0;
// Go through every box for this chunk.
// PARALLEL_FOR_IF( !bc->isFileBacked() )
for (int i = 0; i < int(boxes.size()); i++) {
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxes[i]);
// Perform the binning in this separate method.
if (box) {
// An array to hold the rotated/transformed coordinates
coord_t outCenter[ond];
const std::vector<MDE> &events = box->getConstEvents();
typename std::vector<MDE>::const_iterator it = events.begin();
typename std::vector<MDE>::const_iterator it_end = events.end();
for (; it != it_end; it++) {
// Cache the center of the event (again for speed)
const coord_t *inCenter = it->getCenter();
if (function->isPointContained(inCenter)) {
// Now transform to the output dimensions
m_transformFromOriginal->apply(inCenter, outCenter);
// Create the event
OMDE newEvent(it->getSignal(), it->getErrorSquared(), outCenter);
// Copy extra data, if any
copyEvent(*it, newEvent);
// Add it to the workspace
outRootBox->addEvent(newEvent);
numSinceSplit++;
}
}
box->releaseEvents();
// Ask BC if one needs to split boxes
if (obc->shouldSplitBoxes(totalAdded, numSinceSplit, lastNumBoxes))
// if (numSinceSplit > 20000000 || (i == int(boxes.size()-1)))
{
// This splits up all the boxes according to split thresholds and sizes.
Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
outWS->splitAllIfNeeded(ts);
tp.joinAll();
// Accumulate stats
totalAdded += numSinceSplit;
numSinceSplit = 0;
lastNumBoxes = obc->getTotalNumMDBoxes();
// Progress reporting
if (!fileBackedWS)
prog->report(i);
}
if (fileBackedWS) {
if (!(i % 10))
prog->report(i);
}
} // is box
} // for each box in the vector
prog->report();
outWS->splitAllIfNeeded(NULL);
// Refresh all cache.
outWS->refreshCache();
g_log.notice() << totalAdded << " " << OMDE::getTypeName()
<< "'s added to the output workspace." << std::endl;
if (outWS->isFileBacked()) {
// Update the file-back-end
g_log.notice() << "Running SaveMD" << std::endl;
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setProperty("UpdateFileBackEnd", true);
alg->setProperty("InputWorkspace", outWS);
alg->executeAsChildAlg();
}
// return the size of the input workspace write buffer to its initial value
// bc->setCacheParameters(sizeof(MDE),writeBufSize);
this->setProperty("OutputWorkspace",
boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS));
delete prog;
}
void BinMD::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) {
BoxController_sptr bc = ws->getBoxController();
// store exisiting write buffer size for the future
// uint64_t writeBufSize =bc->getDiskBuffer().getWriteBufferSize();
// and disable write buffer (if any) for input MD Events for this algorithm
// purposes;
// bc->setCacheParameters(1,0);
// Cache some data to speed up accessing them a bit
indexMultiplier = new size_t[m_outD];
for (size_t d = 0; d < m_outD; d++) {
if (d > 0)
indexMultiplier[d] = outWS->getIndexMultiplier()[d - 1];
else
indexMultiplier[d] = 1;
}
signals = outWS->getSignalArray();
errors = outWS->getErrorSquaredArray();
numEvents = outWS->getNumEventsArray();
// Start with signal/error/numEvents at 0.0
outWS->setTo(0.0, 0.0, 0.0);
// The dimension (in the output workspace) along which we chunk for parallel
// processing
// TODO: Find the smartest dimension to chunk against
size_t chunkDimension = 0;
// How many bins (in that dimension) per chunk.
// Try to split it so each core will get 2 tasks:
int chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins() /
(PARALLEL_GET_MAX_THREADS * 2));
if (chunkNumBins < 1)
chunkNumBins = 1;
// Do we actually do it in parallel?
bool doParallel = getProperty("Parallel");
// Not if file-backed!
if (bc->isFileBacked())
doParallel = false;
if (!doParallel)
chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins());
// Total number of steps
size_t progNumSteps = 0;
if (prog)
prog->setNotifyStep(0.1);
if (prog)
prog->resetNumSteps(100, 0.00, 1.0);
// Run the chunks in parallel. There is no overlap in the output workspace so
// it is thread safe to write to it..
// cppcheck-suppress syntaxError
PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) )
for (int chunk = 0;
chunk < int(m_binDimensions[chunkDimension]->getNBins());
chunk += chunkNumBins) {
PARALLEL_START_INTERUPT_REGION
// Region of interest for this chunk.
std::vector<size_t> chunkMin(m_outD);
std::vector<size_t> chunkMax(m_outD);
for (size_t bd = 0; bd < m_outD; bd++) {
// Same limits in the other dimensions
chunkMin[bd] = 0;
chunkMax[bd] = m_binDimensions[bd]->getNBins();
}
// Parcel out a chunk in that single dimension dimension
chunkMin[chunkDimension] = size_t(chunk);
if (size_t(chunk + chunkNumBins) >
m_binDimensions[chunkDimension]->getNBins())
chunkMax[chunkDimension] = m_binDimensions[chunkDimension]->getNBins();
else
chunkMax[chunkDimension] = size_t(chunk + chunkNumBins);
// Build an implicit function (it needs to be in the space of the
// MDEventWorkspace)
MDImplicitFunction *function =
this->getImplicitFunctionForChunk(chunkMin.data(), chunkMax.data());
// Use getBoxes() to get an array with a pointer to each box
std::vector<API::IMDNode *> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time,
// hopefully.
if (bc->isFileBacked())
API::IMDNode::sortObjByID(boxes);
// For progress reporting, the # of boxes
if (prog) {
PARALLEL_CRITICAL(BinMD_progress) {
g_log.debug() << "Chunk " << chunk << ": found " << boxes.size()
<< " boxes within the implicit function.\n";
progNumSteps += boxes.size();
prog->setNumSteps(progNumSteps);
}
}
// Go through every box for this chunk.
for (auto &boxe : boxes) {
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxe);
// Perform the binning in this separate method.
if (box && !box->getIsMasked())
this->binMDBox(box, chunkMin.data(), chunkMax.data());
// Progress reporting
if (prog)
prog->report();
// For early cancelling of the loop
if (this->m_cancel)
break;
} // for each box in the vector
PARALLEL_END_INTERUPT_REGION
} // for each chunk in parallel
PARALLEL_CHECK_INTERUPT_REGION
// Now the implicit function
if (implicitFunction) {
if (prog)
prog->report("Applying implicit function.");
signal_t nan = std::numeric_limits<signal_t>::quiet_NaN();
outWS->applyImplicitFunction(implicitFunction, nan, nan);
}
// return the size of the input workspace write buffer to its initial value
// bc->setCacheParameters(sizeof(MDE),writeBufSize);
}
void BinToMDHistoWorkspace::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
BoxController_sptr bc = ws->getBoxController();
// Start with signal at 0.0
outWS->setTo(0.0, 0.0);
// Cache some data to speed up accessing them a bit
indexMultiplier = new size_t[outD];
for (size_t d=0; d<outD; d++)
{
if (d > 0)
indexMultiplier[d] = outWS->getIndexMultiplier()[d-1];
else
indexMultiplier[d] = 1;
}
signals = outWS->getSignalArray();
errors = outWS->getErrorSquaredArray();
// The dimension (in the output workspace) along which we chunk for parallel processing
// TODO: Find the smartest dimension to chunk against
size_t chunkDimension = 0;
// How many bins (in that dimension) per chunk.
// Try to split it so each core will get 2 tasks:
int chunkNumBins = int(binDimensions[chunkDimension]->getNBins() / (Mantid::Kernel::ThreadPool::getNumPhysicalCores() * 2));
if (chunkNumBins < 1) chunkNumBins = 1;
// Do we actually do it in parallel?
bool doParallel = getProperty("Parallel");
// Not if file-backed!
if (bc->isFileBacked()) doParallel = false;
if (!doParallel)
chunkNumBins = int(binDimensions[chunkDimension]->getNBins());
// Total number of steps
size_t progNumSteps = 0;
if (prog) prog->setNotifyStep(0.1);
if (prog) prog->resetNumSteps(100, 0.00, 1.0);
// Run the chunks in parallel. There is no overlap in the output workspace so it is
// thread safe to write to it..
PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) )
for(int chunk=0; chunk < int(binDimensions[chunkDimension]->getNBins()); chunk += chunkNumBins)
{
PARALLEL_START_INTERUPT_REGION
// Region of interest for this chunk.
size_t * chunkMin = new size_t[outD];
size_t * chunkMax = new size_t[outD];
for (size_t bd=0; bd<outD; bd++)
{
// Same limits in the other dimensions
chunkMin[bd] = 0;
chunkMax[bd] = binDimensions[bd]->getNBins();
}
// Parcel out a chunk in that single dimension dimension
chunkMin[chunkDimension] = size_t(chunk);
if (size_t(chunk+chunkNumBins) > binDimensions[chunkDimension]->getNBins())
chunkMax[chunkDimension] = binDimensions[chunkDimension]->getNBins();
else
chunkMax[chunkDimension] = size_t(chunk+chunkNumBins);
// Build an implicit function (it needs to be in the space of the MDEventWorkspace)
MDImplicitFunction * function = this->getImplicitFunctionForChunk(chunkMin, chunkMax);
// Use getBoxes() to get an array with a pointer to each box
std::vector<IMDBox<MDE,nd>*> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time, hopefully.
if (bc->isFileBacked())
IMDBox<MDE, nd>::sortBoxesByFilePos(boxes);
// For progress reporting, the # of boxes
if (prog)
{
PARALLEL_CRITICAL(BinToMDHistoWorkspace_progress)
{
std::cout << "Chunk " << chunk << ": found " << boxes.size() << " boxes within the implicit function." << std::endl;
progNumSteps += boxes.size();
prog->setNumSteps( progNumSteps );
}
}
// Go through every box for this chunk.
for (size_t i=0; i<boxes.size(); i++)
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]);
// Perform the binning in this separate method.
if (box)
this->binMDBox(box, chunkMin, chunkMax);
// Progress reporting
if (prog) prog->report();
}// for each box in the vector
PARALLEL_END_INTERUPT_REGION
} // for each chunk in parallel
PARALLEL_CHECK_INTERUPT_REGION
// Now the implicit function
if (implicitFunction)
{
prog->report("Applying implicit function.");
signal_t nan = std::numeric_limits<signal_t>::quiet_NaN();
outWS->applyImplicitFunction(implicitFunction, nan, nan);
}
}
