void BinMD::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) {
BoxController_sptr bc = ws->getBoxController();
// store exisiting write buffer size for the future
// uint64_t writeBufSize =bc->getDiskBuffer().getWriteBufferSize();
// and disable write buffer (if any) for input MD Events for this algorithm
// purposes;
// bc->setCacheParameters(1,0);
// Cache some data to speed up accessing them a bit
indexMultiplier = new size_t[m_outD];
for (size_t d = 0; d < m_outD; d++) {
if (d > 0)
indexMultiplier[d] = outWS->getIndexMultiplier()[d - 1];
else
indexMultiplier[d] = 1;
}
signals = outWS->getSignalArray();
errors = outWS->getErrorSquaredArray();
numEvents = outWS->getNumEventsArray();
// Start with signal/error/numEvents at 0.0
outWS->setTo(0.0, 0.0, 0.0);
// The dimension (in the output workspace) along which we chunk for parallel
// processing
// TODO: Find the smartest dimension to chunk against
size_t chunkDimension = 0;
// How many bins (in that dimension) per chunk.
// Try to split it so each core will get 2 tasks:
int chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins() /
(PARALLEL_GET_MAX_THREADS * 2));
if (chunkNumBins < 1)
chunkNumBins = 1;
// Do we actually do it in parallel?
bool doParallel = getProperty("Parallel");
// Not if file-backed!
if (bc->isFileBacked())
doParallel = false;
if (!doParallel)
chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins());
// Total number of steps
size_t progNumSteps = 0;
if (prog)
prog->setNotifyStep(0.1);
if (prog)
prog->resetNumSteps(100, 0.00, 1.0);
// Run the chunks in parallel. There is no overlap in the output workspace so
// it is thread safe to write to it..
// cppcheck-suppress syntaxError
PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) )
for (int chunk = 0;
chunk < int(m_binDimensions[chunkDimension]->getNBins());
chunk += chunkNumBins) {
PARALLEL_START_INTERUPT_REGION
// Region of interest for this chunk.
std::vector<size_t> chunkMin(m_outD);
std::vector<size_t> chunkMax(m_outD);
for (size_t bd = 0; bd < m_outD; bd++) {
// Same limits in the other dimensions
chunkMin[bd] = 0;
chunkMax[bd] = m_binDimensions[bd]->getNBins();
}
// Parcel out a chunk in that single dimension dimension
chunkMin[chunkDimension] = size_t(chunk);
if (size_t(chunk + chunkNumBins) >
m_binDimensions[chunkDimension]->getNBins())
chunkMax[chunkDimension] = m_binDimensions[chunkDimension]->getNBins();
else
chunkMax[chunkDimension] = size_t(chunk + chunkNumBins);
// Build an implicit function (it needs to be in the space of the
// MDEventWorkspace)
MDImplicitFunction *function =
this->getImplicitFunctionForChunk(chunkMin.data(), chunkMax.data());
// Use getBoxes() to get an array with a pointer to each box
std::vector<API::IMDNode *> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time,
// hopefully.
if (bc->isFileBacked())
API::IMDNode::sortObjByID(boxes);
// For progress reporting, the # of boxes
if (prog) {
PARALLEL_CRITICAL(BinMD_progress) {
g_log.debug() << "Chunk " << chunk << ": found " << boxes.size()
<< " boxes within the implicit function.\n";
progNumSteps += boxes.size();
prog->setNumSteps(progNumSteps);
}
}
// Go through every box for this chunk.
for (auto &boxe : boxes) {
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxe);
// Perform the binning in this separate method.
if (box && !box->getIsMasked())
this->binMDBox(box, chunkMin.data(), chunkMax.data());
// Progress reporting
if (prog)
prog->report();
// For early cancelling of the loop
if (this->m_cancel)
break;
} // for each box in the vector
PARALLEL_END_INTERUPT_REGION
} // for each chunk in parallel
PARALLEL_CHECK_INTERUPT_REGION
// Now the implicit function
if (implicitFunction) {
if (prog)
prog->report("Applying implicit function.");
signal_t nan = std::numeric_limits<signal_t>::quiet_NaN();
outWS->applyImplicitFunction(implicitFunction, nan, nan);
}
// return the size of the input workspace write buffer to its initial value
// bc->setCacheParameters(sizeof(MDE),writeBufSize);
}
void SliceMD::slice(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Create the ouput workspace
typename MDEventWorkspace<OMDE, ond>::sptr outWS(
new MDEventWorkspace<OMDE, ond>());
for (auto &binDimension : m_binDimensions) {
outWS->addDimension(binDimension);
}
outWS->setCoordinateSystem(ws->getSpecialCoordinateSystem());
outWS->initialize();
// Copy settings from the original box controller
BoxController_sptr bc = ws->getBoxController();
// store wrute buffer size for the future
// uint64_t writeBufSize =
// bc->getFileIO()getDiskBuffer().getWriteBufferSize();
// and disable write buffer (if any) for input MD Events for this algorithm
// purposes;
// bc->setCacheParameters(1,0);
BoxController_sptr obc = outWS->getBoxController();
// Use the "number of bins" as the "split into" parameter for the top level
for (size_t od = 0; od < m_binDimensions.size(); od++) {
obc->setSplitTopInto(od, m_binDimensions[od]->getNBins());
obc->setSplitInto(od, bc->getSplitInto(od));
}
obc->setSplitThreshold(bc->getSplitThreshold());
bool bTakeDepthFromInputWorkspace =
getProperty("TakeMaxRecursionDepthFromInput");
int tempDepth = getProperty("MaxRecursionDepth");
size_t maxDepth =
bTakeDepthFromInputWorkspace ? bc->getMaxDepth() : size_t(tempDepth);
obc->setMaxDepth(maxDepth);
// size_t outputSize = writeBufSize;
// obc->setCacheParameters(sizeof(OMDE),outputSize);
obc->resetNumBoxes();
// Perform the first box splitting
outWS->splitBox();
size_t lastNumBoxes = obc->getTotalNumMDBoxes();
// --- File back end ? ----------------
std::string filename = getProperty("OutputFilename");
if (!filename.empty()) {
// First save to the NXS file
g_log.notice() << "Running SaveMD to create file back-end\n";
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setPropertyValue("Filename", filename);
alg->setProperty("InputWorkspace", outWS);
alg->setProperty("MakeFileBacked", true);
alg->executeAsChildAlg();
if (!obc->isFileBacked())
throw std::runtime_error("SliceMD with file-backed output: Can not set "
"up file-backed output workspace ");
auto IOptr = obc->getFileIO();
size_t outBufSize = IOptr->getWriteBufferSize();
// the buffer size for resulting workspace; reasonable size is at least 10
// data chunk sizes (nice to verify)
if (outBufSize < 10 * IOptr->getDataChunk()) {
outBufSize = 10 * IOptr->getDataChunk();
IOptr->setWriteBufferSize(outBufSize);
}
}
// Function defining which events (in the input dimensions) to place in the
// output
MDImplicitFunction *function =
this->getImplicitFunctionForChunk(nullptr, nullptr);
std::vector<API::IMDNode *> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time,
// hopefully.
bool fileBackedWS = bc->isFileBacked();
if (fileBackedWS)
API::IMDNode::sortObjByID(boxes);
auto prog = new Progress(this, 0.0, 1.0, boxes.size());
// The root of the output workspace
MDBoxBase<OMDE, ond> *outRootBox = outWS->getBox();
// if target workspace has events, we should count them as added
uint64_t totalAdded = outWS->getNEvents();
uint64_t numSinceSplit = 0;
// Go through every box for this chunk.
// PARALLEL_FOR_IF( !bc->isFileBacked() )
for (int i = 0; i < int(boxes.size()); i++) {
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxes[i]);
// Perform the binning in this separate method.
if (box && !box->getIsMasked()) {
// An array to hold the rotated/transformed coordinates
coord_t outCenter[ond];
const std::vector<MDE> &events = box->getConstEvents();
for (auto it = events.cbegin(); it != events.cend(); ++it) {
// Cache the center of the event (again for speed)
const coord_t *inCenter = it->getCenter();
if (function->isPointContained(inCenter)) {
// Now transform to the output dimensions
m_transformFromOriginal->apply(inCenter, outCenter);
// Create the event
OMDE newEvent(it->getSignal(), it->getErrorSquared(), outCenter);
// Copy extra data, if any
copyEvent(*it, newEvent);
// Add it to the workspace
if (outRootBox->addEvent(newEvent))
numSinceSplit++;
}
}
box->releaseEvents();
// Ask BC if one needs to split boxes
if (obc->shouldSplitBoxes(totalAdded, numSinceSplit, lastNumBoxes))
// if (numSinceSplit > 20000000 || (i == int(boxes.size()-1)))
{
// This splits up all the boxes according to split thresholds and sizes.
Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
outWS->splitAllIfNeeded(ts);
tp.joinAll();
// Accumulate stats
totalAdded += numSinceSplit;
numSinceSplit = 0;
lastNumBoxes = obc->getTotalNumMDBoxes();
// Progress reporting
if (!fileBackedWS)
prog->report(i);
}
if (fileBackedWS) {
if (!(i % 10))
prog->report(i);
}
} // is box
} // for each box in the vector
prog->report();
outWS->splitAllIfNeeded(nullptr);
// Refresh all cache.
outWS->refreshCache();
// Account for events that were added after the last split
totalAdded += numSinceSplit;
g_log.notice() << totalAdded << " " << OMDE::getTypeName()
<< "s added to the output workspace.\n";
if (outWS->isFileBacked()) {
// Update the file-back-end
g_log.notice() << "Running SaveMD\n";
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setProperty("UpdateFileBackEnd", true);
alg->setProperty("InputWorkspace", outWS);
alg->executeAsChildAlg();
}
// Pass on the display normalization from the input event workspace to the
// output event workspace
IMDEventWorkspace_sptr outEvent =
boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS);
outEvent->setDisplayNormalization(ws->displayNormalization());
outEvent->setDisplayNormalizationHisto(ws->displayNormalizationHisto());
// return the size of the input workspace write buffer to its initial value
// bc->setCacheParameters(sizeof(MDE),writeBufSize);
this->setProperty("OutputWorkspace", outEvent);
delete prog;
}
