void FakeMDEventData::addFakeUniformData(
typename MDEventWorkspace<MDE, nd>::sptr ws) {
std::vector<double> params = getProperty("UniformParams");
if (params.empty())
return;
bool randomEvents = true;
if (params[0] < 0) {
randomEvents = false;
params[0] = -params[0];
}
if (params.size() == 1) {
if (randomEvents) {
for (size_t d = 0; d < nd; ++d) {
params.push_back(ws->getDimension(d)->getMinimum());
params.push_back(ws->getDimension(d)->getMaximum());
}
} else // regular events
{
size_t nPoints = size_t(params[0]);
double Vol = 1;
for (size_t d = 0; d < nd; ++d)
Vol *= (ws->getDimension(d)->getMaximum() -
ws->getDimension(d)->getMinimum());
if (Vol == 0 || Vol > std::numeric_limits<float>::max())
throw std::invalid_argument(
" Domain ranges are not defined properly for workspace: " +
ws->getName());
double dV = Vol / double(nPoints);
double delta0 = std::pow(dV, 1. / double(nd));
for (size_t d = 0; d < nd; ++d) {
double min = ws->getDimension(d)->getMinimum();
params.push_back(min * (1 + FLT_EPSILON) - min + FLT_EPSILON);
double extent = ws->getDimension(d)->getMaximum() - min;
size_t nStrides = size_t(extent / delta0);
if (nStrides < 1)
nStrides = 1;
params.push_back(extent / static_cast<double>(nStrides));
}
}
}
if ((params.size() != 1 + nd * 2))
throw std::invalid_argument(
"UniformParams: needs to have ndims*2+1 arguments ");
if (randomEvents)
addFakeRandomData<MDE, nd>(params, ws);
else
addFakeRegularData<MDE, nd>(params, ws);
ws->splitBox();
Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
ws->splitAllIfNeeded(ts);
tp.joinAll();
ws->refreshCache();
}
void MinusMD::doMinus(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
if (!ws1 || !ws2)
throw std::runtime_error("Incompatible workspace types passed to MinusMD.");
MDBoxBase<MDE,nd> * box1 = ws1->getBox();
MDBoxBase<MDE,nd> * box2 = ws2->getBox();
Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());
// How many events you started with
size_t initial_numEvents = ws1->getNPoints();
// Make a leaf-only iterator through all boxes with events in the RHS workspace
MDBoxIterator<MDE,nd> it2(box2, 1000, true);
do
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
if (box)
{
// Copy the events from WS2 and add them into WS1
const std::vector<MDE> & events = box->getConstEvents();
// Perform a copy while flipping the signal
std::vector<MDE> eventsCopy;
eventsCopy.reserve(events.size());
for (auto it = events.begin(); it != events.end(); it++)
{
MDE eventCopy(*it);
eventCopy.setSignal( -eventCopy.getSignal());
eventsCopy.push_back(eventCopy);
}
// Add events, with bounds checking
box1->addEvents(eventsCopy);
box->releaseEvents();
}
prog.report("Adding Events");
} while (it2.next());
this->progress(0.41, "Splitting Boxes");
Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
ThreadScheduler * ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts, 0, prog2);
ws1->splitAllIfNeeded(ts);
prog2->resetNumSteps( ts->size(), 0.4, 0.6);
tp.joinAll();
this->progress(0.95, "Refreshing cache");
ws1->refreshCache();
// Set a marker that the file-back-end needs updating if the # of events changed.
if (ws1->getNPoints() != initial_numEvents)
ws1->setFileNeedsUpdating(true);
}
void MultiplyMD::execEventScalar(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
// Get the scalar multiplying
float scalar = float(m_rhs_scalar->dataY(0)[0]);
float scalarError = float(m_rhs_scalar->dataE(0)[0]);
float scalarRelativeErrorSquared = (scalarError * scalarError) / (scalar * scalar);
// Get all the MDBoxes contained
MDBoxBase<MDE,nd> * parentBox = ws->getBox();
std::vector<API::IMDNode *> boxes;
parentBox->getBoxes(boxes, 1000, true);
bool fileBackedTarget(false);
Kernel::DiskBuffer *dbuff(NULL);
if(ws->isFileBacked())
{
fileBackedTarget = true;
dbuff = ws->getBoxController()->getFileIO();
}
for (size_t i=0; i<boxes.size(); i++)
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]);
if (box)
{
typename std::vector<MDE> & events = box->getEvents();
size_t ic(events.size());
typename std::vector<MDE>::iterator it = events.begin();
typename std::vector<MDE>::iterator it_end = events.end();
for (; it != it_end; it++)
{
// Multiply weight by a scalar, propagating error
float oldSignal = it->getSignal();
float signal = oldSignal * scalar;
float errorSquared = signal * signal * (it->getErrorSquared() / (oldSignal * oldSignal) + scalarRelativeErrorSquared);
it->setSignal(signal);
it->setErrorSquared(errorSquared);
}
box->releaseEvents();
if(fileBackedTarget && ic>0)
{
Kernel::ISaveable *const pSaver(box->getISaveable());
dbuff->toWrite(pSaver);
}
}
}
// Recalculate the totals
ws->refreshCache();
// Mark file-backed workspace as dirty
ws->setFileNeedsUpdating(true);
}
void SliceMD::slice(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Create the ouput workspace
typename MDEventWorkspace<OMDE, ond>::sptr outWS(
new MDEventWorkspace<OMDE, ond>());
for (size_t od = 0; od < m_binDimensions.size(); od++) {
outWS->addDimension(m_binDimensions[od]);
}
outWS->setCoordinateSystem(ws->getSpecialCoordinateSystem());
outWS->initialize();
// Copy settings from the original box controller
BoxController_sptr bc = ws->getBoxController();
// store wrute buffer size for the future
// uint64_t writeBufSize =
// bc->getFileIO()getDiskBuffer().getWriteBufferSize();
// and disable write buffer (if any) for input MD Events for this algorithm
// purposes;
// bc->setCacheParameters(1,0);
BoxController_sptr obc = outWS->getBoxController();
// Use the "number of bins" as the "split into" parameter
for (size_t od = 0; od < m_binDimensions.size(); od++)
obc->setSplitInto(od, m_binDimensions[od]->getNBins());
obc->setSplitThreshold(bc->getSplitThreshold());
bool bTakeDepthFromInputWorkspace =
getProperty("TakeMaxRecursionDepthFromInput");
int tempDepth = getProperty("MaxRecursionDepth");
size_t maxDepth =
bTakeDepthFromInputWorkspace ? bc->getMaxDepth() : size_t(tempDepth);
obc->setMaxDepth(maxDepth);
// size_t outputSize = writeBufSize;
// obc->setCacheParameters(sizeof(OMDE),outputSize);
obc->resetNumBoxes();
// Perform the first box splitting
outWS->splitBox();
size_t lastNumBoxes = obc->getTotalNumMDBoxes();
// --- File back end ? ----------------
std::string filename = getProperty("OutputFilename");
if (!filename.empty()) {
// First save to the NXS file
g_log.notice() << "Running SaveMD to create file back-end" << std::endl;
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setPropertyValue("Filename", filename);
alg->setProperty("InputWorkspace", outWS);
alg->setProperty("MakeFileBacked", true);
alg->executeAsChildAlg();
if (!obc->isFileBacked())
throw std::runtime_error("SliceMD with file-backed output: Can not set "
"up file-backed output workspace ");
auto IOptr = obc->getFileIO();
size_t outBufSize = IOptr->getWriteBufferSize();
// the buffer size for resulting workspace; reasonable size is at least 10
// data chunk sizes (nice to verify)
if (outBufSize < 10 * IOptr->getDataChunk()) {
outBufSize = 10 * IOptr->getDataChunk();
IOptr->setWriteBufferSize(outBufSize);
}
}
// Function defining which events (in the input dimensions) to place in the
// output
MDImplicitFunction *function = this->getImplicitFunctionForChunk(NULL, NULL);
std::vector<API::IMDNode *> boxes;
// Leaf-only; no depth limit; with the implicit function passed to it.
ws->getBox()->getBoxes(boxes, 1000, true, function);
// Sort boxes by file position IF file backed. This reduces seeking time,
// hopefully.
bool fileBackedWS = bc->isFileBacked();
if (fileBackedWS)
API::IMDNode::sortObjByID(boxes);
Progress *prog = new Progress(this, 0.0, 1.0, boxes.size());
// The root of the output workspace
MDBoxBase<OMDE, ond> *outRootBox = outWS->getBox();
// if target workspace has events, we should count them as added
uint64_t totalAdded = outWS->getNEvents();
uint64_t numSinceSplit = 0;
// Go through every box for this chunk.
// PARALLEL_FOR_IF( !bc->isFileBacked() )
for (int i = 0; i < int(boxes.size()); i++) {
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxes[i]);
// Perform the binning in this separate method.
if (box) {
// An array to hold the rotated/transformed coordinates
coord_t outCenter[ond];
const std::vector<MDE> &events = box->getConstEvents();
typename std::vector<MDE>::const_iterator it = events.begin();
typename std::vector<MDE>::const_iterator it_end = events.end();
for (; it != it_end; it++) {
// Cache the center of the event (again for speed)
const coord_t *inCenter = it->getCenter();
if (function->isPointContained(inCenter)) {
// Now transform to the output dimensions
m_transformFromOriginal->apply(inCenter, outCenter);
// Create the event
OMDE newEvent(it->getSignal(), it->getErrorSquared(), outCenter);
// Copy extra data, if any
copyEvent(*it, newEvent);
// Add it to the workspace
outRootBox->addEvent(newEvent);
numSinceSplit++;
}
}
box->releaseEvents();
// Ask BC if one needs to split boxes
if (obc->shouldSplitBoxes(totalAdded, numSinceSplit, lastNumBoxes))
// if (numSinceSplit > 20000000 || (i == int(boxes.size()-1)))
{
// This splits up all the boxes according to split thresholds and sizes.
Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
outWS->splitAllIfNeeded(ts);
tp.joinAll();
// Accumulate stats
totalAdded += numSinceSplit;
numSinceSplit = 0;
lastNumBoxes = obc->getTotalNumMDBoxes();
// Progress reporting
if (!fileBackedWS)
prog->report(i);
}
if (fileBackedWS) {
if (!(i % 10))
prog->report(i);
}
} // is box
} // for each box in the vector
prog->report();
outWS->splitAllIfNeeded(NULL);
// Refresh all cache.
outWS->refreshCache();
g_log.notice() << totalAdded << " " << OMDE::getTypeName()
<< "'s added to the output workspace." << std::endl;
if (outWS->isFileBacked()) {
// Update the file-back-end
g_log.notice() << "Running SaveMD" << std::endl;
IAlgorithm_sptr alg = createChildAlgorithm("SaveMD");
alg->setProperty("UpdateFileBackEnd", true);
alg->setProperty("InputWorkspace", outWS);
alg->executeAsChildAlg();
}
// return the size of the input workspace write buffer to its initial value
// bc->setCacheParameters(sizeof(MDE),writeBufSize);
this->setProperty("OutputWorkspace",
boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS));
delete prog;
}
void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Are we using the file back end?
bool fileBackEnd = getProperty("FileBackEnd");
if (fileBackEnd && m_BoxStructureAndMethadata)
throw std::invalid_argument("Combination of BoxStructureOnly or "
"MetaDataOnly were set to TRUE with "
"fileBackEnd "
": this is not possible.");
CPUTimer tim;
auto prog = new Progress(this, 0.0, 1.0, 100);
prog->report("Opening file.");
std::string title;
try {
m_file->getAttr("title", title);
} catch (std::exception &) {
// Leave the title blank if error on loading
}
ws->setTitle(title);
// Load the WorkspaceHistory "process"
if (this->getProperty("LoadHistory")) {
ws->history().loadNexus(m_file.get());
}
this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws));
m_file->closeGroup();
m_file->close();
// Add each of the dimension
for (size_t d = 0; d < nd; d++)
ws->addDimension(m_dims[d]);
// Coordinate system
ws->setCoordinateSystem(m_coordSystem);
// ----------------------------------------- Box Structure
// ------------------------------
prog->report("Reading box structure from HDD.");
MDBoxFlatTree FlatBoxTree;
int nDims = static_cast<int>(nd); // should be safe
FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName());
BoxController_sptr bc = ws->getBoxController();
bc->fromXMLString(FlatBoxTree.getBCXMLdescr());
prog->report("Restoring box structure and connectivity");
std::vector<API::IMDNode *> boxTree;
FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd,
m_BoxStructureAndMethadata);
size_t numBoxes = boxTree.size();
// ---------------------------------------- DEAL WITH BOXES
// ------------------------------------
if (fileBackEnd) { // TODO:: call to the file format factory
auto loader = boost::shared_ptr<API::IBoxControllerIO>(
new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
bc->setFileBacked(loader, m_filename);
// boxes have been already made file-backed when restoring the boxTree;
// How much memory for the cache?
{
// TODO: Clean up, only a write buffer now
double mb = getProperty("Memory");
// Defaults have changed, default disk buffer size should be 10 data
// chunks TODO: find optimal, 100 may be better.
if (mb <= 0)
mb = double(10 * loader->getDataChunk() * sizeof(MDE)) /
double(1024 * 1024);
// Express the cache memory in units of number of events.
uint64_t cacheMemory =
static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1;
// Set these values in the diskMRU
bc->getFileIO()->setWriteBufferSize(cacheMemory);
g_log.information() << "Setting a DiskBuffer cache size of " << mb
<< " MB, or " << cacheMemory << " events.\n";
}
} // Not file back end
else if (!m_BoxStructureAndMethadata) {
// ---------------------------------------- READ IN THE BOXES
// ------------------------------------
// TODO:: call to the file format factory
auto loader =
file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
loader->openFile(m_filename, "r");
const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex();
prog->setNumSteps(numBoxes);
for (size_t i = 0; i < numBoxes; i++) {
prog->report();
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]);
if (!box)
continue;
if (BoxEventIndex[2 * i + 1] >
0) // Load in memory NOT using the file as the back-end,
{
boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]);
boxTree[i]->loadAndAddFrom(
loader.get(), BoxEventIndex[2 * i],
static_cast<size_t>(BoxEventIndex[2 * i + 1]));
}
}
loader->closeFile();
} else // box structure and metadata only
{
}
g_log.debug() << tim
<< " to create all the boxes and fill them with events.\n";
// Box of ID 0 is the head box.
ws->setBox(boxTree[0]);
// Make sure the max ID is ok for later ID generation
bc->setMaxId(numBoxes);
// end-of bMetaDataOnly
// Refresh cache
// TODO:if(!fileBackEnd)ws->refreshCache();
ws->refreshCache();
g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints()
<< " points after refresh.\n";
g_log.debug() << tim << " to finish up.\n";
delete prog;
}
void PlusMD::doPlus(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
if (!ws1 || !ws2)
throw std::runtime_error("Incompatible workspace types passed to PlusMD.");
MDBoxBase<MDE,nd> * box1 = ws1->getBox();
MDBoxBase<MDE,nd> * box2 = ws2->getBox();
Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());
// How many events you started with
size_t initial_numEvents = ws1->getNPoints();
// Make a leaf-only iterator through all boxes with events in the RHS workspace
MDBoxIterator<MDE,nd> it2(box2, 1000, true);
do
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
if (box)
{
// Copy the events from WS2 and add them into WS1
const std::vector<MDE> & events = box->getConstEvents();
// Add events, with bounds checking
box1->addEvents(events);
box->releaseEvents();
}
prog.report("Adding Events");
} while (it2.next());
this->progress(0.41, "Splitting Boxes");
Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
ThreadScheduler * ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts, 0, prog2);
ws1->splitAllIfNeeded(ts);
prog2->resetNumSteps( ts->size(), 0.4, 0.6);
tp.joinAll();
// // Now we need to save all the data that was not saved before.
// if (ws1->isFileBacked())
// {
// // Flush anything else in the to-write buffer
// BoxController_sptr bc = ws1->getBoxController();
//
// prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0);
// MDBoxIterator<MDE,nd> it1(box1, 1000, true);
// while (true)
// {
// MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox());
// if (box)
// {
// // Something was maybe added to this box
// if (box->getEventVectorSize() > 0)
// {
// // By getting the events, this will merge the newly added and the cached events.
// box->getEvents();
// // The MRU to-write cache will optimize writes by reducing seek times
// box->releaseEvents();
// }
// }
// prog.report("Saving");
// if (!it1.next()) break;
// }
// //bc->getDiskBuffer().flushCache();
// // Flush the data writes to disk.
// box1->flushData();
// }
this->progress(0.95, "Refreshing cache");
ws1->refreshCache();
// Set a marker that the file-back-end needs updating if the # of events changed.
if (ws1->getNPoints() != initial_numEvents)
ws1->setFileNeedsUpdating(true);
}
void FakeMDEventData::addFakePeak(typename MDEventWorkspace<MDE, nd>::sptr ws) {
std::vector<double> params = getProperty("PeakParams");
bool RandomizeSignal = getProperty("RandomizeSignal");
if (params.empty())
return;
if (params.size() != nd + 2)
throw std::invalid_argument("PeakParams needs to have ndims+2 arguments.");
if (params[0] <= 0)
throw std::invalid_argument("PeakParams: number_of_events needs to be > 0");
size_t num = size_t(params[0]);
Progress prog(this, 0.0, 1.0, 100);
size_t progIncrement = num / 100;
if (progIncrement == 0)
progIncrement = 1;
// Width of the peak
double desiredRadius = params.back();
boost::mt19937 rng;
boost::uniform_real<coord_t> u2(0, 1.0); // Random from 0 to 1.0
boost::variate_generator<boost::mt19937 &, boost::uniform_real<coord_t>>
genUnit(rng, u2);
int randomSeed = getProperty("RandomSeed");
rng.seed((unsigned int)(randomSeed));
// Inserter to help choose the correct event type
auto eventHelper =
MDEvents::MDEventInserter<typename MDEventWorkspace<MDE, nd>::sptr>(ws);
for (size_t i = 0; i < num; ++i) {
// Algorithm to generate points along a random n-sphere (sphere with not
// necessarily 3 dimensions)
// from http://en.wikipedia.org/wiki/N-sphere as of May 6, 2011.
// First, points in a hyper-cube of size 1.0, centered at 0.
coord_t centers[nd];
coord_t radiusSquared = 0;
for (size_t d = 0; d < nd; d++) {
centers[d] = genUnit() - 0.5f; // Distribute around +- the center
radiusSquared += centers[d] * centers[d];
}
// Make a unit vector pointing in this direction
coord_t radius = static_cast<coord_t>(sqrt(radiusSquared));
for (size_t d = 0; d < nd; d++)
centers[d] /= radius;
// Now place the point along this radius, scaled with ^1/n for uniformity.
coord_t radPos = genUnit();
radPos = static_cast<coord_t>(
pow(radPos, static_cast<coord_t>(1.0 / static_cast<coord_t>(nd))));
for (size_t d = 0; d < nd; d++) {
// Multiply by the scaling and the desired peak radius
centers[d] *= (radPos * static_cast<coord_t>(desiredRadius));
// Also offset by the center of the peak, as taken in Params
centers[d] += static_cast<coord_t>(params[d + 1]);
}
// Default or randomized error/signal
float signal = 1.0;
float errorSquared = 1.0;
if (RandomizeSignal) {
signal = float(0.5 + genUnit());
errorSquared = float(0.5 + genUnit());
}
// Create and add the event.
eventHelper.insertMDEvent(signal, errorSquared, 1, pickDetectorID(),
centers); // 1 = run number
// Progress report
if ((i % progIncrement) == 0)
prog.report();
}
ws->splitBox();
Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts);
ws->splitAllIfNeeded(ts);
tp.joinAll();
ws->refreshCache();
}