void MinusMD::doMinus(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
if (!ws1 || !ws2)
throw std::runtime_error("Incompatible workspace types passed to MinusMD.");
MDBoxBase<MDE,nd> * box1 = ws1->getBox();
MDBoxBase<MDE,nd> * box2 = ws2->getBox();
Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());
// How many events you started with
size_t initial_numEvents = ws1->getNPoints();
// Make a leaf-only iterator through all boxes with events in the RHS workspace
MDBoxIterator<MDE,nd> it2(box2, 1000, true);
do
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
if (box)
{
// Copy the events from WS2 and add them into WS1
const std::vector<MDE> & events = box->getConstEvents();
// Perform a copy while flipping the signal
std::vector<MDE> eventsCopy;
eventsCopy.reserve(events.size());
for (auto it = events.begin(); it != events.end(); it++)
{
MDE eventCopy(*it);
eventCopy.setSignal( -eventCopy.getSignal());
eventsCopy.push_back(eventCopy);
}
// Add events, with bounds checking
box1->addEvents(eventsCopy);
box->releaseEvents();
}
prog.report("Adding Events");
} while (it2.next());
this->progress(0.41, "Splitting Boxes");
Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
ThreadScheduler * ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts, 0, prog2);
ws1->splitAllIfNeeded(ts);
prog2->resetNumSteps( ts->size(), 0.4, 0.6);
tp.joinAll();
this->progress(0.95, "Refreshing cache");
ws1->refreshCache();
// Set a marker that the file-back-end needs updating if the # of events changed.
if (ws1->getNPoints() != initial_numEvents)
ws1->setFileNeedsUpdating(true);
}
void PlusMD::doPlus(typename MDEventWorkspace<MDE, nd>::sptr ws)
{
typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
if (!ws1 || !ws2)
throw std::runtime_error("Incompatible workspace types passed to PlusMD.");
MDBoxBase<MDE,nd> * box1 = ws1->getBox();
MDBoxBase<MDE,nd> * box2 = ws2->getBox();
Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());
// How many events you started with
size_t initial_numEvents = ws1->getNPoints();
// Make a leaf-only iterator through all boxes with events in the RHS workspace
MDBoxIterator<MDE,nd> it2(box2, 1000, true);
do
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
if (box)
{
// Copy the events from WS2 and add them into WS1
const std::vector<MDE> & events = box->getConstEvents();
// Add events, with bounds checking
box1->addEvents(events);
box->releaseEvents();
}
prog.report("Adding Events");
} while (it2.next());
this->progress(0.41, "Splitting Boxes");
Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
ThreadScheduler * ts = new ThreadSchedulerFIFO();
ThreadPool tp(ts, 0, prog2);
ws1->splitAllIfNeeded(ts);
prog2->resetNumSteps( ts->size(), 0.4, 0.6);
tp.joinAll();
// // Now we need to save all the data that was not saved before.
// if (ws1->isFileBacked())
// {
// // Flush anything else in the to-write buffer
// BoxController_sptr bc = ws1->getBoxController();
//
// prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0);
// MDBoxIterator<MDE,nd> it1(box1, 1000, true);
// while (true)
// {
// MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox());
// if (box)
// {
// // Something was maybe added to this box
// if (box->getEventVectorSize() > 0)
// {
// // By getting the events, this will merge the newly added and the cached events.
// box->getEvents();
// // The MRU to-write cache will optimize writes by reducing seek times
// box->releaseEvents();
// }
// }
// prog.report("Saving");
// if (!it1.next()) break;
// }
// //bc->getDiskBuffer().flushCache();
// // Flush the data writes to disk.
// box1->flushData();
// }
this->progress(0.95, "Refreshing cache");
ws1->refreshCache();
// Set a marker that the file-back-end needs updating if the # of events changed.
if (ws1->getNPoints() != initial_numEvents)
ws1->setFileNeedsUpdating(true);
}
void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) {
// Are we using the file back end?
bool fileBackEnd = getProperty("FileBackEnd");
if (fileBackEnd && m_BoxStructureAndMethadata)
throw std::invalid_argument("Combination of BoxStructureOnly or "
"MetaDataOnly were set to TRUE with "
"fileBackEnd "
": this is not possible.");
CPUTimer tim;
auto prog = new Progress(this, 0.0, 1.0, 100);
prog->report("Opening file.");
std::string title;
try {
m_file->getAttr("title", title);
} catch (std::exception &) {
// Leave the title blank if error on loading
}
ws->setTitle(title);
// Load the WorkspaceHistory "process"
if (this->getProperty("LoadHistory")) {
ws->history().loadNexus(m_file.get());
}
this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws));
m_file->closeGroup();
m_file->close();
// Add each of the dimension
for (size_t d = 0; d < nd; d++)
ws->addDimension(m_dims[d]);
// Coordinate system
ws->setCoordinateSystem(m_coordSystem);
// ----------------------------------------- Box Structure
// ------------------------------
prog->report("Reading box structure from HDD.");
MDBoxFlatTree FlatBoxTree;
int nDims = static_cast<int>(nd); // should be safe
FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName());
BoxController_sptr bc = ws->getBoxController();
bc->fromXMLString(FlatBoxTree.getBCXMLdescr());
prog->report("Restoring box structure and connectivity");
std::vector<API::IMDNode *> boxTree;
FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd,
m_BoxStructureAndMethadata);
size_t numBoxes = boxTree.size();
// ---------------------------------------- DEAL WITH BOXES
// ------------------------------------
if (fileBackEnd) { // TODO:: call to the file format factory
auto loader = boost::shared_ptr<API::IBoxControllerIO>(
new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
bc->setFileBacked(loader, m_filename);
// boxes have been already made file-backed when restoring the boxTree;
// How much memory for the cache?
{
// TODO: Clean up, only a write buffer now
double mb = getProperty("Memory");
// Defaults have changed, default disk buffer size should be 10 data
// chunks TODO: find optimal, 100 may be better.
if (mb <= 0)
mb = double(10 * loader->getDataChunk() * sizeof(MDE)) /
double(1024 * 1024);
// Express the cache memory in units of number of events.
uint64_t cacheMemory =
static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1;
// Set these values in the diskMRU
bc->getFileIO()->setWriteBufferSize(cacheMemory);
g_log.information() << "Setting a DiskBuffer cache size of " << mb
<< " MB, or " << cacheMemory << " events.\n";
}
} // Not file back end
else if (!m_BoxStructureAndMethadata) {
// ---------------------------------------- READ IN THE BOXES
// ------------------------------------
// TODO:: call to the file format factory
auto loader =
file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get()));
loader->setDataType(sizeof(coord_t), MDE::getTypeName());
loader->openFile(m_filename, "r");
const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex();
prog->setNumSteps(numBoxes);
for (size_t i = 0; i < numBoxes; i++) {
prog->report();
MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]);
if (!box)
continue;
if (BoxEventIndex[2 * i + 1] >
0) // Load in memory NOT using the file as the back-end,
{
boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]);
boxTree[i]->loadAndAddFrom(
loader.get(), BoxEventIndex[2 * i],
static_cast<size_t>(BoxEventIndex[2 * i + 1]));
}
}
loader->closeFile();
} else // box structure and metadata only
{
}
g_log.debug() << tim
<< " to create all the boxes and fill them with events.\n";
// Box of ID 0 is the head box.
ws->setBox(boxTree[0]);
// Make sure the max ID is ok for later ID generation
bc->setMaxId(numBoxes);
// end-of bMetaDataOnly
// Refresh cache
// TODO:if(!fileBackEnd)ws->refreshCache();
ws->refreshCache();
g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints()
<< " points after refresh.\n";
g_log.debug() << tim << " to finish up.\n";
delete prog;
}
void vtkSplatterPlotFactory::doCreate(typename MDEventWorkspace<MDE, nd>::sptr ws) const
{
bool VERBOSE = true;
CPUTimer tim;
// Acquire a scoped read-only lock to the workspace (prevent segfault
// from algos modifying ws)
ReadLock lock(*ws);
// Find out how many events to plot, and the percentage of the largest
// boxes to use.
size_t totalPoints = ws->getNPoints();
size_t numPoints = m_numPoints;
if (numPoints > totalPoints)
{
numPoints = totalPoints;
}
double percent_to_use = m_percentToUse;
// Fail safe limits on fraction of boxes to use
if (percent_to_use <= 0)
{
percent_to_use = 5;
}
if (percent_to_use > 100)
{
percent_to_use = 100;
}
// First we get all the boxes, up to the given depth; with or wo the
// slice function
std::vector<API::IMDNode *> boxes;
if (this->slice)
{
ws->getBox()->getBoxes(boxes, 1000, true, this->sliceImplicitFunction);
}
else
{
ws->getBox()->getBoxes(boxes, 1000, true);
}
if (VERBOSE)
{
std::cout << tim << " to retrieve the "<< boxes.size() << " boxes down."<< std::endl;
}
std::string new_name = ws->getName();
if (new_name != m_wsName || m_buildSortedList)
{
m_wsName = new_name;
m_buildSortedList = false;
m_sortedBoxes.clear();
// get list of boxes with signal > 0 and sort
// the list in order of decreasing signal
for (size_t i = 0; i < boxes.size(); i++)
{
MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]);
if (box)
{
size_t newPoints = box->getNPoints();
if (newPoints > 0)
{
m_sortedBoxes.push_back(box);
}
}
}
if (VERBOSE)
{
std::cout << "START SORTING" << std::endl;
}
std::sort(m_sortedBoxes.begin(), m_sortedBoxes.end(),
CompareNormalizedSignal);
if (VERBOSE)
{
std::cout << "DONE SORTING" << std::endl;
}
}
size_t num_boxes_to_use = static_cast<size_t>(percent_to_use * static_cast<double>(m_sortedBoxes.size()) / 100.0);
if (num_boxes_to_use >= m_sortedBoxes.size())
{
num_boxes_to_use = m_sortedBoxes.size()-1;
}
// restrict the number of points to the
// number of points in boxes being used
size_t total_points_available = 0;
for (size_t i = 0; i < num_boxes_to_use; i++)
{
size_t newPoints = m_sortedBoxes[i]->getNPoints();
total_points_available += newPoints;
}
if (numPoints > total_points_available)
{
numPoints = total_points_available;
}
size_t points_per_box = 0;
// calculate the average number of points to use per box
if (num_boxes_to_use > 0)
{
points_per_box = numPoints / num_boxes_to_use;
}
if (points_per_box < 1)
{
points_per_box = 1;
}
if (VERBOSE)
{
std::cout << "numPoints = " << numPoints << std::endl;
std::cout << "num boxes in all = " << boxes.size() << std::endl;
std::cout << "num boxes above zero = " << m_sortedBoxes.size() << std::endl;
std::cout << "num boxes to use = " << num_boxes_to_use << std::endl;
std::cout << "total_points_available = " << total_points_available << std::endl;
std::cout << "points needed per box = " << points_per_box << std::endl;
}
// First save the events and signals that we actually use.
// For each box, get up to the average number of points
// we want from each box, limited by the number of points
// in the box. NOTE: since boxes have different numbers
// of events, we will not get all the events requested.
// Also, if we are using a smaller number of points, we
// won't get points from some of the boxes with lower signal.
std::vector<float> saved_signals;
std::vector<const coord_t*> saved_centers;
std::vector<size_t> saved_n_points_in_cell;
saved_signals.reserve(numPoints);
saved_centers.reserve(numPoints);
saved_n_points_in_cell.reserve(numPoints);
size_t pointIndex = 0;
size_t box_index = 0;
bool done = false;
while (box_index < num_boxes_to_use && !done)
{
MDBox<MDE,nd> *box = dynamic_cast<MDBox<MDE,nd> *>(m_sortedBoxes[box_index]);
box_index++;
if (NULL == box)
{
continue;
}
float signal_normalized = static_cast<float>(box->getSignalNormalized());
size_t newPoints = box->getNPoints();
size_t num_from_this_box = points_per_box;
if (num_from_this_box > newPoints)
{
num_from_this_box = newPoints;
}
const std::vector<MDE> & events = box->getConstEvents();
size_t startPointIndex = pointIndex;
size_t event_index = 0;
while (event_index < num_from_this_box && !done)
{
const MDE & ev = events[event_index];
event_index++;
const coord_t * center = ev.getCenter();
// Save location
saved_centers.push_back(center);
pointIndex++;
if (pointIndex >= numPoints)
{
done = true;
}
}
box->releaseEvents();
// Save signal
saved_signals.push_back(signal_normalized);
// Save cell size
saved_n_points_in_cell.push_back(pointIndex-startPointIndex);
}
numPoints = saved_centers.size();
size_t numCells = saved_signals.size();
if (VERBOSE)
{
std::cout << "Recorded data for all points" << std::endl;
std::cout << "numPoints = " << numPoints << std::endl;
std::cout << "numCells = " << numCells << std::endl;
}
// Create the point list, one position for each point actually used
vtkPoints *points = vtkPoints::New();
points->Allocate(numPoints);
points->SetNumberOfPoints(numPoints);
// The list of IDs of points used, one ID per point, since points
// are not reused to form polygon facets, etc.
vtkIdType *ids = new vtkIdType[numPoints];
// Only one scalar for each cell, NOT one per point
vtkFloatArray *signal = vtkFloatArray::New();
signal->Allocate(numCells);
signal->SetName(m_scalarName.c_str());
// Create the data set. Need space for each cell, not for each point
vtkUnstructuredGrid *visualDataSet = vtkUnstructuredGrid::New();
this->dataSet = visualDataSet;
visualDataSet->Allocate(numCells);
// Now copy the saved point, cell and signal info into vtk data structures
pointIndex = 0;
for (size_t cell_i = 0; cell_i < numCells; cell_i++)
{
size_t startPointIndex = pointIndex;
for (size_t point_i = 0; point_i < saved_n_points_in_cell[cell_i]; point_i++)
{
points->SetPoint(pointIndex, saved_centers[pointIndex]);
ids[pointIndex] = pointIndex;
pointIndex++;
}
signal->InsertNextTuple1(saved_signals[cell_i]);
visualDataSet->InsertNextCell(VTK_POLY_VERTEX, saved_n_points_in_cell[cell_i], ids+startPointIndex);
}
if (VERBOSE)
{
std::cout << tim << " to create " << pointIndex << " points." << std::endl;
}
// Shrink to fit
//points->Squeeze();
signal->Squeeze();
visualDataSet->Squeeze();
// Add points and scalars
visualDataSet->SetPoints(points);
visualDataSet->GetCellData()->SetScalars(signal);
delete [] ids;
}
