void MultiplyMD::execEventScalar(typename MDEventWorkspace<MDE, nd>::sptr ws) { // Get the scalar multiplying float scalar = float(m_rhs_scalar->dataY(0)[0]); float scalarError = float(m_rhs_scalar->dataE(0)[0]); float scalarRelativeErrorSquared = (scalarError * scalarError) / (scalar * scalar); // Get all the MDBoxes contained MDBoxBase<MDE,nd> * parentBox = ws->getBox(); std::vector<API::IMDNode *> boxes; parentBox->getBoxes(boxes, 1000, true); bool fileBackedTarget(false); Kernel::DiskBuffer *dbuff(NULL); if(ws->isFileBacked()) { fileBackedTarget = true; dbuff = ws->getBoxController()->getFileIO(); } for (size_t i=0; i<boxes.size(); i++) { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]); if (box) { typename std::vector<MDE> & events = box->getEvents(); size_t ic(events.size()); typename std::vector<MDE>::iterator it = events.begin(); typename std::vector<MDE>::iterator it_end = events.end(); for (; it != it_end; it++) { // Multiply weight by a scalar, propagating error float oldSignal = it->getSignal(); float signal = oldSignal * scalar; float errorSquared = signal * signal * (it->getErrorSquared() / (oldSignal * oldSignal) + scalarRelativeErrorSquared); it->setSignal(signal); it->setErrorSquared(errorSquared); } box->releaseEvents(); if(fileBackedTarget && ic>0) { Kernel::ISaveable *const pSaver(box->getISaveable()); dbuff->toWrite(pSaver); } } } // Recalculate the totals ws->refreshCache(); // Mark file-backed workspace as dirty ws->setFileNeedsUpdating(true); }
void MinusMD::doMinus(typename MDEventWorkspace<MDE, nd>::sptr ws) { typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws; typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event); if (!ws1 || !ws2) throw std::runtime_error("Incompatible workspace types passed to MinusMD."); MDBoxBase<MDE,nd> * box1 = ws1->getBox(); MDBoxBase<MDE,nd> * box2 = ws2->getBox(); Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes()); // How many events you started with size_t initial_numEvents = ws1->getNPoints(); // Make a leaf-only iterator through all boxes with events in the RHS workspace MDBoxIterator<MDE,nd> it2(box2, 1000, true); do { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox()); if (box) { // Copy the events from WS2 and add them into WS1 const std::vector<MDE> & events = box->getConstEvents(); // Perform a copy while flipping the signal std::vector<MDE> eventsCopy; eventsCopy.reserve(events.size()); for (auto it = events.begin(); it != events.end(); it++) { MDE eventCopy(*it); eventCopy.setSignal( -eventCopy.getSignal()); eventsCopy.push_back(eventCopy); } // Add events, with bounds checking box1->addEvents(eventsCopy); box->releaseEvents(); } prog.report("Adding Events"); } while (it2.next()); this->progress(0.41, "Splitting Boxes"); Progress * prog2 = new Progress(this, 0.4, 0.9, 100); ThreadScheduler * ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts, 0, prog2); ws1->splitAllIfNeeded(ts); prog2->resetNumSteps( ts->size(), 0.4, 0.6); tp.joinAll(); this->progress(0.95, "Refreshing cache"); ws1->refreshCache(); // Set a marker that the file-back-end needs updating if the # of events changed. if (ws1->getNPoints() != initial_numEvents) ws1->setFileNeedsUpdating(true); }
void CreateMDWorkspace::finish(typename MDEventWorkspace<MDE, nd>::sptr ws) { // ------------ Set up the box controller ---------------------------------- BoxController_sptr bc = ws->getBoxController(); this->setBoxController(bc); // Split to level 1 ws->splitBox(); // Do we split more due to MinRecursionDepth? int minDepth = this->getProperty("MinRecursionDepth"); if (minDepth<0) throw std::invalid_argument("MinRecursionDepth must be >= 0."); ws->setMinRecursionDepth(size_t(minDepth)); }
void FakeMDEventData::addFakeUniformData( typename MDEventWorkspace<MDE, nd>::sptr ws) { std::vector<double> params = getProperty("UniformParams"); if (params.empty()) return; bool randomEvents = true; if (params[0] < 0) { randomEvents = false; params[0] = -params[0]; } if (params.size() == 1) { if (randomEvents) { for (size_t d = 0; d < nd; ++d) { params.push_back(ws->getDimension(d)->getMinimum()); params.push_back(ws->getDimension(d)->getMaximum()); } } else // regular events { size_t nPoints = size_t(params[0]); double Vol = 1; for (size_t d = 0; d < nd; ++d) Vol *= (ws->getDimension(d)->getMaximum() - ws->getDimension(d)->getMinimum()); if (Vol == 0 || Vol > std::numeric_limits<float>::max()) throw std::invalid_argument( " Domain ranges are not defined properly for workspace: " + ws->getName()); double dV = Vol / double(nPoints); double delta0 = std::pow(dV, 1. / double(nd)); for (size_t d = 0; d < nd; ++d) { double min = ws->getDimension(d)->getMinimum(); params.push_back(min * (1 + FLT_EPSILON) - min + FLT_EPSILON); double extent = ws->getDimension(d)->getMaximum() - min; size_t nStrides = size_t(extent / delta0); if (nStrides < 1) nStrides = 1; params.push_back(extent / static_cast<double>(nStrides)); } } } if ((params.size() != 1 + nd * 2)) throw std::invalid_argument( "UniformParams: needs to have ndims*2+1 arguments "); if (randomEvents) addFakeRandomData<MDE, nd>(params, ws); else addFakeRegularData<MDE, nd>(params, ws); ws->splitBox(); Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts); ws->splitAllIfNeeded(ts); tp.joinAll(); ws->refreshCache(); }
void CloneMDWorkspace::doClone(const typename MDEventWorkspace<MDE, nd>::sptr ws) { std::string outWSName = getPropertyValue("OutputWorkspace"); Progress prog(this, 0.0, 10.0, 100); BoxController_sptr bc = ws->getBoxController(); if (!bc) throw std::runtime_error("Error with InputWorkspace: no BoxController!"); if (bc->isFileBacked()) { // Generate a new filename to copy to prog.report("Copying File"); std::string originalFile = bc->getFilename(); std::string outFilename = getPropertyValue("Filename"); if (outFilename.empty()) { // Auto-generated name Poco::Path path = Poco::Path(originalFile).absolute(); std::string newName = path.getBaseName() + "_clone." + path.getExtension(); path.setFileName(newName); outFilename = path.toString(); } // Perform the copying g_log.notice() << "Cloned workspace file being copied to: " << outFilename << std::endl; Poco::File(originalFile).copyTo(outFilename); g_log.information() << "File copied successfully." << std::endl; // Now load it back IAlgorithm_sptr alg = createSubAlgorithm("LoadMD", 0.5, 1.0, false); alg->setPropertyValue("Filename", outFilename); alg->setPropertyValue("FileBackEnd", "1"); alg->setPropertyValue("Memory", "0"); //TODO: How much memory? alg->setPropertyValue("OutputWorkspace", outWSName); alg->executeAsSubAlg(); // Set the output workspace to this IMDEventWorkspace_sptr outWS = alg->getProperty("OutputWorkspace"); this->setProperty("OutputWorkspace", outWS); } else { // Perform the clone in memory. boost::shared_ptr<MDEventWorkspace<MDE,nd> > outWS(new MDEventWorkspace<MDE,nd>(*ws)); this->setProperty("OutputWorkspace", boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS) ); } }
void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) { // Are we using the file back end? bool fileBackEnd = getProperty("FileBackEnd"); if (fileBackEnd && m_BoxStructureAndMethadata) throw std::invalid_argument("Combination of BoxStructureOnly or " "MetaDataOnly were set to TRUE with " "fileBackEnd " ": this is not possible."); CPUTimer tim; auto prog = new Progress(this, 0.0, 1.0, 100); prog->report("Opening file."); std::string title; try { m_file->getAttr("title", title); } catch (std::exception &) { // Leave the title blank if error on loading } ws->setTitle(title); // Load the WorkspaceHistory "process" if (this->getProperty("LoadHistory")) { ws->history().loadNexus(m_file.get()); } this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws)); m_file->closeGroup(); m_file->close(); // Add each of the dimension for (size_t d = 0; d < nd; d++) ws->addDimension(m_dims[d]); // Coordinate system ws->setCoordinateSystem(m_coordSystem); // ----------------------------------------- Box Structure // ------------------------------ prog->report("Reading box structure from HDD."); MDBoxFlatTree FlatBoxTree; int nDims = static_cast<int>(nd); // should be safe FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName()); BoxController_sptr bc = ws->getBoxController(); bc->fromXMLString(FlatBoxTree.getBCXMLdescr()); prog->report("Restoring box structure and connectivity"); std::vector<API::IMDNode *> boxTree; FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd, m_BoxStructureAndMethadata); size_t numBoxes = boxTree.size(); // ---------------------------------------- DEAL WITH BOXES // ------------------------------------ if (fileBackEnd) { // TODO:: call to the file format factory auto loader = boost::shared_ptr<API::IBoxControllerIO>( new DataObjects::BoxControllerNeXusIO(bc.get())); loader->setDataType(sizeof(coord_t), MDE::getTypeName()); bc->setFileBacked(loader, m_filename); // boxes have been already made file-backed when restoring the boxTree; // How much memory for the cache? { // TODO: Clean up, only a write buffer now double mb = getProperty("Memory"); // Defaults have changed, default disk buffer size should be 10 data // chunks TODO: find optimal, 100 may be better. if (mb <= 0) mb = double(10 * loader->getDataChunk() * sizeof(MDE)) / double(1024 * 1024); // Express the cache memory in units of number of events. uint64_t cacheMemory = static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1; // Set these values in the diskMRU bc->getFileIO()->setWriteBufferSize(cacheMemory); g_log.information() << "Setting a DiskBuffer cache size of " << mb << " MB, or " << cacheMemory << " events.\n"; } } // Not file back end else if (!m_BoxStructureAndMethadata) { // ---------------------------------------- READ IN THE BOXES // ------------------------------------ // TODO:: call to the file format factory auto loader = file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get())); loader->setDataType(sizeof(coord_t), MDE::getTypeName()); loader->openFile(m_filename, "r"); const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex(); prog->setNumSteps(numBoxes); for (size_t i = 0; i < numBoxes; i++) { prog->report(); MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]); if (!box) continue; if (BoxEventIndex[2 * i + 1] > 0) // Load in memory NOT using the file as the back-end, { boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]); boxTree[i]->loadAndAddFrom( loader.get(), BoxEventIndex[2 * i], static_cast<size_t>(BoxEventIndex[2 * i + 1])); } } loader->closeFile(); } else // box structure and metadata only { } g_log.debug() << tim << " to create all the boxes and fill them with events.\n"; // Box of ID 0 is the head box. ws->setBox(boxTree[0]); // Make sure the max ID is ok for later ID generation bc->setMaxId(numBoxes); // end-of bMetaDataOnly // Refresh cache // TODO:if(!fileBackEnd)ws->refreshCache(); ws->refreshCache(); g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints() << " points after refresh.\n"; g_log.debug() << tim << " to finish up.\n"; delete prog; }
void CompareMDWorkspaces::compareMDWorkspaces( typename MDEventWorkspace<MDE, nd>::sptr ws) { typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws; typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd>>(inWS2); if (!ws1 || !ws2) throw std::runtime_error("Incompatible workspace types passed to PlusMD."); std::vector<API::IMDNode *> boxes1; std::vector<API::IMDNode *> boxes2; ws1->getBox()->getBoxes(boxes1, 1000, false); ws2->getBox()->getBoxes(boxes2, 1000, false); this->compare(boxes1.size(), boxes2.size(), "Workspaces do not have the same number of boxes"); for (size_t j = 0; j < boxes1.size(); j++) { API::IMDNode *box1 = boxes1[j]; API::IMDNode *box2 = boxes2[j]; if (m_CompareBoxID) this->compare(box1->getID(), box2->getID(), "Boxes have different ID"); else { if (box1->getID() != box2->getID()) g_log.debug() << " Boxes N: " << j << " have box ID: " << box1->getID() << " and " << box2->getID() << " correspondingly\n"; } this->compare(size_t(box1->getDepth()), size_t(box2->getDepth()), "Boxes are at a different depth"); this->compare(box1->getNumChildren(), box2->getNumChildren(), "Boxes do not have the same number of children"); for (size_t i = 0; i < box1->getNumChildren(); i++) { if (m_CompareBoxID) this->compare(box1->getChild(i)->getID(), box2->getChild(i)->getID(), "Child of boxes do not match IDs"); else { if (box1->getID() != box2->getID()) g_log.debug() << " Boxes N: " << j << " children N: " << i << " have box ID: " << box1->getChild(i)->getID() << " and " << box2->getChild(i)->getID() << " correspondingly\n"; } } for (size_t d = 0; d < nd; d++) { this->compareTol(box1->getExtents(d).getMin(), box2->getExtents(d).getMin(), "Extents of box do not match"); this->compareTol(box1->getExtents(d).getMax(), box2->getExtents(d).getMax(), "Extents of box do not match"); } this->compareTol(box1->getInverseVolume(), box2->getInverseVolume(), "Box inverse volume does not match"); this->compareTol(box1->getSignal(), box2->getSignal(), "Box signal does not match"); this->compareTol(box1->getErrorSquared(), box2->getErrorSquared(), "Box error squared does not match"); if (m_CheckEvents) this->compare(box1->getNPoints(), box2->getNPoints(), "Number of points in box does not match"); // Are both MDGridBoxes ? MDGridBox<MDE, nd> *gridbox1 = dynamic_cast<MDGridBox<MDE, nd> *>(box1); MDGridBox<MDE, nd> *gridbox2 = dynamic_cast<MDGridBox<MDE, nd> *>(box2); if (gridbox1 && gridbox2) { for (size_t d = 0; d < nd; d++) this->compareTol(gridbox1->getBoxSize(d), gridbox2->getBoxSize(d), "Box sizes do not match"); } // Are both MDBoxes (with events) MDBox<MDE, nd> *mdbox1 = dynamic_cast<MDBox<MDE, nd> *>(box1); MDBox<MDE, nd> *mdbox2 = dynamic_cast<MDBox<MDE, nd> *>(box2); if (mdbox1 && mdbox2) { if (m_CheckEvents) { const std::vector<MDE> &events1 = mdbox1->getConstEvents(); const std::vector<MDE> &events2 = mdbox2->getConstEvents(); try { this->compare(events1.size(), events2.size(), "Box event vectors are not the same length"); if (events1.size() == events2.size() && events1.size() > 2) { // Check first and last event for (size_t i = 0; i < events1.size(); i++) { for (size_t d = 0; d < nd; d++) { this->compareTol(events1[i].getCenter(d), events2[i].getCenter(d), "Event center does not match"); } this->compareTol(events1[i].getSignal(), events2[i].getSignal(), "Event signal does not match"); this->compareTol(events1[i].getErrorSquared(), events2[i].getErrorSquared(), "Event error does not match"); } } } catch (CompareFailsException &) { // Boxes must release events if the check fails mdbox1->releaseEvents(); mdbox2->releaseEvents(); throw; } mdbox1->releaseEvents(); mdbox2->releaseEvents(); } // Don't compare if BoxStructureOnly } // is mdbox1 } }
void FindPeaksMD::findPeaks(typename MDEventWorkspace<MDE, nd>::sptr ws) { if (nd < 3) throw std::invalid_argument("Workspace must have at least 3 dimensions."); progress(0.01, "Refreshing Centroids"); // TODO: This might be slow, progress report? // Make sure all centroids are fresh ws->getBox()->refreshCentroid(); typedef IMDBox<MDE,nd>* boxPtr; if (ws->getNumExperimentInfo() == 0) throw std::runtime_error("No instrument was found in the MDEventWorkspace. Cannot find peaks."); // TODO: Do we need to pick a different instrument info? ExperimentInfo_sptr ei = ws->getExperimentInfo(0); // Instrument associated with workspace Geometry::Instrument_const_sptr inst = ei->getInstrument(); // Find the run number int runNumber = ei->getRunNumber(); // Check that the workspace dimensions are in Q-sample-frame or Q-lab-frame. eDimensionType dimType; std::string dim0 = ws->getDimension(0)->getName(); if (dim0 == "H") { dimType = HKL; throw std::runtime_error("Cannot find peaks in a workspace that is already in HKL space."); } else if (dim0 == "Q_lab_x") { dimType = QLAB; } else if (dim0 == "Q_sample_x") dimType = QSAMPLE; else throw std::runtime_error("Unexpected dimensions: need either Q_lab_x or Q_sample_x."); // Find the goniometer rotation matrix Mantid::Kernel::Matrix<double> goniometer(3,3, true); // Default IDENTITY matrix try { goniometer = ei->mutableRun().getGoniometerMatrix(); } catch (std::exception & e) { g_log.warning() << "Error finding goniometer matrix. It will not be set in the peaks found." << std::endl; g_log.warning() << e.what() << std::endl; } /// Arbitrary scaling factor for density to make more manageable numbers, especially for older file formats. signal_t densityScalingFactor = 1e-6; // Calculate a threshold below which a box is too diffuse to be considered a peak. signal_t thresholdDensity = 0.0; thresholdDensity = ws->getBox()->getSignalNormalized() * DensityThresholdFactor * densityScalingFactor; g_log.notice() << "Threshold signal density: " << thresholdDensity << std::endl; // We will fill this vector with pointers to all the boxes (up to a given depth) typename std::vector<boxPtr> boxes; // Get all the MDboxes progress(0.10, "Getting Boxes"); ws->getBox()->getBoxes(boxes, 1000, true); // TODO: Here keep only the boxes > e.g. 3 * mean. typedef std::pair<double, boxPtr> dens_box; // Map that will sort the boxes by increasing density. The key = density; value = box *. typename std::multimap<double, boxPtr> sortedBoxes; progress(0.20, "Sorting Boxes by Density"); typename std::vector<boxPtr>::iterator it1; typename std::vector<boxPtr>::iterator it1_end = boxes.end(); for (it1 = boxes.begin(); it1 != it1_end; it1++) { boxPtr box = *it1; double density = box->getSignalNormalized() * densityScalingFactor; // Skip any boxes with too small a signal density. if (density > thresholdDensity) sortedBoxes.insert(dens_box(density,box)); } // List of chosen possible peak boxes. std::vector<boxPtr> peakBoxes; prog = new Progress(this, 0.30, 0.95, MaxPeaks); int64_t numBoxesFound = 0; // Now we go (backwards) through the map // e.g. from highest density down to lowest density. typename std::multimap<double, boxPtr>::reverse_iterator it2; typename std::multimap<double, boxPtr>::reverse_iterator it2_end = sortedBoxes.rend(); for (it2 = sortedBoxes.rbegin(); it2 != it2_end; it2++) { signal_t density = it2->first; boxPtr box = it2->second; #ifndef MDBOX_TRACK_CENTROID coord_t boxCenter[nd]; box->calculateCentroid(boxCenter); #else const coord_t * boxCenter = box->getCentroid(); #endif // Compare to all boxes already picked. bool badBox = false; for (typename std::vector<boxPtr>::iterator it3=peakBoxes.begin(); it3 != peakBoxes.end(); it3++) { #ifndef MDBOX_TRACK_CENTROID coord_t otherCenter[nd]; (*it3)->calculateCentroid(otherCenter); #else const coord_t * otherCenter = (*it3)->getCentroid(); #endif // Distance between this box and a box we already put in. coord_t distSquared = 0.0; for (size_t d=0; d<nd; d++) { coord_t dist = otherCenter[d] - boxCenter[d]; distSquared += (dist * dist); } // Reject this box if it is too close to another previously found box. if (distSquared < peakRadiusSquared) { badBox = true; break; } } // The box was not rejected for another reason. if (!badBox) { if (numBoxesFound++ >= MaxPeaks) { g_log.notice() << "Number of peaks found exceeded the limit of " << MaxPeaks << ". Stopping peak finding." << std::endl; break; } peakBoxes.push_back(box); g_log.information() << "Found box at "; for (size_t d=0; d<nd; d++) g_log.information() << (d>0?",":"") << boxCenter[d]; g_log.information() << "; Density = " << density << std::endl; // Report progres for each box found. prog->report("Finding Peaks"); } } prog->resetNumSteps(numBoxesFound, 0.95, 1.0); // Copy the instrument, sample, run to the peaks workspace. peakWS->copyExperimentInfoFrom(ei.get()); // --- Convert the "boxes" to peaks ---- for (typename std::vector<boxPtr>::iterator it3=peakBoxes.begin(); it3 != peakBoxes.end(); it3++) { // The center of the box = Q in the lab frame boxPtr box = *it3; #ifndef MDBOX_TRACK_CENTROID coord_t boxCenter[nd]; box->calculateCentroid(boxCenter); #else const coord_t * boxCenter = box->getCentroid(); #endif V3D Q(boxCenter[0], boxCenter[1], boxCenter[2]); // Create a peak and add it // Empty starting peak. Peak p; try { if (dimType == QLAB) { // Build using the Q-lab-frame constructor p = Peak(inst, Q); // Save gonio matrix for later p.setGoniometerMatrix(goniometer); } else if (dimType == QSAMPLE) { // Build using the Q-sample-frame constructor p = Peak(inst, Q, goniometer); } } catch (std::exception &e) { g_log.notice() << "Error creating peak at " << Q << " because of '" << e.what() << "'. Peak will be skipped." << std::endl; continue; } try { // Look for a detector p.findDetector(); } catch (...) { /* Ignore errors in ray-tracer TODO: Handle for WISH data later */ } // The "bin count" used will be the box density. p.setBinCount( box->getSignalNormalized() * densityScalingFactor); // Save the run number found before. p.setRunNumber(runNumber); peakWS->addPeak(p); // Report progres for each box found. prog->report("Adding Peaks"); } // for each box found }
void SaveMD::doSaveEvents(typename MDEventWorkspace<MDE, nd>::sptr ws) { std::string filename = getPropertyValue("Filename"); bool update = getProperty("UpdateFileBackEnd"); bool MakeFileBacked = getProperty("MakeFileBacked"); bool wsIsFileBacked = ws->isFileBacked(); if (update && MakeFileBacked) throw std::invalid_argument( "Please choose either UpdateFileBackEnd or MakeFileBacked, not both."); if (MakeFileBacked && wsIsFileBacked) throw std::invalid_argument( "You picked MakeFileBacked but the workspace is already file-backed!"); BoxController_sptr bc = ws->getBoxController(); if (!wsIsFileBacked) { // Erase the file if it exists Poco::File oldFile(filename); if (oldFile.exists()) oldFile.remove(); } auto prog = new Progress(this, 0.0, 0.05, 1); if (update) // workspace has its own file and ignores any changes to the // algorithm parameters { if (!ws->isFileBacked()) throw std::runtime_error(" attempt to update non-file backed workspace"); filename = bc->getFileIO()->getFileName(); } //----------------------------------------------------------------------------------------------------- // create or open WS group and put there additional information about WS and // its dimensions int nDims = static_cast<int>(nd); bool data_exist; auto file = file_holder_type(MDBoxFlatTree::createOrOpenMDWSgroup( filename, nDims, MDE::getTypeName(), false, data_exist)); // Save each NEW ExperimentInfo to a spot in the file MDBoxFlatTree::saveExperimentInfos(file.get(), ws); if (!update || !data_exist) { MDBoxFlatTree::saveWSGenericInfo(file.get(), ws); } file->closeGroup(); file->close(); MDBoxFlatTree BoxFlatStruct; //----------------------------------------------------------------------------------------------------- if (update) // the workspace is already file backed; { // remove all boxes from the DiskBuffer. DB will calculate boxes positions // on HDD. bc->getFileIO()->flushCache(); // flatten the box structure; this will remember boxes file positions in the // box structure BoxFlatStruct.initFlatStructure(ws, filename); } else // not file backed; { // the boxes file positions are unknown and we need to calculate it. BoxFlatStruct.initFlatStructure(ws, filename); // create saver class auto Saver = boost::shared_ptr<API::IBoxControllerIO>( new DataObjects::BoxControllerNeXusIO(bc.get())); Saver->setDataType(sizeof(coord_t), MDE::getTypeName()); if (MakeFileBacked) { // store saver with box controller bc->setFileBacked(Saver, filename); // get access to boxes array std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes(); // calculate the position of the boxes on file, indicating to make them // saveable and that the boxes were not saved. BoxFlatStruct.setBoxesFilePositions(true); prog->resetNumSteps(boxes.size(), 0.06, 0.90); for (auto &boxe : boxes) { auto saveableTag = boxe->getISaveable(); if (saveableTag) // only boxes can be saveable { // do not spend time on empty boxes if (boxe->getDataInMemorySize() == 0) continue; // save boxes directly using the boxes file postion, precalculated in // boxFlatStructure. saveableTag->save(); // remove boxes data from memory. This will actually correctly set the // tag indicatin that data were not loaded. saveableTag->clearDataFromMemory(); // put boxes into write buffer wich will save them when necessary // Saver->toWrite(saveTag); prog->report("Saving Box"); } } // remove everything from diskBuffer; (not sure if it really necessary // but just in case , should not make any harm) Saver->flushCache(); // drop NeXus on HDD (not sure if it really necessary but just in case ) Saver->flushData(); } else // just save data, and finish with it { Saver->openFile(filename, "w"); BoxFlatStruct.setBoxesFilePositions(false); std::vector<API::IMDNode *> &boxes = BoxFlatStruct.getBoxes(); std::vector<uint64_t> &eventIndex = BoxFlatStruct.getEventIndex(); prog->resetNumSteps(boxes.size(), 0.06, 0.90); for (size_t i = 0; i < boxes.size(); i++) { if (eventIndex[2 * i + 1] == 0) continue; boxes[i]->saveAt(Saver.get(), eventIndex[2 * i]); prog->report("Saving Box"); } Saver->closeFile(); } } // -------------- Save Box Structure ------------------------------------- // OK, we've filled these big arrays of data representing flat box structrre. // Save them. progress(0.91, "Writing Box Data"); prog->resetNumSteps(8, 0.92, 1.00); // Save box structure; BoxFlatStruct.saveBoxStructure(filename); delete prog; ws->setFileNeedsUpdating(false); }
void BinToMDHistoWorkspace::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) { BoxController_sptr bc = ws->getBoxController(); // Start with signal at 0.0 outWS->setTo(0.0, 0.0); // Cache some data to speed up accessing them a bit indexMultiplier = new size_t[outD]; for (size_t d=0; d<outD; d++) { if (d > 0) indexMultiplier[d] = outWS->getIndexMultiplier()[d-1]; else indexMultiplier[d] = 1; } signals = outWS->getSignalArray(); errors = outWS->getErrorSquaredArray(); // The dimension (in the output workspace) along which we chunk for parallel processing // TODO: Find the smartest dimension to chunk against size_t chunkDimension = 0; // How many bins (in that dimension) per chunk. // Try to split it so each core will get 2 tasks: int chunkNumBins = int(binDimensions[chunkDimension]->getNBins() / (Mantid::Kernel::ThreadPool::getNumPhysicalCores() * 2)); if (chunkNumBins < 1) chunkNumBins = 1; // Do we actually do it in parallel? bool doParallel = getProperty("Parallel"); // Not if file-backed! if (bc->isFileBacked()) doParallel = false; if (!doParallel) chunkNumBins = int(binDimensions[chunkDimension]->getNBins()); // Total number of steps size_t progNumSteps = 0; if (prog) prog->setNotifyStep(0.1); if (prog) prog->resetNumSteps(100, 0.00, 1.0); // Run the chunks in parallel. There is no overlap in the output workspace so it is // thread safe to write to it.. PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) ) for(int chunk=0; chunk < int(binDimensions[chunkDimension]->getNBins()); chunk += chunkNumBins) { PARALLEL_START_INTERUPT_REGION // Region of interest for this chunk. size_t * chunkMin = new size_t[outD]; size_t * chunkMax = new size_t[outD]; for (size_t bd=0; bd<outD; bd++) { // Same limits in the other dimensions chunkMin[bd] = 0; chunkMax[bd] = binDimensions[bd]->getNBins(); } // Parcel out a chunk in that single dimension dimension chunkMin[chunkDimension] = size_t(chunk); if (size_t(chunk+chunkNumBins) > binDimensions[chunkDimension]->getNBins()) chunkMax[chunkDimension] = binDimensions[chunkDimension]->getNBins(); else chunkMax[chunkDimension] = size_t(chunk+chunkNumBins); // Build an implicit function (it needs to be in the space of the MDEventWorkspace) MDImplicitFunction * function = this->getImplicitFunctionForChunk(chunkMin, chunkMax); // Use getBoxes() to get an array with a pointer to each box std::vector<IMDBox<MDE,nd>*> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, hopefully. if (bc->isFileBacked()) IMDBox<MDE, nd>::sortBoxesByFilePos(boxes); // For progress reporting, the # of boxes if (prog) { PARALLEL_CRITICAL(BinToMDHistoWorkspace_progress) { std::cout << "Chunk " << chunk << ": found " << boxes.size() << " boxes within the implicit function." << std::endl; progNumSteps += boxes.size(); prog->setNumSteps( progNumSteps ); } } // Go through every box for this chunk. for (size_t i=0; i<boxes.size(); i++) { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]); // Perform the binning in this separate method. if (box) this->binMDBox(box, chunkMin, chunkMax); // Progress reporting if (prog) prog->report(); }// for each box in the vector PARALLEL_END_INTERUPT_REGION } // for each chunk in parallel PARALLEL_CHECK_INTERUPT_REGION // Now the implicit function if (implicitFunction) { prog->report("Applying implicit function."); signal_t nan = std::numeric_limits<signal_t>::quiet_NaN(); outWS->applyImplicitFunction(implicitFunction, nan, nan); } }
void SliceMD::slice(typename MDEventWorkspace<MDE, nd>::sptr ws) { // Create the ouput workspace typename MDEventWorkspace<OMDE, ond>::sptr outWS( new MDEventWorkspace<OMDE, ond>()); for (size_t od = 0; od < m_binDimensions.size(); od++) { outWS->addDimension(m_binDimensions[od]); } outWS->setCoordinateSystem(ws->getSpecialCoordinateSystem()); outWS->initialize(); // Copy settings from the original box controller BoxController_sptr bc = ws->getBoxController(); // store wrute buffer size for the future // uint64_t writeBufSize = // bc->getFileIO()getDiskBuffer().getWriteBufferSize(); // and disable write buffer (if any) for input MD Events for this algorithm // purposes; // bc->setCacheParameters(1,0); BoxController_sptr obc = outWS->getBoxController(); // Use the "number of bins" as the "split into" parameter for (size_t od = 0; od < m_binDimensions.size(); od++) obc->setSplitInto(od, m_binDimensions[od]->getNBins()); obc->setSplitThreshold(bc->getSplitThreshold()); bool bTakeDepthFromInputWorkspace = getProperty("TakeMaxRecursionDepthFromInput"); int tempDepth = getProperty("MaxRecursionDepth"); size_t maxDepth = bTakeDepthFromInputWorkspace ? bc->getMaxDepth() : size_t(tempDepth); obc->setMaxDepth(maxDepth); // size_t outputSize = writeBufSize; // obc->setCacheParameters(sizeof(OMDE),outputSize); obc->resetNumBoxes(); // Perform the first box splitting outWS->splitBox(); size_t lastNumBoxes = obc->getTotalNumMDBoxes(); // --- File back end ? ---------------- std::string filename = getProperty("OutputFilename"); if (!filename.empty()) { // First save to the NXS file g_log.notice() << "Running SaveMD to create file back-end" << std::endl; IAlgorithm_sptr alg = createChildAlgorithm("SaveMD"); alg->setPropertyValue("Filename", filename); alg->setProperty("InputWorkspace", outWS); alg->setProperty("MakeFileBacked", true); alg->executeAsChildAlg(); if (!obc->isFileBacked()) throw std::runtime_error("SliceMD with file-backed output: Can not set " "up file-backed output workspace "); auto IOptr = obc->getFileIO(); size_t outBufSize = IOptr->getWriteBufferSize(); // the buffer size for resulting workspace; reasonable size is at least 10 // data chunk sizes (nice to verify) if (outBufSize < 10 * IOptr->getDataChunk()) { outBufSize = 10 * IOptr->getDataChunk(); IOptr->setWriteBufferSize(outBufSize); } } // Function defining which events (in the input dimensions) to place in the // output MDImplicitFunction *function = this->getImplicitFunctionForChunk(NULL, NULL); std::vector<API::IMDNode *> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, // hopefully. bool fileBackedWS = bc->isFileBacked(); if (fileBackedWS) API::IMDNode::sortObjByID(boxes); Progress *prog = new Progress(this, 0.0, 1.0, boxes.size()); // The root of the output workspace MDBoxBase<OMDE, ond> *outRootBox = outWS->getBox(); // if target workspace has events, we should count them as added uint64_t totalAdded = outWS->getNEvents(); uint64_t numSinceSplit = 0; // Go through every box for this chunk. // PARALLEL_FOR_IF( !bc->isFileBacked() ) for (int i = 0; i < int(boxes.size()); i++) { MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxes[i]); // Perform the binning in this separate method. if (box) { // An array to hold the rotated/transformed coordinates coord_t outCenter[ond]; const std::vector<MDE> &events = box->getConstEvents(); typename std::vector<MDE>::const_iterator it = events.begin(); typename std::vector<MDE>::const_iterator it_end = events.end(); for (; it != it_end; it++) { // Cache the center of the event (again for speed) const coord_t *inCenter = it->getCenter(); if (function->isPointContained(inCenter)) { // Now transform to the output dimensions m_transformFromOriginal->apply(inCenter, outCenter); // Create the event OMDE newEvent(it->getSignal(), it->getErrorSquared(), outCenter); // Copy extra data, if any copyEvent(*it, newEvent); // Add it to the workspace outRootBox->addEvent(newEvent); numSinceSplit++; } } box->releaseEvents(); // Ask BC if one needs to split boxes if (obc->shouldSplitBoxes(totalAdded, numSinceSplit, lastNumBoxes)) // if (numSinceSplit > 20000000 || (i == int(boxes.size()-1))) { // This splits up all the boxes according to split thresholds and sizes. Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts); outWS->splitAllIfNeeded(ts); tp.joinAll(); // Accumulate stats totalAdded += numSinceSplit; numSinceSplit = 0; lastNumBoxes = obc->getTotalNumMDBoxes(); // Progress reporting if (!fileBackedWS) prog->report(i); } if (fileBackedWS) { if (!(i % 10)) prog->report(i); } } // is box } // for each box in the vector prog->report(); outWS->splitAllIfNeeded(NULL); // Refresh all cache. outWS->refreshCache(); g_log.notice() << totalAdded << " " << OMDE::getTypeName() << "'s added to the output workspace." << std::endl; if (outWS->isFileBacked()) { // Update the file-back-end g_log.notice() << "Running SaveMD" << std::endl; IAlgorithm_sptr alg = createChildAlgorithm("SaveMD"); alg->setProperty("UpdateFileBackEnd", true); alg->setProperty("InputWorkspace", outWS); alg->executeAsChildAlg(); } // return the size of the input workspace write buffer to its initial value // bc->setCacheParameters(sizeof(MDE),writeBufSize); this->setProperty("OutputWorkspace", boost::dynamic_pointer_cast<IMDEventWorkspace>(outWS)); delete prog; }
void PlusMD::doPlus(typename MDEventWorkspace<MDE, nd>::sptr ws) { typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws; typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event); if (!ws1 || !ws2) throw std::runtime_error("Incompatible workspace types passed to PlusMD."); MDBoxBase<MDE,nd> * box1 = ws1->getBox(); MDBoxBase<MDE,nd> * box2 = ws2->getBox(); Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes()); // How many events you started with size_t initial_numEvents = ws1->getNPoints(); // Make a leaf-only iterator through all boxes with events in the RHS workspace MDBoxIterator<MDE,nd> it2(box2, 1000, true); do { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox()); if (box) { // Copy the events from WS2 and add them into WS1 const std::vector<MDE> & events = box->getConstEvents(); // Add events, with bounds checking box1->addEvents(events); box->releaseEvents(); } prog.report("Adding Events"); } while (it2.next()); this->progress(0.41, "Splitting Boxes"); Progress * prog2 = new Progress(this, 0.4, 0.9, 100); ThreadScheduler * ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts, 0, prog2); ws1->splitAllIfNeeded(ts); prog2->resetNumSteps( ts->size(), 0.4, 0.6); tp.joinAll(); // // Now we need to save all the data that was not saved before. // if (ws1->isFileBacked()) // { // // Flush anything else in the to-write buffer // BoxController_sptr bc = ws1->getBoxController(); // // prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0); // MDBoxIterator<MDE,nd> it1(box1, 1000, true); // while (true) // { // MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox()); // if (box) // { // // Something was maybe added to this box // if (box->getEventVectorSize() > 0) // { // // By getting the events, this will merge the newly added and the cached events. // box->getEvents(); // // The MRU to-write cache will optimize writes by reducing seek times // box->releaseEvents(); // } // } // prog.report("Saving"); // if (!it1.next()) break; // } // //bc->getDiskBuffer().flushCache(); // // Flush the data writes to disk. // box1->flushData(); // } this->progress(0.95, "Refreshing cache"); ws1->refreshCache(); // Set a marker that the file-back-end needs updating if the # of events changed. if (ws1->getNPoints() != initial_numEvents) ws1->setFileNeedsUpdating(true); }
void FakeMDEventData::addFakePeak(typename MDEventWorkspace<MDE, nd>::sptr ws) { std::vector<double> params = getProperty("PeakParams"); bool RandomizeSignal = getProperty("RandomizeSignal"); if (params.empty()) return; if (params.size() != nd + 2) throw std::invalid_argument("PeakParams needs to have ndims+2 arguments."); if (params[0] <= 0) throw std::invalid_argument("PeakParams: number_of_events needs to be > 0"); size_t num = size_t(params[0]); Progress prog(this, 0.0, 1.0, 100); size_t progIncrement = num / 100; if (progIncrement == 0) progIncrement = 1; // Width of the peak double desiredRadius = params.back(); boost::mt19937 rng; boost::uniform_real<coord_t> u2(0, 1.0); // Random from 0 to 1.0 boost::variate_generator<boost::mt19937 &, boost::uniform_real<coord_t>> genUnit(rng, u2); int randomSeed = getProperty("RandomSeed"); rng.seed((unsigned int)(randomSeed)); // Inserter to help choose the correct event type auto eventHelper = MDEvents::MDEventInserter<typename MDEventWorkspace<MDE, nd>::sptr>(ws); for (size_t i = 0; i < num; ++i) { // Algorithm to generate points along a random n-sphere (sphere with not // necessarily 3 dimensions) // from http://en.wikipedia.org/wiki/N-sphere as of May 6, 2011. // First, points in a hyper-cube of size 1.0, centered at 0. coord_t centers[nd]; coord_t radiusSquared = 0; for (size_t d = 0; d < nd; d++) { centers[d] = genUnit() - 0.5f; // Distribute around +- the center radiusSquared += centers[d] * centers[d]; } // Make a unit vector pointing in this direction coord_t radius = static_cast<coord_t>(sqrt(radiusSquared)); for (size_t d = 0; d < nd; d++) centers[d] /= radius; // Now place the point along this radius, scaled with ^1/n for uniformity. coord_t radPos = genUnit(); radPos = static_cast<coord_t>( pow(radPos, static_cast<coord_t>(1.0 / static_cast<coord_t>(nd)))); for (size_t d = 0; d < nd; d++) { // Multiply by the scaling and the desired peak radius centers[d] *= (radPos * static_cast<coord_t>(desiredRadius)); // Also offset by the center of the peak, as taken in Params centers[d] += static_cast<coord_t>(params[d + 1]); } // Default or randomized error/signal float signal = 1.0; float errorSquared = 1.0; if (RandomizeSignal) { signal = float(0.5 + genUnit()); errorSquared = float(0.5 + genUnit()); } // Create and add the event. eventHelper.insertMDEvent(signal, errorSquared, 1, pickDetectorID(), centers); // 1 = run number // Progress report if ((i % progIncrement) == 0) prog.report(); } ws->splitBox(); Kernel::ThreadScheduler *ts = new ThreadSchedulerFIFO(); ThreadPool tp(ts); ws->splitAllIfNeeded(ts); tp.joinAll(); ws->refreshCache(); }
void FakeMDEventData::addFakeRegularData( const std::vector<double> ¶ms, typename MDEventWorkspace<MDE, nd>::sptr ws) { // the parameters for regular distribution of events over the box std::vector<double> startPoint(nd), delta(nd); std::vector<size_t> indexMax(nd); size_t gridSize(0); // bool RandomizeSignal = getProperty("RandomizeSignal"); size_t num = size_t(params[0]); if (num == 0) throw std::invalid_argument( " number of distributed events can not be equal to 0"); Progress prog(this, 0.0, 1.0, 100); size_t progIncrement = num / 100; if (progIncrement == 0) progIncrement = 1; // Inserter to help choose the correct event type auto eventHelper = MDEvents::MDEventInserter<typename MDEventWorkspace<MDE, nd>::sptr>(ws); gridSize = 1; for (size_t d = 0; d < nd; ++d) { double min = ws->getDimension(d)->getMinimum(); double max = ws->getDimension(d)->getMaximum(); double shift = params[d * 2 + 1]; double step = params[d * 2 + 2]; if (shift < 0) shift = 0; if (shift >= step) shift = step * (1 - FLT_EPSILON); startPoint[d] = min + shift; if ((startPoint[d] < min) || (startPoint[d] >= max)) throw std::invalid_argument("RegularData: starting point must be within " "the box for all dimensions."); if (step <= 0) throw(std::invalid_argument( "Step of the regular grid is less or equal to 0")); indexMax[d] = size_t((max - min) / step); if (indexMax[d] == 0) indexMax[d] = 1; // deal with round-off errors while ((startPoint[d] + double(indexMax[d] - 1) * step) >= max) step *= (1 - FLT_EPSILON); delta[d] = step; gridSize *= indexMax[d]; } // Create all the requested events std::vector<size_t> indexes; size_t cellCount(0); for (size_t i = 0; i < num; ++i) { coord_t centers[nd]; Kernel::Utils::getIndicesFromLinearIndex(cellCount, indexMax, indexes); ++cellCount; if (cellCount >= gridSize) cellCount = 0; for (size_t d = 0; d < nd; d++) { centers[d] = coord_t(startPoint[d] + delta[d] * double(indexes[d])); } // Default or randomized error/signal float signal = 1.0; float errorSquared = 1.0; // if (RandomizeSignal) //{ // signal = float(0.5 + genUnit()); // errorSquared = float(0.5 + genUnit()); //} // Create and add the event. eventHelper.insertMDEvent(signal, errorSquared, 1, pickDetectorID(), centers); // 1 = run number // Progress report if ((i % progIncrement) == 0) prog.report(); } }
void vtkSplatterPlotFactory::doCreate(typename MDEventWorkspace<MDE, nd>::sptr ws) const { bool VERBOSE = true; CPUTimer tim; // Acquire a scoped read-only lock to the workspace (prevent segfault // from algos modifying ws) ReadLock lock(*ws); // Find out how many events to plot, and the percentage of the largest // boxes to use. size_t totalPoints = ws->getNPoints(); size_t numPoints = m_numPoints; if (numPoints > totalPoints) { numPoints = totalPoints; } double percent_to_use = m_percentToUse; // Fail safe limits on fraction of boxes to use if (percent_to_use <= 0) { percent_to_use = 5; } if (percent_to_use > 100) { percent_to_use = 100; } // First we get all the boxes, up to the given depth; with or wo the // slice function std::vector<API::IMDNode *> boxes; if (this->slice) { ws->getBox()->getBoxes(boxes, 1000, true, this->sliceImplicitFunction); } else { ws->getBox()->getBoxes(boxes, 1000, true); } if (VERBOSE) { std::cout << tim << " to retrieve the "<< boxes.size() << " boxes down."<< std::endl; } std::string new_name = ws->getName(); if (new_name != m_wsName || m_buildSortedList) { m_wsName = new_name; m_buildSortedList = false; m_sortedBoxes.clear(); // get list of boxes with signal > 0 and sort // the list in order of decreasing signal for (size_t i = 0; i < boxes.size(); i++) { MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]); if (box) { size_t newPoints = box->getNPoints(); if (newPoints > 0) { m_sortedBoxes.push_back(box); } } } if (VERBOSE) { std::cout << "START SORTING" << std::endl; } std::sort(m_sortedBoxes.begin(), m_sortedBoxes.end(), CompareNormalizedSignal); if (VERBOSE) { std::cout << "DONE SORTING" << std::endl; } } size_t num_boxes_to_use = static_cast<size_t>(percent_to_use * static_cast<double>(m_sortedBoxes.size()) / 100.0); if (num_boxes_to_use >= m_sortedBoxes.size()) { num_boxes_to_use = m_sortedBoxes.size()-1; } // restrict the number of points to the // number of points in boxes being used size_t total_points_available = 0; for (size_t i = 0; i < num_boxes_to_use; i++) { size_t newPoints = m_sortedBoxes[i]->getNPoints(); total_points_available += newPoints; } if (numPoints > total_points_available) { numPoints = total_points_available; } size_t points_per_box = 0; // calculate the average number of points to use per box if (num_boxes_to_use > 0) { points_per_box = numPoints / num_boxes_to_use; } if (points_per_box < 1) { points_per_box = 1; } if (VERBOSE) { std::cout << "numPoints = " << numPoints << std::endl; std::cout << "num boxes in all = " << boxes.size() << std::endl; std::cout << "num boxes above zero = " << m_sortedBoxes.size() << std::endl; std::cout << "num boxes to use = " << num_boxes_to_use << std::endl; std::cout << "total_points_available = " << total_points_available << std::endl; std::cout << "points needed per box = " << points_per_box << std::endl; } // First save the events and signals that we actually use. // For each box, get up to the average number of points // we want from each box, limited by the number of points // in the box. NOTE: since boxes have different numbers // of events, we will not get all the events requested. // Also, if we are using a smaller number of points, we // won't get points from some of the boxes with lower signal. std::vector<float> saved_signals; std::vector<const coord_t*> saved_centers; std::vector<size_t> saved_n_points_in_cell; saved_signals.reserve(numPoints); saved_centers.reserve(numPoints); saved_n_points_in_cell.reserve(numPoints); size_t pointIndex = 0; size_t box_index = 0; bool done = false; while (box_index < num_boxes_to_use && !done) { MDBox<MDE,nd> *box = dynamic_cast<MDBox<MDE,nd> *>(m_sortedBoxes[box_index]); box_index++; if (NULL == box) { continue; } float signal_normalized = static_cast<float>(box->getSignalNormalized()); size_t newPoints = box->getNPoints(); size_t num_from_this_box = points_per_box; if (num_from_this_box > newPoints) { num_from_this_box = newPoints; } const std::vector<MDE> & events = box->getConstEvents(); size_t startPointIndex = pointIndex; size_t event_index = 0; while (event_index < num_from_this_box && !done) { const MDE & ev = events[event_index]; event_index++; const coord_t * center = ev.getCenter(); // Save location saved_centers.push_back(center); pointIndex++; if (pointIndex >= numPoints) { done = true; } } box->releaseEvents(); // Save signal saved_signals.push_back(signal_normalized); // Save cell size saved_n_points_in_cell.push_back(pointIndex-startPointIndex); } numPoints = saved_centers.size(); size_t numCells = saved_signals.size(); if (VERBOSE) { std::cout << "Recorded data for all points" << std::endl; std::cout << "numPoints = " << numPoints << std::endl; std::cout << "numCells = " << numCells << std::endl; } // Create the point list, one position for each point actually used vtkPoints *points = vtkPoints::New(); points->Allocate(numPoints); points->SetNumberOfPoints(numPoints); // The list of IDs of points used, one ID per point, since points // are not reused to form polygon facets, etc. vtkIdType *ids = new vtkIdType[numPoints]; // Only one scalar for each cell, NOT one per point vtkFloatArray *signal = vtkFloatArray::New(); signal->Allocate(numCells); signal->SetName(m_scalarName.c_str()); // Create the data set. Need space for each cell, not for each point vtkUnstructuredGrid *visualDataSet = vtkUnstructuredGrid::New(); this->dataSet = visualDataSet; visualDataSet->Allocate(numCells); // Now copy the saved point, cell and signal info into vtk data structures pointIndex = 0; for (size_t cell_i = 0; cell_i < numCells; cell_i++) { size_t startPointIndex = pointIndex; for (size_t point_i = 0; point_i < saved_n_points_in_cell[cell_i]; point_i++) { points->SetPoint(pointIndex, saved_centers[pointIndex]); ids[pointIndex] = pointIndex; pointIndex++; } signal->InsertNextTuple1(saved_signals[cell_i]); visualDataSet->InsertNextCell(VTK_POLY_VERTEX, saved_n_points_in_cell[cell_i], ids+startPointIndex); } if (VERBOSE) { std::cout << tim << " to create " << pointIndex << " points." << std::endl; } // Shrink to fit //points->Squeeze(); signal->Squeeze(); visualDataSet->Squeeze(); // Add points and scalars visualDataSet->SetPoints(points); visualDataSet->GetCellData()->SetScalars(signal); delete [] ids; }
void BinToMDHistoWorkspace::do_centerpointBin(typename MDEventWorkspace<MDE, nd>::sptr ws) { bool DODEBUG = true; CPUTimer tim; // Number of output binning dimensions found size_t outD = binDimensions.size(); //Since the costs are not known ahead of time, use a simple FIFO buffer. ThreadScheduler * ts = new ThreadSchedulerFIFO(); // Create the threadpool with: all CPUs, a progress reporter ThreadPool tp(ts, 0, prog); // Big efficiency gain is obtained by grouping a few bins per task. size_t binsPerTask = 100; // For progress reporting, the approx # of tasks if (prog) prog->setNumSteps( int(outWS->getNPoints()/100) ); // The root-level box. IMDBox<MDE,nd> * rootBox = ws->getBox(); // This is the limit to loop over in each dimension size_t * index_max = new size_t[outD]; for (size_t bd=0; bd<outD; bd++) index_max[bd] = binDimensions[bd]->getNBins(); // Cache a calculation to convert indices x,y,z,t into a linear index. size_t * index_maker = new size_t[outD]; Utils::NestedForLoop::SetUpIndexMaker(outD, index_maker, index_max); int numPoints = int(outWS->getNPoints()); // Run in OpenMP with dynamic scheduling and a smallish chunk size (binsPerTask) // Right now, not parallel for file-backed systems. bool fileBacked = (ws->getBoxController()->getFile() != NULL); PRAGMA_OMP(parallel for schedule(dynamic, binsPerTask) if (!fileBacked) ) for (int i=0; i < numPoints; i++) { PARALLEL_START_INTERUPT_REGION size_t linear_index = size_t(i); // nd >= outD in all cases so this is safe. size_t index[nd]; // Get the index at each dimension for this bin. Utils::NestedForLoop::GetIndicesFromLinearIndex(outD, linear_index, index_maker, index_max, index); // Construct the bin and its coordinates MDBin<MDE,nd> bin; for (size_t bd=0; bd<outD; bd++) { // Index in this binning dimension (i_x, i_y, etc.) size_t idx = index[bd]; // Dimension in the MDEventWorkspace size_t d = dimensionToBinFrom[bd]; // Corresponding extents bin.m_min[d] = binDimensions[bd]->getX(idx); bin.m_max[d] = binDimensions[bd]->getX(idx+1); } bin.m_index = linear_index; bool dimensionsUsed[nd]; for (size_t d=0; d<nd; d++) dimensionsUsed[d] = (d<3); // Check if the bin is in the ImplicitFunction (if any) bool binContained = true; if (implicitFunction) { binContained = implicitFunction->isPointContained(bin.m_min); //TODO. Correct argument passed to this method? } if (binContained) { // Array of bools set to true when a dimension is fully contained (binary splitting only) bool fullyContained[nd]; for (size_t d=0; d<nd; d++) fullyContained[d] = false; // This will recursively bin into the sub grids rootBox->centerpointBin(bin, fullyContained); // Save the data into the dense histogram outWS->setSignalAt(linear_index, bin.m_signal); outWS->setErrorAt(linear_index, bin.m_errorSquared); } // Report progress but not too often. if (((linear_index % 100) == 0) && prog ) prog->report(); PARALLEL_END_INTERUPT_REGION } // (for each linear index) PARALLEL_CHECK_INTERUPT_REGION if (DODEBUG) std::cout << tim << " to run the openmp loop.\n"; delete [] index_max; delete [] index_maker; }
void BinMD::binByIterating(typename MDEventWorkspace<MDE, nd>::sptr ws) { BoxController_sptr bc = ws->getBoxController(); // store exisiting write buffer size for the future // uint64_t writeBufSize =bc->getDiskBuffer().getWriteBufferSize(); // and disable write buffer (if any) for input MD Events for this algorithm // purposes; // bc->setCacheParameters(1,0); // Cache some data to speed up accessing them a bit indexMultiplier = new size_t[m_outD]; for (size_t d = 0; d < m_outD; d++) { if (d > 0) indexMultiplier[d] = outWS->getIndexMultiplier()[d - 1]; else indexMultiplier[d] = 1; } signals = outWS->getSignalArray(); errors = outWS->getErrorSquaredArray(); numEvents = outWS->getNumEventsArray(); // Start with signal/error/numEvents at 0.0 outWS->setTo(0.0, 0.0, 0.0); // The dimension (in the output workspace) along which we chunk for parallel // processing // TODO: Find the smartest dimension to chunk against size_t chunkDimension = 0; // How many bins (in that dimension) per chunk. // Try to split it so each core will get 2 tasks: int chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins() / (PARALLEL_GET_MAX_THREADS * 2)); if (chunkNumBins < 1) chunkNumBins = 1; // Do we actually do it in parallel? bool doParallel = getProperty("Parallel"); // Not if file-backed! if (bc->isFileBacked()) doParallel = false; if (!doParallel) chunkNumBins = int(m_binDimensions[chunkDimension]->getNBins()); // Total number of steps size_t progNumSteps = 0; if (prog) prog->setNotifyStep(0.1); if (prog) prog->resetNumSteps(100, 0.00, 1.0); // Run the chunks in parallel. There is no overlap in the output workspace so // it is thread safe to write to it.. // cppcheck-suppress syntaxError PRAGMA_OMP( parallel for schedule(dynamic,1) if (doParallel) ) for (int chunk = 0; chunk < int(m_binDimensions[chunkDimension]->getNBins()); chunk += chunkNumBins) { PARALLEL_START_INTERUPT_REGION // Region of interest for this chunk. std::vector<size_t> chunkMin(m_outD); std::vector<size_t> chunkMax(m_outD); for (size_t bd = 0; bd < m_outD; bd++) { // Same limits in the other dimensions chunkMin[bd] = 0; chunkMax[bd] = m_binDimensions[bd]->getNBins(); } // Parcel out a chunk in that single dimension dimension chunkMin[chunkDimension] = size_t(chunk); if (size_t(chunk + chunkNumBins) > m_binDimensions[chunkDimension]->getNBins()) chunkMax[chunkDimension] = m_binDimensions[chunkDimension]->getNBins(); else chunkMax[chunkDimension] = size_t(chunk + chunkNumBins); // Build an implicit function (it needs to be in the space of the // MDEventWorkspace) MDImplicitFunction *function = this->getImplicitFunctionForChunk(chunkMin.data(), chunkMax.data()); // Use getBoxes() to get an array with a pointer to each box std::vector<API::IMDNode *> boxes; // Leaf-only; no depth limit; with the implicit function passed to it. ws->getBox()->getBoxes(boxes, 1000, true, function); // Sort boxes by file position IF file backed. This reduces seeking time, // hopefully. if (bc->isFileBacked()) API::IMDNode::sortObjByID(boxes); // For progress reporting, the # of boxes if (prog) { PARALLEL_CRITICAL(BinMD_progress) { g_log.debug() << "Chunk " << chunk << ": found " << boxes.size() << " boxes within the implicit function.\n"; progNumSteps += boxes.size(); prog->setNumSteps(progNumSteps); } } // Go through every box for this chunk. for (auto &boxe : boxes) { MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxe); // Perform the binning in this separate method. if (box && !box->getIsMasked()) this->binMDBox(box, chunkMin.data(), chunkMax.data()); // Progress reporting if (prog) prog->report(); // For early cancelling of the loop if (this->m_cancel) break; } // for each box in the vector PARALLEL_END_INTERUPT_REGION } // for each chunk in parallel PARALLEL_CHECK_INTERUPT_REGION // Now the implicit function if (implicitFunction) { if (prog) prog->report("Applying implicit function."); signal_t nan = std::numeric_limits<signal_t>::quiet_NaN(); outWS->applyImplicitFunction(implicitFunction, nan, nan); } // return the size of the input workspace write buffer to its initial value // bc->setCacheParameters(sizeof(MDE),writeBufSize); }
void vtkMDHexFactory::doCreate( typename MDEventWorkspace<MDE, nd>::sptr ws) const { bool VERBOSE = true; CPUTimer tim; // Acquire a scoped read-only lock to the workspace (prevent segfault from // algos modifying ws) ReadLock lock(*ws); // First we get all the boxes, up to the given depth; with or wo the slice // function std::vector<API::IMDNode *> boxes; if (this->slice) ws->getBox()->getBoxes(boxes, m_maxDepth, true, this->sliceImplicitFunction.get()); else ws->getBox()->getBoxes(boxes, m_maxDepth, true); vtkIdType numBoxes = boxes.size(); vtkIdType imageSizeActual = 0; if (VERBOSE) std::cout << tim << " to retrieve the " << numBoxes << " boxes down to depth " << m_maxDepth << '\n'; // Create 8 points per box. vtkNew<vtkPoints> points; vtkFloatArray *pointsArray = vtkFloatArray::FastDownCast(points->GetData()); float *pointsPtr = pointsArray->WritePointer(0, numBoxes * 8 * 3); // One scalar per box vtkNew<vtkFloatArray> signals; signals->SetName(ScalarName.c_str()); signals->SetNumberOfComponents(1); float *signalsPtr = signals->WritePointer(0, numBoxes); // To cache the signal std::vector<float> signalCache(numBoxes, 0); // True for boxes that we will use // We do not use vector<bool> here because of the parallelization below // Simultaneous access to different elements of vector<bool> is not safe auto useBox = Mantid::Kernel::make_unique<bool[]>(numBoxes); memset(useBox.get(), 0, sizeof(bool) * numBoxes); // Create the data set (will outlive this object - output of create) auto visualDataSet = vtkSmartPointer<vtkUnstructuredGrid>::New(); this->dataSet = visualDataSet; visualDataSet->Allocate(numBoxes); vtkNew<vtkIdList> hexPointList; hexPointList->SetNumberOfIds(8); auto hexPointList_ptr = hexPointList->WritePointer(0, 8); NormFuncIMDNodePtr normFunction = makeMDEventNormalizationFunction(m_normalizationOption, ws.get()); // This can be parallelized // cppcheck-suppress syntaxError PRAGMA_OMP( parallel for schedule (dynamic) ) for (int ii = 0; ii < int(boxes.size()); ii++) { // Get the box here size_t i = static_cast<size_t>(ii); API::IMDNode *box = boxes[i]; Mantid::signal_t signal_normalized = (box->*normFunction)(); if (std::isfinite(signal_normalized)) { // Cache the signal and using of it signalCache[i] = static_cast<float>(signal_normalized); useBox[i] = true; // Get the coordinates. size_t numVertexes = 0; std::unique_ptr<coord_t[]> coords; // If slicing down to 3D, specify which dimensions to keep. if (this->slice) { coords = std::unique_ptr<coord_t[]>( box->getVertexesArray(numVertexes, 3, this->sliceMask.get())); } else { coords = std::unique_ptr<coord_t[]>(box->getVertexesArray(numVertexes)); } if (numVertexes == 8) { std::copy_n(coords.get(), 24, std::next(pointsPtr, i * 24)); } } else { useBox[i] = false; } } // For each box if (VERBOSE) std::cout << tim << " to create the necessary points.\n"; // Add points visualDataSet->SetPoints(points.GetPointer()); for (size_t i = 0; i < boxes.size(); i++) { if (useBox[i]) { // The bare point ID vtkIdType pointId = i * 8; // Add signal *signalsPtr = signalCache[i]; std::advance(signalsPtr, 1); const std::array<vtkIdType, 8> idList{{0, 1, 3, 2, 4, 5, 7, 6}}; std::transform( idList.begin(), idList.end(), hexPointList_ptr, std::bind(std::plus<vtkIdType>(), std::placeholders::_1, pointId)); // Add cells visualDataSet->InsertNextCell(VTK_HEXAHEDRON, hexPointList.GetPointer()); imageSizeActual++; } } // for each box. // Shrink to fit signals->Resize(imageSizeActual); signals->Squeeze(); visualDataSet->Squeeze(); // Add scalars visualDataSet->GetCellData()->SetScalars(signals.GetPointer()); // Hedge against empty data sets if (visualDataSet->GetNumberOfPoints() <= 0) { vtkNullUnstructuredGrid nullGrid; visualDataSet = nullGrid.createNullData(); this->dataSet = visualDataSet; } if (VERBOSE) std::cout << tim << " to create " << imageSizeActual << " hexahedrons.\n"; }
void CentroidPeaksMD::integrate(typename MDEventWorkspace<MDE, nd>::sptr ws) { if (nd != 3) throw std::invalid_argument("For now, we expect the input MDEventWorkspace " "to have 3 dimensions only."); /// Peak workspace to centroid Mantid::DataObjects::PeaksWorkspace_sptr inPeakWS = getProperty("PeaksWorkspace"); /// Output peaks workspace, create if needed Mantid::DataObjects::PeaksWorkspace_sptr peakWS = getProperty("OutputWorkspace"); if (peakWS != inPeakWS) peakWS.reset(inPeakWS->clone().release()); std::string CoordinatesToUseStr = getPropertyValue("CoordinatesToUse"); int CoordinatesToUse = ws->getSpecialCoordinateSystem(); if (CoordinatesToUse == 1 && CoordinatesToUseStr != "Q (lab frame)") g_log.warning() << "Warning: used Q (lab frame) coordinates for MD " "workspace, not CoordinatesToUse from input " << std::endl; else if (CoordinatesToUse == 2 && CoordinatesToUseStr != "Q (sample frame)") g_log.warning() << "Warning: used Q (sample frame) coordinates for MD " "workspace, not CoordinatesToUse from input " << std::endl; else if (CoordinatesToUse == 3 && CoordinatesToUseStr != "HKL") g_log.warning() << "Warning: used HKL coordinates for MD workspace, not " "CoordinatesToUse from input " << std::endl; /// Radius to use around peaks double PeakRadius = getProperty("PeakRadius"); // cppcheck-suppress syntaxError PRAGMA_OMP(parallel for schedule(dynamic, 10) ) for (int i = 0; i < int(peakWS->getNumberPeaks()); ++i) { // Get a direct ref to that peak. IPeak &p = peakWS->getPeak(i); double detectorDistance = p.getL2(); // Get the peak center as a position in the dimensions of the workspace V3D pos; if (CoordinatesToUse == 1) //"Q (lab frame)" pos = p.getQLabFrame(); else if (CoordinatesToUse == 2) //"Q (sample frame)" pos = p.getQSampleFrame(); else if (CoordinatesToUse == 3) //"HKL" pos = p.getHKL(); // Build the sphere transformation bool dimensionsUsed[nd]; coord_t center[nd]; for (size_t d = 0; d < nd; ++d) { dimensionsUsed[d] = true; // Use all dimensions center[d] = static_cast<coord_t>(pos[d]); } CoordTransformDistance sphere(nd, center, dimensionsUsed); // Initialize the centroid to 0.0 signal_t signal = 0; coord_t centroid[nd]; for (size_t d = 0; d < nd; d++) centroid[d] = 0.0; // Perform centroid ws->getBox()->centroidSphere( sphere, static_cast<coord_t>(PeakRadius * PeakRadius), centroid, signal); // Normalize by signal if (signal != 0.0) { for (size_t d = 0; d < nd; d++) centroid[d] /= static_cast<coord_t>(signal); V3D vecCentroid(centroid[0], centroid[1], centroid[2]); // Save it back in the peak object, in the dimension specified. if (CoordinatesToUse == 1) //"Q (lab frame)" { p.setQLabFrame(vecCentroid, detectorDistance); p.findDetector(); } else if (CoordinatesToUse == 2) //"Q (sample frame)" { p.setQSampleFrame(vecCentroid, detectorDistance); p.findDetector(); } else if (CoordinatesToUse == 3) //"HKL" { p.setHKL(vecCentroid); } g_log.information() << "Peak " << i << " at " << pos << ": signal " << signal << ", centroid " << vecCentroid << " in " << CoordinatesToUse << std::endl; } else { g_log.information() << "Peak " << i << " at " << pos << " had no signal, and could not be centroided." << std::endl; } } // Save the output setProperty("OutputWorkspace", peakWS); }
void CentroidPeaksMD::integrate(typename MDEventWorkspace<MDE, nd>::sptr ws) { if (nd != 3) throw std::invalid_argument("For now, we expect the input MDEventWorkspace to have 3 dimensions only."); /// Peak workspace to centroid Mantid::DataObjects::PeaksWorkspace_sptr inPeakWS = getProperty("PeaksWorkspace"); /// Output peaks workspace, create if needed Mantid::DataObjects::PeaksWorkspace_sptr peakWS = getProperty("OutputWorkspace"); if (peakWS != inPeakWS) peakWS = inPeakWS->clone(); /// Value of the CoordinatesToUse property. std::string CoordinatesToUse = getPropertyValue("CoordinatesToUse"); // TODO: Confirm that the coordinates requested match those in the MDEventWorkspace /// Radius to use around peaks double PeakRadius = getProperty("PeakRadius"); PRAGMA_OMP(parallel for schedule(dynamic, 10) ) for (int i=0; i < int(peakWS->getNumberPeaks()); ++i) { // Get a direct ref to that peak. IPeak & p = peakWS->getPeak(i); double detectorDistance = p.getL2(); // Get the peak center as a position in the dimensions of the workspace V3D pos; if (CoordinatesToUse == "Q (lab frame)") pos = p.getQLabFrame(); else if (CoordinatesToUse == "Q (sample frame)") pos = p.getQSampleFrame(); else if (CoordinatesToUse == "HKL") pos = p.getHKL(); // Build the sphere transformation bool dimensionsUsed[nd]; coord_t center[nd]; for (size_t d=0; d<nd; ++d) { dimensionsUsed[d] = true; // Use all dimensions center[d] = pos[d]; } CoordTransformDistance sphere(nd, center, dimensionsUsed); // Initialize the centroid to 0.0 signal_t signal = 0; coord_t centroid[nd]; for (size_t d=0; d<nd; d++) centroid[d] = 0.0; // Perform centroid ws->getBox()->centroidSphere(sphere, PeakRadius*PeakRadius, centroid, signal); // Normalize by signal if (signal != 0.0) { for (size_t d=0; d<nd; d++) centroid[d] /= signal; V3D vecCentroid(centroid[0], centroid[1], centroid[2]); // Save it back in the peak object, in the dimension specified. if (CoordinatesToUse == "Q (lab frame)") { p.setQLabFrame( vecCentroid, detectorDistance); p.findDetector(); } else if (CoordinatesToUse == "Q (sample frame)") { p.setQSampleFrame( vecCentroid, detectorDistance); p.findDetector(); } else if (CoordinatesToUse == "HKL") { p.setHKL( vecCentroid ); } g_log.information() << "Peak " << i << " at " << pos << ": signal " << signal << ", centroid " << vecCentroid << " in " << CoordinatesToUse << std::endl; } else { g_log.information() << "Peak " << i << " at " << pos << " had no signal, and could not be centroided." << std::endl; } } // Save the output setProperty("OutputWorkspace", peakWS); }