Пример #1
0
  void MinusMD::doMinus(typename MDEventWorkspace<MDE, nd>::sptr ws)
  {
    typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
    typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
    if (!ws1 || !ws2)
      throw std::runtime_error("Incompatible workspace types passed to MinusMD.");

    MDBoxBase<MDE,nd> * box1 = ws1->getBox();
    MDBoxBase<MDE,nd> * box2 = ws2->getBox();

    Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());

    // How many events you started with
    size_t initial_numEvents = ws1->getNPoints();

    // Make a leaf-only iterator through all boxes with events in the RHS workspace
    MDBoxIterator<MDE,nd> it2(box2, 1000, true);
    do
    {
      MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
      if (box)
      {
        // Copy the events from WS2 and add them into WS1
        const std::vector<MDE> & events = box->getConstEvents();

        // Perform a copy while flipping the signal
        std::vector<MDE> eventsCopy;
        eventsCopy.reserve(events.size());
        for (auto it = events.begin(); it != events.end(); it++)
        {
          MDE eventCopy(*it);
          eventCopy.setSignal( -eventCopy.getSignal());
          eventsCopy.push_back(eventCopy);
        }
        // Add events, with bounds checking
        box1->addEvents(eventsCopy);
        box->releaseEvents();
      }
      prog.report("Adding Events");
    } while (it2.next());

    this->progress(0.41, "Splitting Boxes");
    Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
    ThreadScheduler * ts = new ThreadSchedulerFIFO();
    ThreadPool tp(ts, 0, prog2);
    ws1->splitAllIfNeeded(ts);
    prog2->resetNumSteps( ts->size(), 0.4, 0.6);
    tp.joinAll();


    this->progress(0.95, "Refreshing cache");
    ws1->refreshCache();

    // Set a marker that the file-back-end needs updating if the # of events changed.
    if (ws1->getNPoints() != initial_numEvents)
      ws1->setFileNeedsUpdating(true);
  }
Пример #2
0
  void PlusMD::doPlus(typename MDEventWorkspace<MDE, nd>::sptr ws)
  {
    typename MDEventWorkspace<MDE, nd>::sptr ws1 = ws;
    typename MDEventWorkspace<MDE, nd>::sptr ws2 = boost::dynamic_pointer_cast<MDEventWorkspace<MDE, nd> >(m_operand_event);
    if (!ws1 || !ws2)
      throw std::runtime_error("Incompatible workspace types passed to PlusMD.");

    MDBoxBase<MDE,nd> * box1 = ws1->getBox();
    MDBoxBase<MDE,nd> * box2 = ws2->getBox();

    Progress prog(this, 0.0, 0.4, box2->getBoxController()->getTotalNumMDBoxes());

    // How many events you started with
    size_t initial_numEvents = ws1->getNPoints();

    // Make a leaf-only iterator through all boxes with events in the RHS workspace
    MDBoxIterator<MDE,nd> it2(box2, 1000, true);
    do
    {
      MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it2.getBox());
      if (box)
      {
        // Copy the events from WS2 and add them into WS1
        const std::vector<MDE> & events = box->getConstEvents();
        // Add events, with bounds checking
        box1->addEvents(events);
        box->releaseEvents();
      }
      prog.report("Adding Events");
    } while (it2.next());

    this->progress(0.41, "Splitting Boxes");
    Progress * prog2 = new Progress(this, 0.4, 0.9, 100);
    ThreadScheduler * ts = new ThreadSchedulerFIFO();
    ThreadPool tp(ts, 0, prog2);
    ws1->splitAllIfNeeded(ts);
    prog2->resetNumSteps( ts->size(), 0.4, 0.6);
    tp.joinAll();

//    // Now we need to save all the data that was not saved before.
//    if (ws1->isFileBacked())
//    {
//      // Flush anything else in the to-write buffer
//      BoxController_sptr bc = ws1->getBoxController();
//
//      prog.resetNumSteps(bc->getTotalNumMDBoxes(), 0.6, 1.0);
//      MDBoxIterator<MDE,nd> it1(box1, 1000, true);
//      while (true)
//      {
//        MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(it1.getBox());
//        if (box)
//        {
//          // Something was maybe added to this box
//          if (box->getEventVectorSize() > 0)
//          {
//            // By getting the events, this will merge the newly added and the cached events.
//            box->getEvents();
//            // The MRU to-write cache will optimize writes by reducing seek times
//            box->releaseEvents();
//          }
//        }
//        prog.report("Saving");
//        if (!it1.next()) break;
//      }
//      //bc->getDiskBuffer().flushCache();
//      // Flush the data writes to disk.
//      box1->flushData();
//    }

    this->progress(0.95, "Refreshing cache");
    ws1->refreshCache();

    // Set a marker that the file-back-end needs updating if the # of events changed.
    if (ws1->getNPoints() != initial_numEvents)
      ws1->setFileNeedsUpdating(true);

  }
Пример #3
0
void LoadMD::doLoad(typename MDEventWorkspace<MDE, nd>::sptr ws) {
  // Are we using the file back end?
  bool fileBackEnd = getProperty("FileBackEnd");

  if (fileBackEnd && m_BoxStructureAndMethadata)
    throw std::invalid_argument("Combination of BoxStructureOnly or "
                                "MetaDataOnly were set to TRUE with "
                                "fileBackEnd "
                                ": this is not possible.");

  CPUTimer tim;
  auto prog = new Progress(this, 0.0, 1.0, 100);

  prog->report("Opening file.");
  std::string title;
  try {
    m_file->getAttr("title", title);
  } catch (std::exception &) {
    // Leave the title blank if error on loading
  }
  ws->setTitle(title);

  // Load the WorkspaceHistory "process"
  if (this->getProperty("LoadHistory")) {
    ws->history().loadNexus(m_file.get());
  }

  this->loadAffineMatricies(boost::dynamic_pointer_cast<IMDWorkspace>(ws));

  m_file->closeGroup();
  m_file->close();
  // Add each of the dimension
  for (size_t d = 0; d < nd; d++)
    ws->addDimension(m_dims[d]);

  // Coordinate system
  ws->setCoordinateSystem(m_coordSystem);

  // ----------------------------------------- Box Structure
  // ------------------------------
  prog->report("Reading box structure from HDD.");
  MDBoxFlatTree FlatBoxTree;
  int nDims = static_cast<int>(nd); // should be safe
  FlatBoxTree.loadBoxStructure(m_filename, nDims, MDE::getTypeName());

  BoxController_sptr bc = ws->getBoxController();
  bc->fromXMLString(FlatBoxTree.getBCXMLdescr());

  prog->report("Restoring box structure and connectivity");
  std::vector<API::IMDNode *> boxTree;
  FlatBoxTree.restoreBoxTree(boxTree, bc, fileBackEnd,
                             m_BoxStructureAndMethadata);
  size_t numBoxes = boxTree.size();

  // ---------------------------------------- DEAL WITH BOXES
  // ------------------------------------
  if (fileBackEnd) { // TODO:: call to the file format factory
    auto loader = boost::shared_ptr<API::IBoxControllerIO>(
        new DataObjects::BoxControllerNeXusIO(bc.get()));
    loader->setDataType(sizeof(coord_t), MDE::getTypeName());
    bc->setFileBacked(loader, m_filename);
    // boxes have been already made file-backed when restoring the boxTree;
    // How much memory for the cache?
    {
      // TODO: Clean up, only a write buffer now
      double mb = getProperty("Memory");

      // Defaults have changed, default disk buffer size should be 10 data
      // chunks TODO: find optimal, 100 may be better.
      if (mb <= 0)
        mb = double(10 * loader->getDataChunk() * sizeof(MDE)) /
             double(1024 * 1024);

      // Express the cache memory in units of number of events.
      uint64_t cacheMemory =
          static_cast<uint64_t>((mb * 1024. * 1024.) / sizeof(MDE)) + 1;

      // Set these values in the diskMRU
      bc->getFileIO()->setWriteBufferSize(cacheMemory);

      g_log.information() << "Setting a DiskBuffer cache size of " << mb
                          << " MB, or " << cacheMemory << " events.\n";
    }
  } // Not file back end
  else if (!m_BoxStructureAndMethadata) {
    // ---------------------------------------- READ IN THE BOXES
    // ------------------------------------
    // TODO:: call to the file format factory
    auto loader =
        file_holder_type(new DataObjects::BoxControllerNeXusIO(bc.get()));
    loader->setDataType(sizeof(coord_t), MDE::getTypeName());

    loader->openFile(m_filename, "r");

    const std::vector<uint64_t> &BoxEventIndex = FlatBoxTree.getEventIndex();
    prog->setNumSteps(numBoxes);

    for (size_t i = 0; i < numBoxes; i++) {
      prog->report();
      MDBox<MDE, nd> *box = dynamic_cast<MDBox<MDE, nd> *>(boxTree[i]);
      if (!box)
        continue;

      if (BoxEventIndex[2 * i + 1] >
          0) // Load in memory NOT using the file as the back-end,
      {
        boxTree[i]->reserveMemoryForLoad(BoxEventIndex[2 * i + 1]);
        boxTree[i]->loadAndAddFrom(
            loader.get(), BoxEventIndex[2 * i],
            static_cast<size_t>(BoxEventIndex[2 * i + 1]));
      }
    }
    loader->closeFile();
  } else // box structure and metadata only
  {
  }
  g_log.debug() << tim
                << " to create all the boxes and fill them with events.\n";

  // Box of ID 0 is the head box.
  ws->setBox(boxTree[0]);
  // Make sure the max ID is ok for later ID generation
  bc->setMaxId(numBoxes);

  // end-of bMetaDataOnly
  // Refresh cache
  // TODO:if(!fileBackEnd)ws->refreshCache();
  ws->refreshCache();
  g_log.debug() << tim << " to refreshCache(). " << ws->getNPoints()
                << " points after refresh.\n";

  g_log.debug() << tim << " to finish up.\n";
  delete prog;
}
Пример #4
0
  void vtkSplatterPlotFactory::doCreate(typename MDEventWorkspace<MDE, nd>::sptr ws) const
  {
    bool VERBOSE = true;
    CPUTimer tim;
    // Acquire a scoped read-only lock to the workspace (prevent segfault
    // from algos modifying ws)
    ReadLock lock(*ws);

    // Find out how many events to plot, and the percentage of the largest
    // boxes to use.
    size_t totalPoints = ws->getNPoints();
    size_t numPoints = m_numPoints;

    if (numPoints > totalPoints)
    {
      numPoints = totalPoints;
    }

    double percent_to_use = m_percentToUse;
    // Fail safe limits on fraction of boxes to use
    if (percent_to_use <= 0)
    {
      percent_to_use = 5;
    }

    if (percent_to_use > 100)
    {
      percent_to_use = 100;
    }

    // First we get all the boxes, up to the given depth; with or wo the
    // slice function
    std::vector<API::IMDNode *> boxes;
    if (this->slice)
    {
      ws->getBox()->getBoxes(boxes, 1000, true, this->sliceImplicitFunction);
    }
    else
    {
      ws->getBox()->getBoxes(boxes, 1000, true);
    }

    if (VERBOSE)
    {
      std::cout << tim << " to retrieve the "<< boxes.size() << " boxes down."<< std::endl;
    }

    std::string new_name = ws->getName();
    if (new_name != m_wsName || m_buildSortedList)
    {
      m_wsName = new_name;
      m_buildSortedList = false;
      m_sortedBoxes.clear();
      // get list of boxes with signal > 0 and sort
      // the list in order of decreasing signal
      for (size_t i = 0; i < boxes.size(); i++)
      {
        MDBox<MDE,nd> * box = dynamic_cast<MDBox<MDE,nd> *>(boxes[i]);
        if (box)
        {
          size_t newPoints = box->getNPoints();
          if (newPoints > 0)
          {
            m_sortedBoxes.push_back(box);
          }
        }
      }

      if (VERBOSE)
      {
        std::cout << "START SORTING" << std::endl;
      }
      std::sort(m_sortedBoxes.begin(), m_sortedBoxes.end(),
                CompareNormalizedSignal);
      if (VERBOSE)
      {
        std::cout << "DONE SORTING" << std::endl;
      }
    }
    size_t num_boxes_to_use = static_cast<size_t>(percent_to_use * static_cast<double>(m_sortedBoxes.size()) / 100.0);
    if (num_boxes_to_use >= m_sortedBoxes.size())
    {
      num_boxes_to_use = m_sortedBoxes.size()-1;
    }

    // restrict the number of points to the
    // number of points in boxes being used
    size_t total_points_available = 0;
    for (size_t i = 0; i < num_boxes_to_use; i++)
    {
      size_t newPoints = m_sortedBoxes[i]->getNPoints();
      total_points_available += newPoints;
    }

    if (numPoints > total_points_available)
    {
      numPoints = total_points_available;
    }

    size_t points_per_box = 0;
    // calculate the average number of points to use per box
    if (num_boxes_to_use > 0)
    {
      points_per_box = numPoints / num_boxes_to_use;
    }

    if (points_per_box < 1)
    {
      points_per_box = 1;
    }

    if (VERBOSE)
    {
      std::cout << "numPoints                 = " << numPoints << std::endl;
      std::cout << "num boxes in all          = " << boxes.size() << std::endl;
      std::cout << "num boxes above zero      = " << m_sortedBoxes.size() << std::endl;
      std::cout << "num boxes to use          = " << num_boxes_to_use << std::endl;
      std::cout << "total_points_available    = " << total_points_available << std::endl;
      std::cout << "points needed per box     = " << points_per_box << std::endl;
    }

    // First save the events and signals that we actually use.
    // For each box, get up to the average number of points
    // we want from each box, limited by the number of points
    // in the box.  NOTE: since boxes have different numbers
    // of events, we will not get all the events requested.
    // Also, if we are using a smaller number of points, we
    // won't get points from some of the boxes with lower signal.

    std::vector<float>            saved_signals;
    std::vector<const coord_t*>   saved_centers;
    std::vector<size_t>           saved_n_points_in_cell;
    saved_signals.reserve(numPoints);
    saved_centers.reserve(numPoints);
    saved_n_points_in_cell.reserve(numPoints);
  
    size_t pointIndex = 0;
    size_t box_index  = 0;
    bool   done       = false;
    while (box_index < num_boxes_to_use && !done)
    {
      MDBox<MDE,nd> *box = dynamic_cast<MDBox<MDE,nd> *>(m_sortedBoxes[box_index]);
      box_index++;
      if (NULL == box)
      {
        continue;
      }
      float signal_normalized = static_cast<float>(box->getSignalNormalized());
      size_t newPoints = box->getNPoints();
      size_t num_from_this_box = points_per_box;
      if (num_from_this_box > newPoints)
      {
        num_from_this_box = newPoints;
      }
      const std::vector<MDE> & events = box->getConstEvents();
      size_t startPointIndex = pointIndex;
      size_t event_index = 0;
      while (event_index < num_from_this_box && !done)
      {
        const MDE & ev = events[event_index];
        event_index++;
        const coord_t * center = ev.getCenter();
        // Save location
        saved_centers.push_back(center);
        pointIndex++;
        if (pointIndex >= numPoints)
        {
          done = true;
        }
      }
      box->releaseEvents();
      // Save signal
      saved_signals.push_back(signal_normalized);
      // Save cell size
      saved_n_points_in_cell.push_back(pointIndex-startPointIndex);
    } 

    numPoints = saved_centers.size();
    size_t numCells = saved_signals.size();

    if (VERBOSE)
    {
      std::cout << "Recorded data for all points" << std::endl;
      std::cout << "numPoints = " << numPoints << std::endl;
      std::cout << "numCells  = " << numCells << std::endl;
    }

    // Create the point list, one position for each point actually used
    vtkPoints *points = vtkPoints::New();
    points->Allocate(numPoints);
    points->SetNumberOfPoints(numPoints);

    // The list of IDs of points used, one ID per point, since points
    // are not reused to form polygon facets, etc.
    vtkIdType *ids = new vtkIdType[numPoints];

    // Only one scalar for each cell, NOT one per point
    vtkFloatArray *signal = vtkFloatArray::New();
    signal->Allocate(numCells);
    signal->SetName(m_scalarName.c_str());

    // Create the data set.  Need space for each cell, not for each point
    vtkUnstructuredGrid *visualDataSet = vtkUnstructuredGrid::New();
    this->dataSet = visualDataSet;
    visualDataSet->Allocate(numCells);
    // Now copy the saved point, cell and signal info into vtk data structures
    pointIndex = 0;
    for (size_t cell_i = 0; cell_i < numCells; cell_i++)
    {
      size_t startPointIndex = pointIndex;
      for (size_t point_i = 0; point_i < saved_n_points_in_cell[cell_i]; point_i++)
      {
        points->SetPoint(pointIndex, saved_centers[pointIndex]);
        ids[pointIndex] = pointIndex;
        pointIndex++;
      }
      signal->InsertNextTuple1(saved_signals[cell_i]);
      visualDataSet->InsertNextCell(VTK_POLY_VERTEX, saved_n_points_in_cell[cell_i], ids+startPointIndex);
    }

    if (VERBOSE)
    {
      std::cout << tim << " to create " << pointIndex << " points." << std::endl;
    }

    // Shrink to fit
    //points->Squeeze();
    signal->Squeeze();
    visualDataSet->Squeeze();

    // Add points and scalars
    visualDataSet->SetPoints(points);
    visualDataSet->GetCellData()->SetScalars(signal);

    delete [] ids;
  }