Ejemplo n.º 1
0
//-----------------------------------------------------------------------------
void
ZoltanPartition::compute_partition_rcb(const MPI_Comm mpi_comm,
                                       std::vector<std::size_t>& cell_partition,
                                       const LocalMeshData& mesh_data)
{
  Timer timer0("Partition graph (calling Zoltan RCB)");

  // Get number of local graph vertices
  const std::size_t nlocal = mesh_data.cell_vertices.shape()[0];

  // Initialise Zoltan
  float version;
  int argc = 0;
  char** argv = NULL;
  Zoltan_Initialize(argc, argv, &version);

  // Create Zoltan object
  Zoltan zoltan;

  // Set Zoltan parameters
  zoltan.Set_Param("NUM_GID_ENTRIES", "1");
  zoltan.Set_Param("NUM_LID_ENTRIES", "0");

  zoltan.Set_Param("NUM_GLOBAL_PARTS",
                   std::to_string(MPI::size(mpi_comm)));

  zoltan.Set_Param("NUM_LOCAL_PARTS", "1");
  zoltan.Set_Param("LB_METHOD", "RCB");

  // Set call-back functions
  void *mesh_data_ptr = (void *)&mesh_data;

  zoltan.Set_Num_Obj_Fn(get_number_of_objects, mesh_data_ptr);
  zoltan.Set_Obj_List_Fn(get_object_list, mesh_data_ptr);
  zoltan.Set_Num_Geom_Fn(get_geom, mesh_data_ptr);
  zoltan.Set_Geom_Multi_Fn(get_all_geom, mesh_data_ptr);

  // Call Zoltan function to compute partitions
  int changes = 0;
  int num_gids = 0;
  int num_lids = 0;
  int num_import, num_export;
  ZOLTAN_ID_PTR import_lids;
  ZOLTAN_ID_PTR export_lids;
  ZOLTAN_ID_PTR import_gids;
  ZOLTAN_ID_PTR export_gids;
  int* import_procs;
  int* export_procs;
  int* import_parts;
  int* export_parts;

  int rc = zoltan.LB_Partition(changes, num_gids, num_lids,
           num_import, import_gids, import_lids, import_procs, import_parts,
           num_export, export_gids, export_lids, export_procs, export_parts);

  dolfin_assert(num_gids == 1);
  dolfin_assert(num_lids == 0);


  // Get my process rank
  const std::size_t my_rank = MPI::rank(mpi_comm);

  if (rc != ZOLTAN_OK)
  {
    dolfin_error("ZoltanPartition.cpp",
                 "partition mesh using Zoltan",
                 "Call to Zoltan failed");
  }

  // Assign all nodes to this processor
  cell_partition.assign(nlocal, my_rank);
  std::size_t offset = MPI::global_offset(mpi_comm, nlocal, true);

  // Change nodes to be exported to the appropriate remote processor
  for(int i = 0; i < num_export; ++i)
  {
    const std::size_t idx = export_gids[i] - offset;
    cell_partition[idx] = export_procs[i];
  }

  // Free data structures allocated by Zoltan::LB_Partition
  zoltan.LB_Free_Part(&import_gids, &import_lids, &import_procs, &import_parts);
  zoltan.LB_Free_Part(&export_gids, &export_lids, &export_procs, &export_parts);
}
Ejemplo n.º 2
0
int run(
  const RCP<const Comm<int> > &comm,
  int numGlobalParts,
  int testCnt,
  std::string *thisTest
)
{
#ifdef HAVE_ZOLTAN2_MPI
  // Zoltan needs an MPI comm
  const Teuchos::MpiComm<int> *tmpicomm =
               dynamic_cast<const Teuchos::MpiComm<int> *>(comm.getRawPtr());
  MPI_Comm mpiComm = *(tmpicomm->getRawMpiComm());
#endif

  int me = comm->getRank();
  int np = comm->getSize();
  double tolerance = 1.05;

  //////////////////////////////////////////////
  // Read test data from Zoltan's test directory
  //////////////////////////////////////////////

  UserInputForTests *uinput;
  try{
    uinput = new UserInputForTests(zoltanTestDirectory,
                                   thisTest[TESTNAMEOFFSET],
                                   comm, true);
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: UserInputForTests "
           << e.what() << endl;
    return 1;
  }

  RCP<tMatrix_t> matrix;
  try{
    matrix = uinput->getUITpetraCrsMatrix();
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: get matrix "
           << e.what() << endl;
    return 1;
  }

  RCP<const tMatrix_t> matrixConst = rcp_const_cast<const tMatrix_t>(matrix);

  RCP<tMVector_t> coords;
  try{
   coords = uinput->getUICoordinates();
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: get coordinates "
           << e.what() << endl;
    return 1;
  }

  RCP<tMVector_t> weights;
  try{
   weights = uinput->getUIWeights();
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: get weights "
           << e.what() << endl;
    return 1;
  }
  int nWeights = atoi(thisTest[TESTOBJWGTOFFSET].c_str());

  if (me == 0) {
    cout << "Test " << testCnt << " filename            = "
         << thisTest[TESTNAMEOFFSET] << endl;
    cout << "Test " << testCnt << " num processors      = "
         << np << endl;
    cout << "Test " << testCnt << " zoltan method       = "
         << thisTest[TESTMETHODOFFSET] << endl;
    cout << "Test " << testCnt << " num_global_parts    = "
         << numGlobalParts << endl;
    cout << "Test " << testCnt << " imbalance_tolerance = "
         << tolerance << endl;
    cout << "Test " << testCnt << " num weights per ID  = "
         << nWeights << endl;
  }

  /////////////////////////////////////////
  // PARTITION USING ZOLTAN DIRECTLY
  /////////////////////////////////////////

  if (me == 0) cout << "Calling Zoltan directly" << endl;

# ifdef HAVE_ZOLTAN2_MPI
    Zoltan zz(mpiComm);
# else
    Zoltan zz;
# endif

  char tmp[56];
  zz.Set_Param("LB_METHOD", thisTest[TESTMETHODOFFSET]);
  
  sprintf(tmp, "%d", numGlobalParts);
  zz.Set_Param("NUM_GLOBAL_PARTS", tmp);
  sprintf(tmp, "%d", nWeights);
  zz.Set_Param("OBJ_WEIGHT_DIM", tmp);
  sprintf(tmp, "%f", tolerance);
  zz.Set_Param("IMBALANCE_TOL", tmp);
  zz.Set_Param("RETURN_LISTS", "PART");
  zz.Set_Param("FINAL_OUTPUT", "1");

  zz.Set_Num_Obj_Fn(znumobj, (void *) coords.getRawPtr());
  if (nWeights)
    zz.Set_Obj_List_Fn(zobjlist, (void *) weights.getRawPtr());
  else
    zz.Set_Obj_List_Fn(zobjlist, (void *) coords.getRawPtr());
  zz.Set_Num_Geom_Fn(znumgeom, (void *) coords.getRawPtr());
  zz.Set_Geom_Multi_Fn(zgeom, (void *) coords.getRawPtr());

  int changes, ngid, nlid;
  int numd, nump;
  ZOLTAN_ID_PTR dgid = NULL, dlid = NULL, pgid = NULL, plid = NULL;
  int *dproc = NULL, *dpart = NULL, *pproc = NULL, *ppart = NULL;

  int ierr = zz.LB_Partition(changes, ngid, nlid,
                             numd, dgid, dlid, dproc, dpart,
                             nump, pgid, plid, pproc, ppart);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: direct Zoltan call" << endl;
    zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
    return 1;
  }

  /////////////////////////////////////////
  // PARTITION USING ZOLTAN THROUGH ZOLTAN2
  /////////////////////////////////////////

  if (me == 0) cout << "Calling Zoltan through Zoltan2" << endl;

  matrixAdapter_t *ia;
  try{
    ia = new matrixAdapter_t(matrixConst, nWeights);
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: matrix adapter "
           << e.what() << endl;
    return 1;
  }
  for (int idx=0; idx < nWeights; idx++)
    ia->setRowWeights(weights->getData(idx).getRawPtr(), 1, idx);

  vectorAdapter_t *ca = NULL;
  try{
    ca = new vectorAdapter_t(coords);
  }
  catch(std::exception &e){
    if (me == 0)
      cout << "Test " << testCnt << ":  FAIL: vector adapter "
           << e.what() << endl;
    return 1;
  }
  ia->setCoordinateInput(ca);
  
  Teuchos::ParameterList params;
  params.set("timer_output_stream" , "std::cout");
  params.set("compute_metrics", "true");
  // params.set("debug_level" , "verbose_detailed_status");

  params.set("algorithm", "zoltan");
  params.set("imbalance_tolerance", tolerance );
  params.set("num_global_parts", numGlobalParts);

  if (thisTest[TESTMETHODOFFSET] != "default") {
    // "default" tests case of no Zoltan parameter sublist
    Teuchos::ParameterList &zparams = params.sublist("zoltan_parameters",false);
    zparams.set("LB_METHOD",thisTest[TESTMETHODOFFSET]);
  }

  Zoltan2::PartitioningProblem<matrixAdapter_t> *problem;
# ifdef HAVE_ZOLTAN2_MPI
    try{
      problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, &params,
                                                                  mpiComm);
    }
# else
    try{
      problem = new Zoltan2::PartitioningProblem<matrixAdapter_t>(ia, &params);
    }
# endif
  catch(std::exception &e){
    cout << "Test " << testCnt << " FAIL: problem " << e.what() << endl;
    return 1;
  }

  try {
    problem->solve();
  }
  catch(std::exception &e){
    cout << "Test " << testCnt << " FAIL: solve " << e.what() << endl;
    return 1;
  }

  if (me == 0){
    problem->printMetrics(cout);
  }
  problem->printTimers();

  /////////////////////////////////////////
  // COMPARE RESULTS
  /////////////////////////////////////////
  size_t nObj = coords->getLocalLength();
  const int *z2parts = problem->getSolution().getPartListView();
  int diffcnt = 0, gdiffcnt = 0;
  for (size_t i = 0; i < nObj; i++) {
    if (z2parts[plid[i]] != ppart[i]) {
      diffcnt++;
      cout << me << " DIFF for " << i << " (" 
           << coords->getMap()->getGlobalElement(i) << "):  "
           << "Z2 = " << z2parts[i] << "; Z1 = " << ppart[plid[i]] << endl;
    }
  }

  /////////////////////////////////////////
  // CLEAN UP
  /////////////////////////////////////////
  zz.LB_Free_Part(&pgid, &plid, &pproc, &ppart);
  delete ia;
  delete ca;
  delete problem;
  delete uinput;

  Teuchos::reduceAll(*comm, Teuchos::REDUCE_SUM, 1, &diffcnt, &gdiffcnt);
  if (gdiffcnt > 0) {
    if (me == 0) 
      cout << "Test " << testCnt << " "
           << thisTest[TESTNAMEOFFSET] << " "
           << thisTest[TESTMETHODOFFSET] << " "
           << thisTest[TESTOBJWGTOFFSET] << " "
           << " FAIL: comparison " << endl;
    return 1;
  }

  return 0;
}