Ejemplo n.º 1
0
/*
 * void Zoltan_Oct_fix_tags()
 *
 * fixes the import tags so that region tags that were previously
 * exported aren't counted when imported back.
 */
int Zoltan_Oct_fix_tags(ZZ *zz, ZOLTAN_ID_PTR *import_global_ids, 
			ZOLTAN_ID_PTR *import_local_ids, int **import_procs, 
			int **import_to_part, int nrectags, 
			pRegion import_regs) {
  char *yo = "Zoltan_Oct_fix_tags";
  int i;                                  /* index counter */
  int ierr = ZOLTAN_OK;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;

    /* allocate memory */

    if (!Zoltan_Special_Malloc(zz,(void **)import_global_ids,nrectags,
                           ZOLTAN_SPECIAL_MALLOC_GID)) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)import_local_ids,nrectags,
                           ZOLTAN_SPECIAL_MALLOC_LID)) {
      Zoltan_Special_Free(zz,(void **)import_global_ids,
			  ZOLTAN_SPECIAL_MALLOC_GID); 
      ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)import_procs,nrectags,
                           ZOLTAN_SPECIAL_MALLOC_INT)) {
      Zoltan_Special_Free(zz,(void **)import_global_ids,
			  ZOLTAN_SPECIAL_MALLOC_GID);
      Zoltan_Special_Free(zz,(void **)import_local_ids,
			  ZOLTAN_SPECIAL_MALLOC_LID);
      ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)import_to_part,nrectags,
                           ZOLTAN_SPECIAL_MALLOC_INT)) {
      Zoltan_Special_Free(zz,(void **)import_global_ids,
			  ZOLTAN_SPECIAL_MALLOC_GID);
      Zoltan_Special_Free(zz,(void **)import_local_ids,
			  ZOLTAN_SPECIAL_MALLOC_LID);
      Zoltan_Special_Free(zz,(void **)import_procs,ZOLTAN_SPECIAL_MALLOC_INT);
      ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }

    /* for each region imported, look at its originating processor */
    for(i=0; i<nrectags; i++) {
      ZOLTAN_SET_GID(zz, &((*import_global_ids)[i*num_gid_entries]),
                 import_regs[i].Global_ID);
      ZOLTAN_SET_LID(zz, &((*import_local_ids)[i*num_lid_entries]),
                 import_regs[i].Local_ID);
      (*import_procs)[i]   = import_regs[i].newProc;
      /*(*import_to_part)[i] = zz->Proc;*/
      (*import_to_part)[i] = import_regs[i].newProc;
    }

    return ierr;
}
Ejemplo n.º 2
0
int Zoltan_PHG(
ZZ *zz,                    /* The Zoltan structure  */
float *part_sizes,         /* Input:  Array of size zz->Num_Global_Parts 
                                containing the percentage of work assigned 
                                to each partition. */
int *num_imp,              /* not computed */
ZOLTAN_ID_PTR *imp_gids,   /* not computed */
ZOLTAN_ID_PTR *imp_lids,   /* not computed */
int **imp_procs,           /* not computed */
int **imp_to_part,         /* not computed */
int *num_exp,              /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids,   /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids,   /* local  ids of objects to be exported */
int **exp_procs,           /* list of processors to export to */
int **exp_to_part )         /* list of partitions to which exported objs
                                are assigned. */
{
  char *yo = "Zoltan_PHG";
  ZHG *zoltan_hg = NULL;
  PHGPartParams hgp;               /* Hypergraph parameters. */
  HGraph *hg = NULL;               /* Hypergraph itself */
  Partition parts = NULL;          /* Partition assignments in 
                                      2D distribution. */
  int err = ZOLTAN_OK, p=0;
  struct phg_timer_indices *timer = NULL; 
  int do_timing = 0;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Initialization of return arguments. */
  *num_imp   = *num_exp   = -1;
  *imp_gids  = *exp_gids  = NULL;
  *imp_lids  = *exp_lids  = NULL;
  *imp_procs = *exp_procs = NULL;
  
  /* Initialize HG parameters. */
  err = Zoltan_PHG_Initialize_Params(zz, part_sizes, &hgp);
  if (err != ZOLTAN_OK)
    goto End;

  if (hgp.use_timers) {
    if (!zz->LB.Data_Structure)  {
      zz->LB.Data_Structure = (struct phg_timer_indices *) 
                               ZOLTAN_MALLOC(sizeof(struct phg_timer_indices));
      initialize_timer_indices((struct phg_timer_indices *)zz->LB.Data_Structure);
    }
    timer = zz->LB.Data_Structure;
    if (timer->all < 0) 
      timer->all = Zoltan_Timer_Init(zz->ZTime, 1, "Zoltan_PHG");
  }

  if (hgp.use_timers > 1) {
    do_timing = 1;
    if (timer->build < 0) 
      timer->build = Zoltan_Timer_Init(zz->ZTime, 1, "Build");
    if (timer->setupvmap < 0) 
      timer->setupvmap = Zoltan_Timer_Init(zz->ZTime, 0, "Vmaps");
  }

  if (hgp.use_timers) 
    ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
    
  if (do_timing)
    ZOLTAN_TIMER_START(zz->ZTime, timer->build, zz->Communicator);
    
  /* build initial Zoltan hypergraph from callback functions. */

  err = Zoltan_PHG_Build_Hypergraph (zz, &zoltan_hg, &parts, &hgp);
  if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
    goto End;
  }

  if (zoltan_hg->GnObj == 0){
    /* degenerate case - no objects to partition */
    hgp.final_output = 0;
    goto End;
  }

  hg = &zoltan_hg->HG;
  p = zz->LB.Num_Global_Parts;  
  zoltan_hg->HG.redl = MAX(hgp.redl, p);     /* redl needs to be dynamic */

  /* RTHRTH -- redl may need to be scaled by number of procs */
  /* EBEB -- at least make sure redl > #procs */

  if (hgp.UseFixedVtx)
      hg->bisec_split = 1; /* this will be used only #parts=2
                              otherwise rdivide will set to appropriate
                              value */

  if (hgp.UsePrefPart || hgp.UseFixedVtx) { /* allocate memory for pref_part */
    if (hg->nVtx &&                       
        !(hg->pref_part = (int*) ZOLTAN_MALLOC (sizeof(int) * hg->nVtx))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph.");
        goto End;
    }
  }
  if (hgp.UsePrefPart) /* copy input parts as pref_part;
                          UVCUVC: TODO: this code (and alloc of pref_part)
                          should go to Build_Hypergraph later */
      memcpy(hg->pref_part, parts, sizeof(int) * hg->nVtx);
  
  if (hgp.UseFixedVtx) {
      int i;
      for (i=0; i<hg->nVtx; ++i)
          if (hg->fixed_part[i]>=0)
              hg->pref_part[i] = hg->fixed_part[i];
          else if (!hgp.UsePrefPart)
              hg->pref_part[i] = -1;      
  }
  hgp.UsePrefPart |= hgp.UseFixedVtx;

  if (do_timing)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->build, zz->Communicator);


/*
  UVCUVC DEBUG PRINT
  uprintf(hg->comm, "Zoltan_PHG kway=%d #parts=%d\n", hgp.kway, zz->LB.Num_Global_Parts);
*/

  if (!strcasecmp(hgp.hgraph_pkg, "PARKWAY")){
    if (do_timing) {
      if (timer->parkway < 0)
        timer->parkway = Zoltan_Timer_Init(zz->ZTime, 0, "PHG_ParKway");
      ZOLTAN_TIMER_START(zz->ZTime, timer->parkway, zz->Communicator);
    }
    err = Zoltan_PHG_ParKway(zz, hg, p,
                             parts, &hgp);
    if (err != ZOLTAN_OK) 
        goto End;
    if (do_timing)
      ZOLTAN_TIMER_STOP(zz->ZTime, timer->parkway, zz->Communicator);
  } else if (!strcasecmp(hgp.hgraph_pkg, "PATOH")){
    if (hgp.use_timers > 1) {
      if (timer->patoh < 0)
        timer->patoh = Zoltan_Timer_Init(zz->ZTime, 0, "HG_PaToH");
      ZOLTAN_TIMER_START(zz->ZTime, timer->patoh, zz->Communicator);
    }
    err = Zoltan_PHG_PaToH(zz, hg, p,
                           parts, &hgp);
    if (err != ZOLTAN_OK) 
      goto End;
    if (hgp.use_timers > 1)
      ZOLTAN_TIMER_STOP(zz->ZTime, timer->patoh, zz->Communicator);
  }      
  else { /* it must be PHG  */
    /* UVC: if it is bisection anyways; no need to create vmap etc; 
       rdivide is going to call Zoltan_PHG_Partition anyways... */
    if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
      /* This processor is part of the 2D data distribution; it should
         participate in partitioning. */

        
      if (hgp.kway || zz->LB.Num_Global_Parts == 2) {
        /* call main V cycle routine */
        err = Zoltan_PHG_Partition(zz, hg, p,
                                   hgp.part_sizes, parts, &hgp);
        if (err != ZOLTAN_OK) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
          goto End;
        }
      }
      else {
        int i;
          
        if (do_timing) 
          ZOLTAN_TIMER_START(zz->ZTime, timer->setupvmap, zz->Communicator);
        /* vmap associates original vertices to sub hypergraphs */
        if (hg->nVtx && 
            !(hg->vmap = (int*) ZOLTAN_MALLOC(hg->nVtx*sizeof (int))))  {
          err = ZOLTAN_MEMERR;
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
          goto End;
        }
        for (i = 0; i < hg->nVtx; ++i)
          hg->vmap[i] = i;
  
        if (do_timing) 
          ZOLTAN_TIMER_STOP(zz->ZTime, timer->setupvmap, zz->Communicator);
  
          
        /* partition hypergraph */
        err = Zoltan_PHG_rdivide (0, p-1, parts, zz, hg, &hgp, 0);
  
        if (hgp.output_level >= PHG_DEBUG_LIST)     
          uprintf(hg->comm, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %s/%s/%s/%s p=%d "
                  "bal=%.2f cutl=%.2f\n", 
                  hg->info, hg->nVtx, hg->nEdge, hg->nPins,
                  hgp.convert_str, hgp.redm_str, 
                  hgp.coarsepartition_str, hgp.refinement_str, p,
                  Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
                                             p, parts),
                  Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
            
        if (err != ZOLTAN_OK)  {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph.");
          goto End;
        }
        ZOLTAN_FREE (&hg->vmap);
      }
#ifdef CHECK_LEFTALONE_VERTICES          
      findAndSaveLeftAloneVertices(zz, hg, p, parts, &hgp);
#endif      
    }
  }

  if (!strcasecmp(hgp.hgraph_method, "REPARTITION")) {
    Zoltan_PHG_Remove_Repart_Data(zz, zoltan_hg, hg, &hgp);
  }


/* UVC DEBUG PRINT
  if (!strcasecmp(hgp->hgraph_method, "REFINE")){
      uprintf(hg->comm, 
              "UVC ATTHEEND |V|=%6d |E|=%6d #pins=%6d p=%d bal=%.2f cutl=%.2f\n",
              hg->nVtx, hg->nEdge, hg->nPins, p,
              Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, parts),
              Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err));
      detailed_balance_info(zz, hg, part_sizes, p, parts);
  }
*/

  
  /* Initialize these timers here so their output is near end of printout */
  if (do_timing)
    if (timer->retlist < 0) 
      timer->retlist = Zoltan_Timer_Init(zz->ZTime, 1, "Return_Lists");

  if (hgp.use_timers)
    if (timer->finaloutput < 0) 
      timer->finaloutput = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output");

  if (do_timing) 
    ZOLTAN_TIMER_START(zz->ZTime, timer->retlist, zz->Communicator);

  /* Build Zoltan's Output_Parts, mapped from 2D distribution 
     to input distribution. */

  Zoltan_PHG_Output_Parts(zz, zoltan_hg, parts);

  /* Build Zoltan's return arguments. */
  Zoltan_PHG_Return_Lists(zz, zoltan_hg, num_exp, exp_gids,
   exp_lids, exp_procs, exp_to_part);
    
  if (do_timing)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->retlist, zz->Communicator);

End:
  if (err == ZOLTAN_MEMERR)
    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Memory error.")
  else if (err != ZOLTAN_OK)
    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Error partitioning hypergraph.")
    
  /* KDDKDD The following code prints a final quality result even when
   * KDDKDD phg_output_level is zero.  It is useful for our tests and
   * KDDKDD data collection. */
  if ((err == ZOLTAN_OK) && hgp.final_output) {
    static int nRuns=0;
    static double balsum = 0.0, cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0;
    static double balmax = 0.0, cutlmax = 0.0, cutnmax = 0.0, movemax = 0.0, repartmax = 0.0;
    static double balmin = 1e100, cutlmin = 1e100, cutnmin = 1e100, movemin = 1e100, repartmin = 1e100;
    double bal = 0.; 
    double cutl = 0.; /* Connnectivity cuts:  sum_over_edges((npart-1)*ewgt) */
    double cutn = 0.; /* Net cuts:  sum_over_edges((nparts>1)*ewgt) */

    double rlocal[2];  /* local cut stats for removed edges */
    double rglobal[2]; /* global cut stats for removed edges */
    int gnremove, i;
    double move=0.0, gmove;  /* local and global migration costs */
    double repart=0.0;   /* total repartitioning cost: comcost x multiplier + migration_cost */

    
    if (hgp.use_timers) {
      /* Do not include final output time in partitioning time */
      ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer->finaloutput, zz->Communicator);
    }

    if (hgp.globalcomm.Communicator != MPI_COMM_NULL) {
      /* Processor participated in partitioning */
      bal = Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0,
                                       zz->LB.Num_Global_Parts, parts);
      cutl= Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts,
                                      zz->LB.Num_Global_Parts, &err);
      cutn = Zoltan_PHG_Compute_NetCut(hg->comm, hg, parts,
                                       zz->LB.Num_Global_Parts);
      for (i = 0; i < zoltan_hg->nObj; ++i) {
        /* uprintf(hg->comm, " obj[%d] = %d  in=%d out=%d\n", i, zoltan_hg->AppObjSizes[i], zoltan_hg->Input_Parts[i], zoltan_hg->Output_Parts[i]); */
	if (zoltan_hg->Input_Parts[i] != zoltan_hg->Output_Parts[i])
            move += (double) ((zoltan_hg->AppObjSizes) ? zoltan_hg->AppObjSizes[i] : 1.0);

      }
    }

    if (!err) {
     
      /* Add in cut contributions from removed edges */
      MPI_Allreduce(&(zoltan_hg->nRemove), &gnremove, 1, MPI_INT, MPI_SUM,
                    zz->Communicator);
      if (gnremove) {
        err = Zoltan_PHG_Removed_Cuts(zz, zoltan_hg, rlocal);
        MPI_Allreduce(rlocal, rglobal, 2, MPI_DOUBLE,MPI_SUM,zz->Communicator);
        
        cutl += rglobal[0];
        cutn += rglobal[1];
      }

      MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator);

      repart = cutl*hgp.RepartMultiplier + gmove;
      repartsum += repart;
      if (repart > repartmax) repartmax = repart;
      if (repart < repartmin) repartmin = repart;
      movesum += gmove;
      if (gmove > movemax) movemax = gmove;
      if (gmove < movemin) movemin = gmove;
      cutlsum += cutl;
      if (cutl > cutlmax) cutlmax = cutl;
      if (cutl < cutlmin) cutlmin = cutl;
      cutnsum += cutn;
      if (cutn > cutnmax) cutnmax = cutn;
      if (cutn < cutnmin) cutnmin = cutn;
      balsum += bal;
      if (bal > balmax) balmax = bal;
      if (bal < balmin) balmin = bal;
      nRuns++;
   
      if (zz->Proc == 0) {
        uprintf(hg->comm, 
                "STATS Runs %d  bal  CURRENT %f  MAX %f  MIN %f  AVG %f\n", 
                nRuns, bal, balmax, balmin, balsum/nRuns);
        uprintf(hg->comm, 
                "STATS Runs %d  cutl CURRENT %f  MAX %f  MIN %f  AVG %f\n", 
                nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns);
        uprintf(hg->comm, 
                "STATS Runs %d  cutn CURRENT %f  MAX %f  MIN %f  AVG %f\n", 
                nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns);
	uprintf(hg->comm,
		"STATS Runs %d  %s CURRENT %f  MAX %f  MIN %f  AVG %f\n",
		nRuns, (zoltan_hg->showMoveVol) ? "moveVol" : "moveCnt", gmove, movemax, movemin, movesum/nRuns);
        if (zoltan_hg->showMoveVol) 
            uprintf(hg->comm,
		"STATS Runs %d  repart CURRENT %f  MAX %f  MIN %f  AVG %f\n",
		nRuns, repart, repartmax, repartmin, repartsum/nRuns);        
      }
    }

    if (hgp.use_timers) {
      ZOLTAN_TIMER_STOP(zz->ZTime, timer->finaloutput, zz->Communicator);
      ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator);
    }
  }
  /* KDDKDD  End of printing section. */
  
  ZOLTAN_FREE(&parts);
  if (zoltan_hg != NULL) {
    Zoltan_PHG_Free_Hypergraph_Data(zoltan_hg);
    ZOLTAN_FREE (&zoltan_hg);
  }

  if (hgp.use_timers) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator);
    if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
      Zoltan_Timer_PrintAll(zz->ZTime, 0, hgp.globalcomm.Communicator, stdout);
  }

  if (hgp.globalcomm.row_comm != MPI_COMM_NULL)
    MPI_Comm_free(&(hgp.globalcomm.row_comm));
  if (hgp.globalcomm.col_comm != MPI_COMM_NULL)
    MPI_Comm_free(&(hgp.globalcomm.col_comm));
  if (hgp.globalcomm.Communicator != MPI_COMM_NULL)
    MPI_Comm_free(&(hgp.globalcomm.Communicator));

  /* Free part_sizes if created new due to ADD_OBJ_WEIGHT */
  if (hgp.part_sizes != part_sizes)
    ZOLTAN_FREE(&hgp.part_sizes);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return err;
}
Ejemplo n.º 3
0
int Zoltan_ParMetis_Order(
  ZZ *zz,               /* Zoltan structure */
  int num_obj,          /* Number of (local) objects to order. */
  ZOLTAN_ID_PTR gids,   /* List of global ids (local to this proc) */
                        /* The application must allocate enough space */
  ZOLTAN_ID_PTR lids,   /* List of local ids (local to this proc) */
                        /* The application must allocate enough space */
  ZOLTAN_ID_PTR rank,   /* rank[i] is the rank of gids[i] */
  int *iperm,
  ZOOS *order_opt       /* Ordering options, parsed by Zoltan_Order */
)
{
  static char *yo = "Zoltan_ParMetis_Order";
  int i, n, ierr;
  ZOLTAN_Output_Order ord;
  ZOLTAN_Third_Graph gr;

#ifdef ZOLTAN_PARMETIS
  MPI_Comm comm = zz->Communicator;/* don't want to risk letting external 
                                      packages changing our communicator */
#endif
  indextype numflag = 0;

  int timer_p = 0;
  int get_times = 0;
  int use_timers = 0;
  double times[5];

  ZOLTAN_ID_PTR       l_gids = NULL;
  ZOLTAN_ID_PTR       l_lids = NULL;

  indextype options[MAX_PARMETIS_OPTIONS];
  char alg[MAX_PARAM_STRING_LEN];

  ZOLTAN_TRACE_ENTER(zz, yo);

#ifdef ZOLTAN_PARMETIS
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES

#ifdef TPL_FLOAT_WEIGHT
  i = 1;
#else
  i = 0;
#endif

  if ((sizeof(indextype) != sizeof(idxtype)) ||
      (sizeof(weighttype) != sizeof(idxtype)) || i){

    ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
          "Not supported: Multiple 3rd party libraries with incompatible "
          "data types.");
    return ZOLTAN_FATAL;
  }
#endif
#endif

  memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
  memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
  memset(times, 0, sizeof(times));

  ord.order_opt = order_opt;

  if (!order_opt){
    /* If for some reason order_opt is NULL, allocate a new ZOOS here. */
    /* This should really never happen. */
    order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
    strcpy(order_opt->method,"PARMETIS");
  }

  ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
  /* ParMetis only computes the rank vector */
  order_opt->return_args = RETURN_RANK;

  /* Check that num_obj equals the number of objects on this proc. */
  /* This constraint may be removed in the future. */
  n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
  if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
    /* Return error code */
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
    return(ZOLTAN_FATAL);
  }
  if (n != num_obj){
    /* Currently this is a fatal error. */
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
                                     "number of objects.");
    return(ZOLTAN_FATAL);
  }

  /* Do not use weights for ordering */
  gr.obj_wgt_dim = -1;
  gr.edge_wgt_dim = -1;
  gr.num_obj = num_obj;

  /* Check what ordering type is requested */
  if (order_opt){
      SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */

#ifdef ZOLTAN_PARMETIS
      if ((strcmp(order_opt->method, "METIS") == 0))
#endif /* ZOLTAN_PARMETIS */
      SET_LOCAL_GRAPH(&gr.graph_type);
  }
  gr.get_data = 1;

  if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
                                     "set ParMetis instead.");
    return(ZOLTAN_FATAL);
  }

  timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);

    /* Start timer */
  get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
  if (get_times){
    MPI_Barrier(zz->Communicator);
    times[0] = Zoltan_Time(zz->Timer);
  }

  ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids,  &gr, NULL, NULL, NULL);
  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
    Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
    return (ierr);
  }

  /* Allocate space for separator sizes */

  if (IS_GLOBAL_GRAPH(gr.graph_type)) {
    if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
      /* Not enough memory */
      Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
      ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
    }
    ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
    if (ord.sep_sizes == NULL) {
      Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
      ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
    }
    memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
  }

  /* Allocate space for direct perm */
  ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
  if (!ord.rank){
    /* Not enough memory */
    Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
    ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
  }
  if (IS_LOCAL_GRAPH(gr.graph_type)){
  /* Allocate space for inverse perm */
    ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
    if (!ord.iperm){
      /* Not enough memory */
      Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
      ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
    }
  }
  else
    ord.iperm = NULL;

  /* Get a time here */
  if (get_times) times[1] = Zoltan_Time(zz->Timer);

#ifdef ZOLTAN_PARMETIS
  if (IS_GLOBAL_GRAPH(gr.graph_type)){
    ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");

    ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy, 
                        &numflag, options, ord.rank, ord.sep_sizes, &comm);
    ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");

  }
  else
#endif /* ZOLTAN_PARMETIS */
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
 if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
    indextype numobj = gr.num_obj;
    ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
    order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
    options[0] = 0;  /* Use default options for METIS. */
    METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options, 
                 ord.iperm, ord.rank);
#else
    METIS_SetDefaultOptions(options);
    METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options, 
                 ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
#endif


    ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
  }
#endif /* ZOLTAN_METIS */

  /* Get a time here */
  if (get_times) times[2] = Zoltan_Time(zz->Timer);

  if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
    int numbloc;
    int start;
    int leaf;
    int *converttab;
    int levelmax;

    levelmax = mylog2(zz->Num_Proc) + 1;
    converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));

    memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
     /* Determine the first node in each separator, store it in zz->TPL_Order.start */
    for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
      int father;

      father = zz->Num_Proc + numbloc;
      converttab[start] = 2*numbloc;
      zz->TPL_Order.leaves[leaf++]=start;
      zz->TPL_Order.ancestor[start] = start + 2;
      converttab[start+1] = 2*numbloc+1;
      zz->TPL_Order.leaves[leaf++]=start+1;
      zz->TPL_Order.ancestor[start+1] = start + 2;
      start+=2;
      do {
        converttab[start] = father;
        if (father %2 == 0) {
          int nextoffset;
          int level;

          level = mylog2(2*zz->Num_Proc - 1 - father);
          nextoffset = (1<<(levelmax-level));
          zz->TPL_Order.ancestor[start] = start+nextoffset;
          start++;
          break;
        }
        else {
          zz->TPL_Order.ancestor[start] = start+1;
          start++;
          father = zz->Num_Proc + father/2;
        }
      } while (father < 2*zz->Num_Proc - 1);
    }

    zz->TPL_Order.start[0] = 0;
    zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
    for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
      int oldblock=converttab[numbloc-1];
      zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
    }

    ZOLTAN_FREE(&converttab);
    ZOLTAN_FREE(&ord.sep_sizes);

    zz->TPL_Order.leaves[zz->Num_Proc] = -1;
    zz->TPL_Order.nbr_leaves = zz->Num_Proc;
    zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
  }
  else { /* No tree */
    zz->TPL_Order.nbr_blocks = 0;
    zz->TPL_Order.start = NULL;
    zz->TPL_Order.ancestor = NULL;
    zz->TPL_Order.leaves = NULL;
  }

  /* Correct because no redistribution */
  memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
  memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));

  ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);

  ZOLTAN_FREE(&l_gids); 
  ZOLTAN_FREE(&l_lids);

  /* Get a time here */
  if (get_times) times[3] = Zoltan_Time(zz->Timer);

  if (get_times) Zoltan_Third_DisplayTime(zz, times);

  if (use_timers)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);

  if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
    memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
  }
  else{
    for (i=0; i < gr.num_obj; i++){
      rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
    }
  }

  if ((ord.iperm != NULL) && (iperm != NULL)){
    if (sizeof(indextype) == sizeof(int)){
      memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
    }
    else{
      for (i=0; i < gr.num_obj; i++){
        iperm[i] = (int)ord.iperm[i];
      }
    }
  }

  if (ord.iperm != NULL)  ZOLTAN_FREE(&ord.iperm);
  ZOLTAN_FREE(&ord.rank);

  /* Free all other "graph" stuff */
  Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ZOLTAN_OK);
}
Ejemplo n.º 4
0
int
Zoltan_PHG_2ways_hyperedge_partition (
  ZZ *zz,                            /* Input : Zoltan data structure */
  HGraph *hg,
  Partition parts,
  Zoltan_PHG_Tree *tree,
  struct Zoltan_DD_Struct * gnoToGID,
  struct Zoltan_DD_Struct **dd,
  int *numParts,
  int **sizeParts
)
{
  int ierr = ZOLTAN_OK;
  char *yo = "Zoltan_PHG_2ways_hyperedge_partition";
  int nEdge, hEdge;
  int *interval;
  int *partnumber = NULL;
  int tree_size;
  ZOLTAN_ID_TYPE *rowpart =NULL;  /* ZOLTAN_ID_TYPE because it's used in Zoltan_DD_* */
  ZOLTAN_GNO_TYPE *rowGNO = NULL;
  ZOLTAN_ID_PTR rowGID=NULL;
  int index;
  int offset;

  ZOLTAN_TRACE_ENTER(zz, yo);

  nEdge = hg->nEdge;
  fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y,  hg->nVtx, nEdge);

  interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int));
  if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR;

  tree_size = get_tree_size(tree) + 1;
  for (index = 0 ; index < nEdge ; ++index){
    SET_MIN_NODE(interval, index, tree_size);
    SET_MAX_NODE(interval, index, -1);
  }

  /* Update interval with the local knowledge */
  /* XXX: I loop on the hyperedges, as I think it's more cache friendly
   * and it allows me to discoupled the computation if a non blocking MPI_Reduce is
   * available
   */
  for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){
    int part;
    int max = -1;                     /* Trick : we use the initialized values */
    int min = tree_size + 1;

    for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) {
      part = parts[hg->hvertex[index]];

      max = MAX(max, part);
      min = MIN(min, part);
    }
    SET_MIN_NODE(interval, hEdge, min);
    SET_MAX_NODE(interval, hEdge, max);
  }

  /* Update results to view the complete hyperedges */
  Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm);

  /* Now I have to compute the partition of hyperedges according to the "interval"
   * and the tree */

  /* First, compute the partition number corresponding to the nodes in the tree */
  partnumber = compute_part_number(tree);
  if (partnumber == NULL) {
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  (*numParts) = get_tree_size(tree);

  rowpart = (ZOLTAN_ID_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_ID_TYPE));
  if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR;

  rowGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_GNO_TYPE));
  if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR;

  (*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int));
  if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR;

  offset = hg->dist_y[hg->comm->myProc_y];
  /* Then we search we is the hyperedge in the tree */
  for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
    int node;
    node = find_interval_in_tree(tree, interval+2*hEdge);
    rowpart[hEdge] = partnumber[node];
    (*sizeParts)[rowpart[hEdge]] ++;
    rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge);
#if 0
    fprintf (stderr, "%zd : " ZOLTAN_ID_SPEC " (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]);
#endif
  }

  partnumber += 1;
  ZOLTAN_FREE(&partnumber);
  ZOLTAN_FREE(&interval);

  /* Compute number of elements per parts */
  /* TODO: support processor which are not part of the distribution */

  /* Update results to view the complete hyperedges */
  Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm);


  /* Export results to data directory */
  /* First, get the GIDs of our edges */
  rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge);
  if (nEdge && rowGID == NULL) MEMORY_ERROR;
  ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL,
			 nEdge, NULL);
  ZOLTAN_FREE(&rowGNO);

  ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0);
  CHECK_IERR;

  /* Make our new numbering public */
  Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, rowpart, NULL,  NULL, nEdge);

#ifdef CEDRIC_PRINT
  for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) {
    fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]);
  }
#endif


 End:
  ZOLTAN_FREE(&rowGID);
  ZOLTAN_FREE(&rowGNO);
  ZOLTAN_FREE(&rowpart);

  if (partnumber != NULL)
    partnumber += 1;
  ZOLTAN_FREE(&partnumber);
  ZOLTAN_FREE(&interval);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 5
0
/*
 * void lb_oct_init();
 *
 * initialize the calls needed to start the octree load balancing rounties
 */
static int lb_oct_init(
  ZZ *zz,                       /* The Zoltan structure with info for
                                   the OCTPART balancer.                    */
  int *num_export,              /* Number of non-local objs assigned to this
                                   processor in the new decomposition.      */
  ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for
                                   non-local objects in this processor's new
                                   decomposition.                           */
  ZOLTAN_ID_PTR *export_local_ids,  /* Returned value:  array of local IDs for
                                   non-local objects in this processor's new
                                   decomposition.                           */
  int **export_procs,           /* Returned value:  array of processor IDs for
                                   processors owning the non-local objects in
                                   this processor's new decomposition.      */
  int **export_to_part,         /* Returned value:  array of partitions to 
                                   which objects are imported.
                                   KDDKDD Assume #parts==#procs.            */
  int oct_dim,                  /* Dimension of method (2D or 3D)           */
  int oct_method,               /* Flag specifying curve to be used.        */
  int oct_maxoctregions,        /* max # of objects in leaves of octree.    */
  int oct_minoctregions,        /* min # of objects in leaves of octree.    */
  int oct_output_level,         /* Flag specifying amount of output.        */
  int oct_wgtflag,              /* Flag specifying use of object weights.   */
  float *part_sizes             /* Array of size zz->Num_Global_Parts
                                   containing the percentage of work to be
                                   assigned to each partition.              */
) 
{
  char *yo = "lb_oct_init";
  OCT_Global_Info *OCT_info;
  int nsentags;                    /* number of tags being sent */
  pRegion export_regs;             /* */
  int nrectags;                    /* number of tags received */
  int kk;
  double time1,time2;              /* timers */
  double timestart,timestop;       /* timers */
  double timers[4];                /* diagnostic timers 
			              0 = start-up time before recursion
				      1 = time before median iterations
				      2 = time in median iterations
				      3 = communication time */
  int    counters[6];              /* diagnostic counts
			              0 = # of median iterations
				      1 = # of objects sent
				      2 = # of objects received
				      3 = most objects this proc ever owns
				      */
  float  c[4];
  int createpartree = 0;
  /*int num_gid_entries = zz->Num_GID;*/
  /*int num_lid_entries = zz->Num_LID;*/
  
  ZOLTAN_TRACE_ENTER(zz, yo);

  MPI_Barrier(zz->Communicator);
  timestart = MPI_Wtime();

  /* initialize timers and counters */
  counters[0] = 0;
  counters[1] = 0;
  counters[2] = 0;
  counters[3] = 0;
  counters[4] = 0;
  counters[5] = 0;
  c[0] = 0;
  c[1] = 0;
  c[2] = 0;
  c[3] = 0;
  timers[1] = 0.0;
  timers[2] = 0.0;
  timers[3] = 0.0;

  nsentags = nrectags = 0;

  if(zz->LB.Data_Structure == NULL) {
    OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim);
    Zoltan_Oct_set_method(OCT_info, oct_method);
    Zoltan_Oct_set_maxregions(oct_maxoctregions);
    Zoltan_Oct_set_minregions(oct_minoctregions);
    createpartree = 1;
  }
  else {
    OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);
  }

  /* create the octree structure */
  time1 = MPI_Wtime();

  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data");
  Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1],
				      &counters[2], &counters[3], &c[0], 
				      createpartree);

  time2 = MPI_Wtime();
  timers[0] = time2 - time1;                 /* time took to create octree */
  /* Zoltan_Oct_POct_printResults(OCT_info); */
  /* partition the octree structure */
  time1 = MPI_Wtime();
  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition");
  /* old call to dfs_paritition: */ 
#if 0
  Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]);
#else
  /***************************
  if(zz->Proc == 0) {
    int debug_i;
    for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) {
      fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]);
    }
  }
  ****************************/
  Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes);
#endif
  time2 = MPI_Wtime();
  timers[1] = time2 - time1;              /* time took to partition octree */

  if (oct_output_level > 2) {
    Zoltan_Oct_Plots(zz);
  }

  /* set up tags for migrations */
  time1 = MPI_Wtime();

#if 0  /* KDDKDD -- Count is never used; why is it computed? */
  {
  pRList  RootList;               /* list of all local roots */
  pOctant RootOct;                /* root octree octant */
  int count = 0; 
  RootList = Zoltan_Oct_POct_localroots(OCT_info);
  while((RootOct = RL_nextRootOctant(&RootList))) {
    while(RootOct) {
      if(Zoltan_Oct_isTerminal(RootOct)) {	
	count += Zoltan_Oct_nRegions(RootOct);
      }
      RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
    }
  }
  }
#endif

  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate");
  Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags, 
	         &c[2], &c[3], &counters[3], &counters[5]);

  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags");
  if (zz->LB.Return_Lists) {
    *num_export = nrectags;
    if (nrectags > 0)
      Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids, 
			  export_procs, export_to_part, nrectags,
			  export_regs);
  }

  time2 = MPI_Wtime();
  timers[2] = time2 - time1;               /* time took to setup migration */


#if 0  /* KDDKDD -- Count is never used; why is it computed? */
  {
  /* count the number of objects on this processor */
  pRList  RootList;               /* list of all local roots */
  pOctant RootOct;                /* root octree octant */
  int count = 0; 
  RootList = Zoltan_Oct_POct_localroots(OCT_info);
  while((RootOct = RL_nextRootOctant(&RootList))) {
    while(RootOct) {
      if(Zoltan_Oct_isTerminal(RootOct)) {	
	count += Zoltan_Oct_nRegions(RootOct);
      }
      RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct);
    }
  }
  }
#endif

  counters[4] = nsentags;
  MPI_Barrier(zz->Communicator);
  timestop = MPI_Wtime();

  if (oct_output_level > 0) {
    ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats");
    Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c, 
                       oct_output_level);
  }

  for (kk = 0; kk < nrectags; kk++) {
    ZOLTAN_FREE(&(export_regs[kk].Global_ID));
    ZOLTAN_FREE(&(export_regs[kk].Local_ID));
  }
  ZOLTAN_FREE(&export_regs);
  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear");
  Zoltan_Oct_global_clear(OCT_info);
  /* KDDKDD Don't understand how re-used octree will work, especially without
   * KDDKDD the Zoltan_Oct_Bounds_Geom function.  For now, we'll delete everything;
   * KDDKDD we can move back to saving some of the tree later.
   */
  Zoltan_Oct_Free_Structure(zz);
  /* KDDKDD END */

  /* Temporary return value until error codes are fully implemented. */
  ZOLTAN_TRACE_EXIT(zz, yo);
  return(ZOLTAN_OK);
}
Ejemplo n.º 6
0
static int Zoltan_LB(
  ZZ *zz, 
  int include_parts,             /* Flag indicating whether to generate
                                    partition informtion;
                                    0 if called by Zoltan_LB_Balance,
                                    1 if called by Zoltan_LB_Partition.       */
  int *changes,                  /* Set to zero or one depending on if 
                                    Zoltan determines a new
                                    decomposition or not:
                                    zero - No changes to the decomposition
                                           were made by the load-balancing
                                           algorithm; migration is not needed.
                                    one  - A new decomposition is suggested
                                           by the load-balancer; migration is
                                           needed to establish the new
                                           decomposition.                     */
  int *num_gid_entries,          /* The number of array entries in a global ID;
                                    set to be the max over all processors in
                                    zz->Communicator of the parameter
                                    Num_Global_ID_Entries.                    */
  int *num_lid_entries,          /* The number of array entries in a local ID;
                                    set to be the max over all processors in
                                    zz->Communicator of the parameter
                                    Num_Local_ID_Entries.                     */
  int *num_import_objs,          /* The number of non-local objects in the
                                    processor's new decomposition.            */
  ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
                                    (i.e., objs to be imported) in
                                    the processor's new decomposition.        */
  ZOLTAN_ID_PTR *import_local_ids,   /* Array of local IDs for non-local objects
                                    (i.e., objs to be imported) in
                                    the processor's new decomposition.        */
  int **import_procs,            /* Array of processor IDs for processors 
                                    currently owning non-local objects (i.e.,
                                    objs to be imported) in this processor's
                                    new decomposition.                        */
  int **import_to_part,          /* Partition to which the objects should be
                                    imported.                                 */
  int *num_export_objs,          /* The number of local objects that need to
                                    be exported from the processor to establish
                                    the new decomposition.                    */
  ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
                                    to be exported (assigned and sent to other
                                    processors) to establish the new 
                                    decomposition.                            */
  ZOLTAN_ID_PTR *export_local_ids,   /* Array of local IDs for objects that need
                                    to be exported (assigned and sent to other
                                    processors) to establish the new 
                                    decomposition.                            */
  int **export_procs,            /* Array of destination processor IDs for
                                    objects that need to be exported 
                                    to establish the new decomposition.       */
  int **export_to_part           /* Partition to which objects should be 
                                    exported.                                 */
)
{
/*
 * Main load-balancing routine.
 * Input:  a Zoltan structure with appropriate function pointers set.
 * Output: 
 *   changes
 *   num_import_objs
 *   import_global_ids
 *   import_local_ids
 *   import_procs
 *   import_to_part
 *   num_export_objs
 *   export_global_ids
 *   export_local_ids
 *   export_procs
 *   export_to_part
 * Return values:
 *   Zoltan error code.
 */

char *yo = "Zoltan_LB";
int gmax;    /* Maximum number of imported/exported objects 
                over all processors.                       */
int error = ZOLTAN_OK;    /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3]; 
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;

  ZOLTAN_TRACE_ENTER(zz, yo);

  if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
    Zoltan_Print_Key_Params(zz);

  start_time = Zoltan_Time(zz->Timer);

#ifdef ZOLTAN_DRUM
  /* initialize DRUM if needed */
  Zoltan_Drum_Create_Model(zz);

  /* stop DRUM monitors */
  Zoltan_Drum_Stop_Monitors(zz);
#endif

  /* 
   * Compute Max number of array entries per ID over all processors.
   * Compute Max number of return arguments for Zoltan_LB_Balance.
   * This is a sanity-maintaining step; we don't want different
   * processors to have different values for these numbers.
   */
  comm[0] = zz->Num_GID;
  comm[1] = zz->Num_LID;
  comm[2] = zz->LB.Return_Lists;

  MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);

  zz->Num_GID = *num_gid_entries = gcomm[0];
  zz->Num_LID = *num_lid_entries = gcomm[1];
  zz->LB.Return_Lists = gcomm[2];

  /* assume no changes */
  *changes = 0;

  *num_import_objs = *num_export_objs = 0;
  *import_global_ids = NULL;
  *import_local_ids = NULL;
  *import_procs = NULL;
  *import_to_part = NULL;
  *export_global_ids = NULL;
  *export_local_ids = NULL;
  *export_procs = NULL;
  *export_to_part = NULL;

  /*
   *  Return if this processor is not in the Zoltan structure's
   *  communicator.
   */

  if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) 
    goto End;

  if (zz->LB.Method == NONE) {
    if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
      printf("%s Balancing method selected == NONE; no balancing performed\n",
              yo);

    error = ZOLTAN_WARN;
    goto End;
  }

  /*
   *  Sync the random number generator across processors.
   */

  Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);

  /*
   *  Construct the heterogenous machine description.
   */

  error = Zoltan_Build_Machine_Desc(zz);

  if (error == ZOLTAN_FATAL)
    goto End;

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");

  /* Since generating a new partition, need to free old mapping vector */
  zz->LB.OldRemap = zz->LB.Remap;
  zz->LB.Remap = NULL;

  error = Zoltan_LB_Build_PartDist(zz);
  if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
    goto End;

  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
    int i, np, fp;
    for (i = 0; i < zz->Num_Proc; i++) {
      Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
      printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", 
             zz->Proc, i, np, fp);
    }
  }

  /*
   * Generate partitions sizes.
   */

#ifdef ZOLTAN_DRUM
  /* set partition sizes computed by DRUM, if requested */
  Zoltan_Drum_Set_Part_Sizes(zz);
#endif

  wgt_dim = zz->Obj_Weight_Dim;
  part_dim = ((wgt_dim > 0) ? wgt_dim : 1);

  part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim 
                                     * zz->LB.Num_Global_Parts);
  if (part_sizes == NULL) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
    error = ZOLTAN_MEMERR;
    goto End;
  }

  /* Get partition sizes. */
  Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
    part_sizes);

  /*
   * Call the actual load-balancing function.
   */

  error = zz->LB.LB_Fn(zz, part_sizes,
                       num_import_objs, import_global_ids, import_local_ids,
                       import_procs, import_to_part, 
                       num_export_objs, export_global_ids, export_local_ids, 
                       export_procs, export_to_part);

  ZOLTAN_FREE(&part_sizes);

  if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
    sprintf(msg, "Partitioning routine returned code %d.", error);
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    goto End;
  }
  else if (error){
    if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
      sprintf(msg, "Partitioning routine returned code %d.", error);
      ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");

#ifdef ZOLTAN_DRUM
  /* restart DRUM monitors -- should happen later but there are a lot
     of ways out of Zoltan_LB and we want to make sure they do start */
  Zoltan_Drum_Start_Monitors(zz);
#endif

  if (*num_import_objs >= 0)
    MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, 
                zz->Communicator);
  else /* use export data */
    MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, 
                zz->Communicator);

  if (gmax == 0) {

    /*
     *  Decomposition was not changed by the load balancing; no migration
     *  is needed.
     */

    if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
      printf("%s No changes to the decomposition due to partitioning; "
             "no migration is needed.\n", yo);

    /*
     *  Reset num_import_objs and num_export_objs; don't want to return
     *  -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
     */

    *num_import_objs = *num_export_objs = 0;

    if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
      /*
       * This parameter setting requires that all local objects
       * and their assignments appear in the export list.
       */
      error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, 
               export_global_ids, export_local_ids,
               wgt_dim, &fdummy, export_to_part);

      if (error == ZOLTAN_OK){
        ZOLTAN_FREE(&fdummy);
        if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
                            ZOLTAN_SPECIAL_MALLOC_INT)){
          for (i=0; i<*num_export_objs; i++)
            (*export_procs)[i] = zz->Proc;
        }
        else{
          error = ZOLTAN_MEMERR;
        }
      }
    }
    goto End;
  }

  /*
   *  Check whether we know the import data, export data, or both.
   *
   *  If we were given the import data,
   *  we know what the new decomposition should look like on the
   *  processor, but we don't know which of our local objects we have
   *  to export to other processors to establish the new decomposition.
   *  Reverse the argument if we were given the export data.
   *
   *  Unless we were given both maps, compute the inverse map.
   */

  if (*num_import_objs >= 0){
    if (*num_export_objs >= 0) {
      /* Both maps already available; nothing to do. */;

      if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
        /* This condition should never happen!! */
        /* Methods should not return arrays if no lists are requested. */
        *num_import_objs = *num_export_objs = -1;
        Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, 
                            import_procs, import_to_part);
        Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, 
                            export_procs, export_to_part);
        ZOLTAN_PRINT_WARN(zz->Proc, yo, 
                      "Method returned lists, but no lists requested.");
      }
    }
    else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || 
             zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
             zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
      /* Export lists are requested; compute export map */
      error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, 
                                      *import_local_ids, *import_procs,
                                      *import_to_part,
                                      num_export_objs, export_global_ids,
                                      export_local_ids, export_procs,
                                      export_to_part);
      if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
        sprintf(msg, "Error building return arguments; "
                     "%d returned by Zoltan_Compute_Destinations\n", error);
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
        goto End;
      }
      if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
          zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
        /* Method returned import lists, but only export lists were desired. */
        /* Import lists not needed; free them. */
        *num_import_objs = -1;
        Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, 
                            import_procs, import_to_part);
      }
    }
  }
  else { /* (*num_import_objs < 0) */
    if (*num_export_objs >= 0) {
      /* Only export lists have been returned. */
      if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || 
          zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
        /* Compute import map */
        error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, 
                                        *export_local_ids, *export_procs,
                                        *export_to_part,
                                        num_import_objs, import_global_ids,
                                        import_local_ids, import_procs, 
                                        import_to_part);

        if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
          sprintf(msg, "Error building return arguments; "
                       "%d returned by Zoltan_Compute_Destinations\n", error);
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
          goto End;
        }
        if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
          /* Method returned export lists, but only import lists are desired. */
          /* Export lists not needed; free them. */
          *num_export_objs = -1;
          Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, 
                              export_procs, export_to_part);
        }
      }
    }
    else {  /* *num_export_objs < 0 && *num_import_objs < 0) */
      if (zz->LB.Return_Lists) {
        /* No map at all available */
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
               "neither import nor export data.");
        error = ZOLTAN_WARN;
        goto End;
      }
    }
  }

  if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
    /*
     * Normally, Zoltan_LB returns in the export lists all local
     * objects that are moving off processor, or that are assigned
     * to a partition on the local processor that is not the
     * default partition.  This setting of Return_Lists requests
     * that all local objects be included in the export list.
     */

    if (*num_export_objs == 0){
      /* all local objects are remaining on processor */

      error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
               export_global_ids, export_local_ids,
               wgt_dim, &fdummy, export_to_part);

      if (error == ZOLTAN_OK){
        ZOLTAN_FREE(&fdummy);
        if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
                            ZOLTAN_SPECIAL_MALLOC_INT)){
          for (i=0; i<*num_export_objs; i++)
            (*export_procs)[i] = zz->Proc;
        }
        else{
          error = ZOLTAN_MEMERR;
        }
      }
      if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
    }
    else{
      all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);

      if (*num_export_objs < all_num_obj){
  
        /* Create a lookup table for exported IDs */
  
        if (*num_export_objs > 16){   /* could be 0, maybe only importing */
          ts = (*num_export_objs) / 4;   /* what's a good table size? */
        }
        else{
          ts = *num_export_objs;
        }
  
        ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
  
        /* Create a list of all gids, lids and partitions */
  
        error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, 
                 &all_global_ids, &all_local_ids,
                 wgt_dim, &fdummy, &parts);

        if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
          ZOLTAN_FREE(&fdummy);
          if ((Zoltan_Special_Malloc(zz, (void **)&export_all_procs, 
                 all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
              (Zoltan_Special_Malloc(zz, (void **)&export_all_to_part, 
                 all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){

            error = ZOLTAN_MEMERR;
          }
        }
  
        if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
          sprintf(msg, "Error building complete export list; "
                       "%d returned by Zoltan_Get_Obj_List\n", error);
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
          goto End;
        }
  
        gid = all_global_ids;
  
        for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
  
          idIdx = search_hash_table(zz, gid, ht, ts);
  
          if (idIdx >= 0){

            export_all_procs[i] = (*export_procs)[idIdx];
            export_all_to_part[i] = (*export_to_part)[idIdx];
          }
          else{
            export_all_procs[i] = zz->Proc;
            export_all_to_part[i] = parts[i];
          }
        }
  
        free_hash_table(ht, ts);

        Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, 
                            export_procs, export_to_part);
        Zoltan_Special_Free(zz, (void **)&parts, 
                            ZOLTAN_SPECIAL_MALLOC_INT);
  
        *export_global_ids = all_global_ids;
        *export_local_ids = all_local_ids;
        *export_procs = export_all_procs;
        *export_to_part = export_all_to_part;
        *num_export_objs = all_num_obj;
      }
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");

  end_time = Zoltan_Time(zz->Timer);
  lb_time[0] = end_time - start_time;

  if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
    int i;
    Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
    printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
    for (i = 0; i < *num_import_objs; i++) {
      printf("    Obj: ");
      ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
      printf("  To partition: %4d", 
             (*import_to_part != NULL ? (*import_to_part)[i] 
                                      : zz->Proc));
      printf("  From processor: %4d\n", (*import_procs)[i]);
    }
    printf("\n");
    printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
    for (i = 0; i < *num_export_objs; i++) {
      printf("    Obj: ");
      ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
      printf("  To partition: %4d",
             (*export_to_part != NULL ? (*export_to_part)[i] 
                                      : (*export_procs)[i]));
      printf("  To processor: %4d\n", (*export_procs)[i]);
    }
    Zoltan_Print_Sync_End(zz->Communicator, TRUE);
  }

  /*
   *  If the Help_Migrate flag is set, perform migration for the application.
   */

  if (zz->Migrate.Auto_Migrate) {
    ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");

    start_time = Zoltan_Time(zz->Timer);
    error = Zoltan_Migrate(zz,
                            *num_import_objs, *import_global_ids,
                            *import_local_ids, *import_procs, *import_to_part,
                            *num_export_objs, *export_global_ids,
                            *export_local_ids, *export_procs, *export_to_part);
    if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
      sprintf(msg, "Error in auto-migration; %d returned from "
                    "Zoltan_Help_Migrate\n", error);
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      goto End;
    }
    end_time = Zoltan_Time(zz->Timer);
    lb_time[1] = end_time - start_time;

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
  }
  
  /* Print timing info */
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
    if (zz->Proc == zz->Debug_Proc) {
      printf("ZOLTAN Times:  \n");
    }
    Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], 
                   "ZOLTAN     Partition:     ");
    if (zz->Migrate.Auto_Migrate)
      Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], 
                      "ZOLTAN     Migrate: ");
  }

  *changes = 1;

End:
  ZOLTAN_TRACE_EXIT(zz, yo);
  return (error);
}
Ejemplo n.º 7
0
  /*
   * Each processor:
   *   owns a set of pins (nonzeros)
   *   may provide some edge weights
   *
   * We assume that no two processes will supply the same pin.
   * But more than one process may supply pins for the same edge.
   */
static int
matrix_get_edges(ZZ *zz, Zoltan_matrix *matrix, ZOLTAN_ID_PTR *yGID, ZOLTAN_ID_PTR *pinID, int nX,
		 ZOLTAN_ID_PTR *xGID, ZOLTAN_ID_PTR *xLID, ZOLTAN_GNO_TYPE **xGNO, float **xwgt, int use_full_dd)
{
  static char *yo = "matrix_get_edges";
  int ierr = ZOLTAN_OK;
  int hypergraph_callbacks = 0, graph_callbacks = 0;
  int *nbors_proc = NULL; /* Pointers are global for the function to ensure proper free */
  int *edgeSize = NULL;

  ZOLTAN_TRACE_ENTER(zz, yo);
  if (zz->Get_HG_Size_CS && zz->Get_HG_CS) {
    hypergraph_callbacks = 1;
  }
  if ((zz->Get_Num_Edges != NULL || zz->Get_Num_Edges_Multi != NULL) &&
           (zz->Get_Edge_List != NULL || zz->Get_Edge_List_Multi != NULL)) {
    graph_callbacks = 1;
  }

/* TEMPORARY FIX */
if (!graph_callbacks){
  fprintf(stderr,"Bug #5470: matrix_get_edges fails for hypergraph queries\n");
  return ZOLTAN_FATAL; 
}
hypergraph_callbacks=0;
/* TEMPORARY FIX */

  if (graph_callbacks && hypergraph_callbacks){
/*     if (hgraph_model == GRAPH) */
/*       hypergraph_callbacks = 0; */
    graph_callbacks = 1; /* I prefer graph (allow to do "inplace") ! */
  }

  if (hypergraph_callbacks) {
    matrix->redist = 1;
    if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID))
      ZOLTAN_FREE(xGID);
    else
      *xGID = NULL;
    ZOLTAN_FREE(xLID);
    ZOLTAN_FREE(xGNO);
    ierr = Zoltan_Hypergraph_Queries(zz, &matrix->nY,
				     &matrix->nPins, yGID, &matrix->ystart,
				     pinID);
    CHECK_IERR;
    matrix->yend = matrix->ystart + 1;
  }
  else if (graph_callbacks) {
    int max_edges = 0;
    int vertex;
    int numGID, numLID;

    matrix->opts.enforceSquare = 1;
    matrix->nY = nX; /* It is square ! */
    matrix->yGNO = *xGNO;
    *yGID = NULL;
    matrix->ywgtdim = zz->Obj_Weight_Dim;
    *xwgt = NULL;

    numGID = zz->Num_GID;
    numLID = zz->Num_LID;

    /* TODO : support local graphs */
    /* TODO : support weights ! */
    /* Get edge data */
    Zoltan_Get_Num_Edges_Per_Obj(zz, matrix->nY, *xGID, *xLID,
				 &edgeSize, &max_edges, &matrix->nPins);

    (*pinID) = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nPins);
    nbors_proc = (int *)ZOLTAN_MALLOC(matrix->nPins * sizeof(int));

    if (matrix->nPins && ((*pinID) == NULL || nbors_proc == NULL))
      MEMORY_ERROR;

    matrix->pinwgt = (float*)ZOLTAN_MALLOC(matrix->nPins*matrix->pinwgtdim*sizeof(float));
    if (matrix->nPins && matrix->pinwgtdim && matrix->pinwgt == NULL)
      MEMORY_ERROR;

    if (zz->Get_Edge_List_Multi) {
      zz->Get_Edge_List_Multi(zz->Get_Edge_List_Multi_Data,
			      numGID, numLID,
			      matrix->nY, *xGID, *xLID,
			      edgeSize,
			      (*pinID), nbors_proc, matrix->pinwgtdim,
			      matrix->pinwgt, &ierr);
    }
    else {
      int edge;
      for (vertex = 0, edge = 0 ; vertex < matrix->nY ; ++vertex) {
	zz->Get_Edge_List(zz->Get_Edge_List_Data, numGID, numLID,
                          (*xGID)+vertex*numGID, (*xLID)+vertex*numLID,
                          (*pinID)+edge*numGID, nbors_proc+edge, matrix->pinwgtdim,
                          matrix->pinwgt+edge*matrix->pinwgtdim, &ierr);
	edge += edgeSize[vertex];
      }
    }
    CHECK_IERR;

    /* Not Useful anymore */
    ZOLTAN_FREE(xLID);
    if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID)) {
      ZOLTAN_FREE(xGID);
      *xGNO = NULL;
    }
    else {
      *xGID = NULL;
      *xGNO = NULL;
    }
    ZOLTAN_FREE(&nbors_proc);

    /* Now construct CSR indexing */
    matrix->ystart = (int*) ZOLTAN_MALLOC((matrix->nY+1)*sizeof(int));
    if (matrix->ystart == NULL)
      MEMORY_ERROR;

    matrix->ystart[0] = 0;
    matrix->yend = matrix->ystart + 1;
    for (vertex = 0 ; vertex < matrix->nY ; ++vertex)
      matrix->ystart[vertex+1] = matrix->ystart[vertex] + edgeSize[vertex];
  }
  else {
    FATAL_ERROR ("You have to define Hypergraph or Graph queries");
  }

  if (matrix->opts.enforceSquare) {
    matrix->globalY = matrix->globalX;
    matrix->ddY = matrix->ddX;
    matrix->ywgtdim = zz->Obj_Weight_Dim;
  }

 End:
  ZOLTAN_FREE(&edgeSize);
  ZOLTAN_FREE(&nbors_proc);
  ZOLTAN_FREE(xLID);
  ZOLTAN_FREE(xGID);
  ZOLTAN_FREE(xGNO);
  ZOLTAN_FREE(xwgt);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 8
0
void Zoltan_HG_Print(
  ZZ *zz,
  HGraph *hg,
  Partition parts,
  FILE *fp,
  char *str
)
{
/* Routine to print hypergraph weights and edges. Assumes serial execution;
 * put inside Zoltan_Print_Sync_Start/Zoltan_Print_Sync_End for parallel
 * programs. 
 */
int i, j;
int num_vwgt;
int num_ewgt;
float *sum;
char *yo = "Zoltan_HG_Print";

  if (hg == NULL)
    return;

  ZOLTAN_TRACE_ENTER(zz, yo);

  num_vwgt = hg->VtxWeightDim;
  num_ewgt = hg->EdgeWeightDim;

  sum = (float *) ZOLTAN_MALLOC(MAX(num_vwgt, num_ewgt) * sizeof(float));

  fprintf(fp, "%s nVtx=%d nEdge=%d nPins=%d vWgt=%d eWgt=%d\n", 
          str, hg->nVtx, hg->nEdge, hg->nPins, 
          hg->VtxWeightDim, hg->EdgeWeightDim);

  /* Print Vertex Info */
  fprintf(fp, "%s Vertices:  (edges)\n", str);
  for (i = 0; i < hg->nVtx; i++) {
    fprintf(fp, "%d (" ZOLTAN_GNO_SPEC ") in part %d:  ", 
            i, VTX_LNO_TO_GNO(hg, i), (parts ? parts[i] : -1));
    fprintf(fp, "(");
    for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
      fprintf(fp, "%d ", hg->vedge[j]);
    fprintf(fp, ")\n");
  }

  if (hg->vwgt != NULL) {
    for (j = 0; j < num_vwgt; j++) sum[j] = 0;
    fprintf(fp, "%s Vertices: [weights])\n", str);
    for (i = 0; i < hg->nVtx; i++) {
      fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "):  [", i, VTX_LNO_TO_GNO(hg, i));
      for (j = 0; j < num_vwgt; j++) {
        fprintf(fp, "%f ", hg->vwgt[i*num_vwgt + j]);
        sum[j] += hg->vwgt[i*num_vwgt + j];
      }
      fprintf(fp, "])\n");
    }
    fprintf(fp, "Total vertex weight = [");
    for (j = 0; j < num_vwgt; j++) fprintf(fp, "%f  ", sum[j]);
    fprintf(fp, "]\n");
  }

  /* Print Hyperedge Info */
  fprintf(fp, "%s Hyperedges:  (vertices)\n", str);
  for (i = 0; i < hg->nEdge; i++) {
    fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "):  ", i, EDGE_LNO_TO_GNO(hg, i));
    fprintf(fp, "(");
    for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
      fprintf(fp, "%d ", hg->hvertex[j]);
    fprintf(fp, ")\n");
  }

  if (hg->ewgt != NULL) {
    for (j = 0; j < num_ewgt; j++) sum[j] = 0;
    fprintf(fp, "%s Hyperedge Weights:  [weights]\n", str);
    for (i = 0; i < hg->nEdge; i++) {
      fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "):  ", i, EDGE_LNO_TO_GNO(hg, i));
      fprintf(fp, "[");
      for (j = 0; j < num_ewgt; j++) {
        fprintf(fp, "%f ", hg->ewgt[i*num_ewgt + j]);
        sum[j] += hg->ewgt[i*num_ewgt + j];
      }
      fprintf(fp, "])\n");
    }
    fprintf(fp, "Total hyperedge weight = [");
    for (j = 0; j < num_ewgt; j++) fprintf(fp, "%f  ", sum[j]);
    fprintf(fp, "]\n");
  }

  ZOLTAN_FREE(&sum);
  ZOLTAN_TRACE_EXIT(zz, yo);
}
Ejemplo n.º 9
0
int Zoltan_HG_Info (
  ZZ *zz,
  HGraph *hg
)
{
  int i, dd, size, size_min, size_max, count;
  float wgt_min, wgt_max, wgt_tot;
  double mean, var, temp;
  char *yo = "Zoltan_HG_Info";

  ZOLTAN_TRACE_ENTER(zz, yo);

  printf("--------- HGraph Information (min/ave/max/tot) ------------------\n");
  printf("INFO MAY BE WRONG FOR 2D DATA DISTRIBUTION. KDDKDD \n");
  printf("info:%d |V|=%d |E|=%d |P|=%d \n", hg->info, hg->nVtx, hg->nEdge,
   hg->nPins);

  /* print weights */
  if (hg->nVtx && hg->vwgt) {
    for (dd = 0; dd < hg->VtxWeightDim; dd++) {
      wgt_tot = 0.0;
      wgt_min = FLT_MAX;
      wgt_max = FLT_MIN;
      for (i = 0; i < hg->nVtx; i++) {
        wgt_tot += hg->vwgt[i*hg->VtxWeightDim+dd];
        wgt_min = MIN(wgt_min, hg->vwgt[i*hg->VtxWeightDim+dd]);
        wgt_max = MAX(wgt_max, hg->vwgt[i*hg->VtxWeightDim+dd]);
      }
      printf("Vertex weights   :    %9.2f %9.2f %9.2f %12.2f\n", wgt_min,
       wgt_tot/hg->nVtx, wgt_max, wgt_tot);
  
      mean = var = 0.0;
      if (hg->nVtx > 1) {
        mean = wgt_tot / hg->nVtx;
        for (i = 0; i < hg->nVtx; i++) {
          temp = hg->vwgt[i*hg->VtxWeightDim+dd] - mean;
          var += (temp*temp);
        }
        var = sqrt(var/(hg->nVtx-1));
        printf ("Vertex Stats: stdev %.2f,   Coef of Var %.2f\n", var, var/mean);
      }
  
      count=0;
      temp=0.0;
      for (i = 0; i < hg->nVtx; i++)
        if (hg->vwgt[i*hg->VtxWeightDim+dd] > (mean + 3.0 * var)) {
          count++;
          temp += hg->vwgt[i*hg->VtxWeightDim+dd];
        }
      printf ("Vertex >3sigma: count %d, rel count x10000 %.1f, "
       "rel weight x100 %.1f\n",
       count, (float)10000*count/(float)hg->nVtx, 100.0*temp/wgt_tot);
    }
  }
  if (hg->nEdge && hg->ewgt) {
    wgt_tot = 0.0;
    wgt_min = FLT_MAX;
    wgt_max = FLT_MIN;
    for (i = 0; i < hg->nEdge; i++) {
      wgt_tot += hg->ewgt[i];
      wgt_min = MIN(wgt_min, hg->ewgt[i]);
      wgt_max = MAX(wgt_max, hg->ewgt[i]);
    }
    printf("HEdge weights    :    %9.2f %9.2f %9.2f %12.2f\n", wgt_min,
     wgt_tot/hg->nEdge, wgt_max, wgt_tot);

    if (hg->nEdge > 1) {
      var = 0.0;
      mean = wgt_tot / hg->nEdge;
      for (i = 0; i < hg->nEdge; i++) {
        temp = hg->ewgt[i] - mean;
        var += (temp*temp);
      }
      var = sqrt(var/(hg->nEdge-1));
      printf ("HEdge Stats: STDV %.2f,   Coef of Var %.2f\n", var, var/mean);
    }
  }

  /* print sizes */
  if (hg->nPins && hg->hindex) {
    size_min = INT_MAX;
    size_max = INT_MIN;
    for (i = 0; i < hg->nEdge; i++) {
      size     = hg->hindex[i+1] - hg->hindex[i];
      size_min = MIN(size_min, size);
      size_max = MAX(size_max, size);
    }
    printf("Edge sizes       :    %6d    %9.2f %6d    %9d\n", size_min,
     (float)hg->nPins / hg->nEdge, size_max, hg->nPins);

    if (hg->nEdge > 1) {
      var = 0.0;
      mean = (float)hg->nPins / hg->nEdge;
      for (i = 0; i < hg->nEdge; i++) {
        temp = (float)(hg->hindex[i+1]-hg->hindex[i]) - mean;
        var += (temp*temp);
      }
      var = sqrt(var/(hg->nEdge-1));
      printf ("Edge Stats: stdev %.2f,   Coef of Var %.2f\n", var, var/mean);
    }
  }
  if (hg->nPins && hg->vindex) {
    size_min = INT_MAX;
    size_max = INT_MIN;
    for (i = 0; i < hg->nVtx; i++) {
      size     = hg->vindex[i+1] - hg->vindex[i];
      size_min = MIN(size_min, size);
      size_max = MAX(size_max, size);
    }
    printf("Vertex sizes     :    %6d    %9.2f %6d    %9d\n", size_min,
     (float)hg->nPins / hg->nVtx, size_max, hg->nPins);

    if (hg->nVtx > 1) {
      var = 0.0;
      mean = (float)hg->nPins / hg->nVtx;
      for (i = 0; i < hg->nVtx; i++) {
        temp = (float)(hg->vindex[i+1]-hg->vindex[i]) - mean;
        var += (temp*temp);
      }
      var = sqrt(var/(hg->nVtx-1));
      printf ("Vertex Stats: stdev %.2f,  Coef of Var %.2f\n", var, var/mean);
    }
  }

  printf("-----------------------------------------------------------------\n");

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ZOLTAN_OK;
}
Ejemplo n.º 10
0
int Zoltan_HG_Create_Mirror (
  ZZ *zz,
  HGraph *hg
)
{
  int inlength, outlength;   /* input/output array lengths */
  int *index;         /* pointers to input information */
  int *outindex;
  int *data, *outdata;
  char *yo = "Zoltan_HG_Create_Mirror";

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* determine which data to "mirror" and set corresponding data pointers. */
  if (hg &&  (hg->nEdge == 0 || hg->hindex) && (hg->nPins == 0 || hg->hvertex)
   && !hg->vindex && !hg->vedge) {
    ZOLTAN_TRACE_DETAIL(zz, yo, "Have hindex; building vindex.");

    inlength  = hg->nEdge;
    outlength = hg->nVtx;
    index     = hg->hindex;
    data      = hg->hvertex;
    outindex  = hg->vindex = (int*) ZOLTAN_MALLOC((hg->nVtx+1) * sizeof(int));
    outdata   = hg->vedge  = (int*) ZOLTAN_MALLOC (hg->nPins * sizeof(int));

    if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
      Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->vindex, &hg->vedge);
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
  }
  else if (hg && (hg->nVtx == 0 || hg->vindex) && (hg->nPins == 0 || hg->vedge)
   && !hg->hindex && !hg->hvertex) {
    ZOLTAN_TRACE_DETAIL(zz, yo, "Have vindex; building hindex.");

    inlength  = hg->nVtx;
    outlength = hg->nEdge;
    index     = hg->vindex;
    data      = hg->vedge;
    outindex  = hg->hindex  = (int*) ZOLTAN_MALLOC((hg->nEdge+1) * sizeof(int));
    outdata   = hg->hvertex = (int*) ZOLTAN_MALLOC(hg->nPins * sizeof(int));

    if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) {
      Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->hindex, &hg->hvertex);
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
  }
  else {
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input error.");
    return ZOLTAN_FATAL;  /* unable to proceed */
  }

  Zoltan_HG_Mirror(inlength, index, data, 
                    outlength, outindex, outdata);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ZOLTAN_OK;
}
Ejemplo n.º 11
0
int Zoltan_HG_Check (
  ZZ *zz,
  HGraph *hg
)
{
  int i;
  int iedge;               /* iedge denotes an hyperedge */
  int j;                   /* j is the index of a vertex */
  int k;                   /* k is an index of hyperedge */
  int *check;
  char str[256];
  int err = ZOLTAN_OK;
  char *yo = "Zoltan_HG_Check";

  ZOLTAN_TRACE_ENTER(zz, yo);

  if ((hg->nEdge && !hg->hindex) ||
      (hg->nVtx && !hg->vindex) || 
      (hg->nPins && (!hg->vedge || !hg->hvertex))) {
    ZOLTAN_PRINT_WARN(zz->Proc, yo, "NULL arrays found");
    err = ZOLTAN_WARN;
    goto End;
  }

  if (hg->nPins) {
      check = (int*) ZOLTAN_MALLOC(((hg->nEdge > hg->nVtx) ? hg->nEdge : hg->nVtx)
   * sizeof(int));
  if (check == NULL) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to allocate memory.");
    err = ZOLTAN_MEMERR;
    goto End;
  }

  for (i = 0; i < hg->nEdge; i++)
    check[i] = -1;
  for (i = 0; i < hg->nVtx; i++)
    for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
      if (check[hg->vedge[j]] < i)
        check[hg->vedge[j]] = i;
      else {
        ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple hedge for same vertex.");
        err = ZOLTAN_WARN;
      }
  for (i = 0; i < hg->nVtx; i++)
    check[i] = -1;
  for (i = 0; i < hg->nEdge; i++)
    for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
      if (check[hg->hvertex[j]] < i)
        check[hg->hvertex[j]] = i;
      else {
        ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple vertex for same hedge.");
        err =  ZOLTAN_WARN;
      }
  ZOLTAN_FREE (&check);
  }

  for (i = 0; i < hg->VtxWeightDim * hg->nVtx; i++)
    if (hg->vwgt[i] < 0.0) {
      ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative vertex weight.");
      err = ZOLTAN_WARN;
    }

  for (i = 0; i < hg->EdgeWeightDim * hg->nEdge; i++)
    if (hg->ewgt[i] < 0.0) {
      ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative edge weight.");
      err = ZOLTAN_WARN;
    }

  if (hg->comm && hg->comm->nProc_x == 1) { 
    /* In 2D distribution, check makes sense only if proc has entire hedge */
    for (i = 0; i < hg->nEdge; i++)
      if ((hg->hindex[i+1] == hg->hindex[i])) {
        sprintf (str, "Found hyperedge with no vertices: "
         "edge %d has %d vtxs\n", i, (hg->hindex[i+1] - hg->hindex[i]));
        ZOLTAN_PRINT_WARN(zz->Proc, yo, str);
        err = ZOLTAN_WARN;
      }
  }

  for (i = 0; i < hg->nEdge; i++)
    for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
      if (hg->hvertex[j] < 0  ||  hg->hvertex[j] >= hg->nVtx) {
        ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found vertex out of range in hvertex.");
        err = ZOLTAN_WARN;
      }

  for (i = 0; i < hg->nVtx; i++)
    for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
      if (hg->vedge[j] < 0  ||  hg->vedge[j] >= hg->nEdge) {
        ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found edge out of range in vedge.");
        err = ZOLTAN_WARN;
      }

  /* starting from (hindex,hvertex), for each edge determine each associated
   * vertex. Then for each vertex lookup associated edges using (vindex, vedge)
   */
  for (iedge = 0; iedge < hg->nEdge; iedge++)
    for (j = hg->hindex[iedge]; j < hg->hindex[iedge+1]; j++) {
      /* for each hyperedge get index to vertices */
      for (k=hg->vindex[hg->hvertex[j]]; k<hg->vindex[hg->hvertex[j]+1]; k++)
        /* for each vertex of current hyperedge get index to hyperedges */
        if (hg->vedge[k] == iedge)    /* does it match with original edge? */
          break;
      if (k == hg->vindex[hg->hvertex[j]+1]) { 
        /* if no match was found then failure, else keep on */
        ZOLTAN_PRINT_WARN(zz->Proc, yo, "Inconsistent hvertex/vedge");
        err = ZOLTAN_WARN;                    
        break;
      }
    }

End:
  ZOLTAN_TRACE_EXIT(zz, yo);
  return err;
}
Ejemplo n.º 12
0
int Zoltan_Get_Coordinates(
  ZZ *zz, 
  int num_obj,               /* Input:  number of objects */
  ZOLTAN_ID_PTR global_ids,  /* Input:  global IDs of objects */
  ZOLTAN_ID_PTR local_ids,   /* Input:  local IDs of objects; may be NULL. */
  int *num_dim,              /* Output: dimension of coordinates */
  double **coords            /* Output: array of coordinates; malloc'ed by
                                        fn if NULL upon input. */
)
{
  char *yo = "Zoltan_Get_Coordinates";
  int i,j,rc;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;
  int alloced_coords = 0;
  ZOLTAN_ID_PTR lid;   /* Temporary pointers to local IDs; used to pass 
                          NULL to query functions when NUM_LID_ENTRIES == 0. */
  double dist[3];
  double im[3][3];
  double deg_ratio;
  double x;
  int order[3];
  int reduce_dimensions, d, axis_aligned;
  int target_dim;
  int ierr = ZOLTAN_OK;
  char msg[256];
  ZZ_Transform *tran;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Error check -- make sure needed callback functions are registered. */

  if (zz->Get_Num_Geom == NULL || 
     (zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and "
                       "either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN");
    goto End;
  }

  /* Get problem dimension. */

  *num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                       "Error returned from ZOLTAN_GET_NUM_GEOM_FN");
    goto End;
  }

  if (*num_dim < 0 || *num_dim > 3) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                       "Invalid dimension returned from ZOLTAN_NUM_GEOM_FN");
    goto End;
  }

  /* Get coordinates for object; allocate memory if not already provided. */

  if (*num_dim > 0 && num_obj > 0) {
    if (*coords == NULL) {
      alloced_coords = 1;
      *coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double));
      if (*coords == NULL) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error");
        goto End;
      }
    }

    if (zz->Get_Geom_Multi != NULL) {
      zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID,
                         num_obj, global_ids, local_ids, *num_dim, *coords,
                         &ierr);
      if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                           "Error returned from ZOLTAN_GET_GEOM_MULTI_FN");
        goto End;
      }
    }
    else {
      for (i = 0; i < num_obj; i++) {
        lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL);
        zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries,
                     global_ids + i*num_gid_entries, lid, 
                     (*coords) + i*(*num_dim), &ierr);
        if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                             "Error returned from ZOLTAN_GET_GEOM_FN");
          goto End;
        }
      }
    }
  }

  /*
   * For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the
   * center of mass and inertial matrix of the coordinates.  
   *
   * For 3D problems: If the geometry is "flat", transform the points so the
   * two primary directions lie along the X and Y coordinate axes and project 
   * to the Z=0 plane.  If in addition the geometry is "skinny", project to 
   * the X axis.  (This creates a 2D or 1D problem respectively.)
   *
   * For 2D problems: If the geometry is essentially a line, transform it's
   * primary direction to the X axis and project to the X axis, yielding a
   * 1D problem.
   *
   * Return these points to the partitioning algorithm, in effect partitioning
   * in only the 2 (or 1) significant dimensions.  
   */

  if (((*num_dim == 3) || (*num_dim == 2)) && 
      ((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){

    Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i);
    Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS", 
                     (void *)&reduce_dimensions);
    Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)&deg_ratio);

    i = 0;
    reduce_dimensions = 0;
    deg_ratio = 10.0;

    Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level,
                             zz->Proc, zz->Debug_Proc);

    if (reduce_dimensions == 0){
      goto End;
    }

    if (deg_ratio <= 1){
      if (zz->Proc == 0){
        ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0");
      }
      deg_ratio = 10.0;
    }

    if (zz->LB.Method == RCB){
      tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran);
    } 
    else if (zz->LB.Method == RIB){
      tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran);
    }
    else{
      tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran);
    }

    d = *num_dim;

    if (tran->Target_Dim >= 0){
      /*
       * On a previous load balancing call, we determined whether
       * or not the geometry was degenerate.  If the geometry was 
       * determined to be not degenerate, then we assume it is still 
       * not degenerate, and we skip the degeneracy calculation.  
       */
      if (tran->Target_Dim > 0){
        /*
         * The geometry *was* degenerate.  We test the extent
         * of the geometry along the principal directions determined
         * last time to determine if it is still degenerate with that
         * orientation.  If so, we transform the coordinates using the
         * same transformation we used last time.  If not, we do the 
         * entire degeneracy calculation again.
         */
 
        if ((tran->Axis_Order[0] >= 0) && 
            (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
          axis_aligned = 1;
        }
        else{
          axis_aligned = 0;
        }

        projected_distances(zz, *coords, num_obj, tran->CM, 
             tran->Evecs, dist, d, axis_aligned, tran->Axis_Order); 

        target_dim = get_target_dimension(dist, order, deg_ratio, d);

        if (target_dim > 0){
          transform_coordinates(*coords, num_obj, d, tran);
        }
        else{
          /* Set's Target_Dim to -1, flag to recompute degeneracy */
          Zoltan_Initialize_Transformation(tran);
        }
      }
    }

    if (tran->Target_Dim < 0){

      tran->Target_Dim = 0;

      /*
       * Get the center of mass and inertial matrix of coordinates.  Ignore
       * vertex weights, we are only interested in geometry.  Global operation.
       */
      if (d == 2){
        inertial_matrix2D(zz, *coords, num_obj, tran->CM, im);
      }
      else{
        inertial_matrix3D(zz, *coords, num_obj, tran->CM, im);
      }

      /*
       * The inertial matrix is a 3x3 or 2x2 real symmetric matrix.  Get its
       * three or two orthonormal eigenvectors.  These usually indicate the 
       * orientation of the geometry.
       */

      rc = eigenvectors(im, tran->Evecs, d);

      if (rc){
        if (zz->Proc == 0){
          ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed");
        }
        goto End; 
      }

      /*
       * Here we check to see if the eigenvectors are very close
       * to the coordinate axes.  If so, we can more quickly
       * determine whether the geometry is degenerate, and also more
       * quickly transform the geometry to the lower dimensional
       * space.
       */

      axis_aligned = 0;

      for (i=0; i<d; i++){
        tran->Axis_Order[i] = -1;
      }

      for (j=0; j<d; j++){
        for (i=0; i<d; i++){
          x = fabs(tran->Evecs[i][j]);

          if (NEAR_ONE(x)){
            tran->Axis_Order[j] = i;  /* e'vector j is very close to i axis */
            break;
          }
        }
        if (tran->Axis_Order[j] < 0){
          break;
        }
      }

      if ((tran->Axis_Order[0] >= 0) && 
          (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
        axis_aligned = 1;
      }

      /*
       * Calculate the extent of the geometry along the three lines defined
       * by the direction of the eigenvectors through the center of mass.
       */

      projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist, 
                          d, axis_aligned, tran->Axis_Order); 

      /*
       * Decide whether these distances indicate the geometry is
       * very flat in one or two directions.
       */

      target_dim = get_target_dimension(dist, order, deg_ratio, d);

      if (target_dim > 0){
        /*
         * Yes, geometry is degenerate
         */
        if ((zz->Debug_Level > 0) && (zz->Proc == 0)){
          if (d == 2){
            sprintf(msg,
             "Geometry (~%f x %f), exceeds %f to 1.0 ratio",
              dist[order[0]], dist[order[1]], deg_ratio);
          }
          else{
            sprintf(msg,
             "Geometry (~%f x %f x %f), exceeds %f to 1.0 ratio",
              dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio);
          }

          ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
          sprintf(msg, "We'll treat it as %d dimensional",target_dim);
          ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
        }

        if (axis_aligned){
          /*
          ** Create new geometry, transforming the primary direction
          ** to the X-axis, and the secondary to the Y-axis.
          */

          tran->Permutation[0] = tran->Axis_Order[order[0]];
          if (target_dim == 2){
            tran->Permutation[1] = tran->Axis_Order[order[1]];
          }
        }
        else{
          /*
           * Reorder the eigenvectors (they're the columns of evecs) from 
           * longest projected distance to shorted projected distance.  Compute
           * the transpose (the inverse) of the matrix.  This will transform
           * the geometry to align along the X-Y plane, or along the X axis. 
           */
  
          for (i=0; i< target_dim; i++){
            tran->Transformation[i][2] = 0.0;
            for (j=0; j<d; j++){
              tran->Transformation[i][j] = tran->Evecs[j][order[i]];

            }
          }
          for (i=target_dim; i< 3; i++){
            for (j=0; j<3; j++){
              tran->Transformation[i][j] = 0.0;
            }
          }
        }

        tran->Target_Dim = target_dim;

        transform_coordinates(*coords, num_obj, d, tran);

      } /* If geometry is very flat */
    }  /* If REDUCE_DIMENSIONS is true */
  } /* If 2-D or 3-D rcb, rib or hsfc */

End:
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned.");
    if (alloced_coords) ZOLTAN_FREE(coords);
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
  return ierr;
}
Ejemplo n.º 13
0
/* Point drop for refinement after above partitioning */
int Zoltan_HSFC_Point_Assign (
   ZZ *zz, 
   double *x, 
   int *proc,
   int *part)
   {
   double     scaled[3];
   double     pt[3];
   double     fsfc;
   Partition *p;
   int        i;
   int        dim;
   HSFC_Data *d;
   int        err;
   char *yo = "Zoltan_HSFC_Point_Assign";

   ZOLTAN_TRACE_ENTER (zz, yo);
   d = (HSFC_Data *) zz->LB.Data_Structure;
   if (d == NULL)
      ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL,
       "No Decomposition Data available; use KEEP_CUTS parameter.");

   for (i=0; i<d->ndimension; i++){
     pt[i] = x[i];  /* we don't want to change caller's "x" */
   }

   if (d->tran.Target_Dim > 0){   /* degenerate geometry */
     dim = d->tran.Target_Dim;
     Zoltan_Transform_Point(pt, d->tran.Transformation, d->tran.Permutation,
       d->ndimension, dim, pt);
   }
   else{
     dim = d->ndimension;
   }

   /* Calculate scaled coordinates, calculate HSFC coordinate */
   for (i = 0; i < dim; i++)
      {
      scaled[i] = (pt[i] - d->bbox_lo[i]) / d->bbox_extent[i];
      if (scaled[i] < HSFC_EPSILON)         scaled[i] = HSFC_EPSILON;
      if (scaled[i] > 1.0 - HSFC_EPSILON)   scaled[i] = 1.0 - HSFC_EPSILON;
      }
   fsfc = d->fhsfc (zz, scaled);           /* Note, this is a function call */

   /* Find partition containing point and return its number */
   p = (Partition *) bsearch (&fsfc, d->final_partition, zz->LB.Num_Global_Parts,
    sizeof (Partition), Zoltan_HSFC_compare);

   if (p == NULL)
      ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL, "programming error, shouldn't happen");
   if (part != NULL) {
      if (zz->LB.Remap)
         *part = zz->LB.Remap[p->index];
      else
         *part = p->index;
      }
   if (proc != NULL) {
      if (zz->LB.Remap) 
         *proc = Zoltan_LB_Part_To_Proc(zz, zz->LB.Remap[p->index], NULL);
      else
         *proc = Zoltan_LB_Part_To_Proc(zz, p->index, NULL);
      }
   err = ZOLTAN_OK;

End:
   ZOLTAN_TRACE_EXIT (zz, yo);
   return err;
   }
Ejemplo n.º 14
0
int Zoltan_LocalHSFC_Order(
			   ZZ *zz,               /* Zoltan structure */
			   int num_obj,          /* Number of (local) objects to order. */
			   ZOLTAN_ID_PTR gids,   /* List of global ids (local to this proc) */
			                         /* The application must allocate enough space */
			   ZOLTAN_ID_PTR lids,   /* List of local ids (local to this proc) */
			                         /* The application must allocate enough space */
			   int *rank,            /* rank[i] is the rank of gids[i] */
			   int *iperm,
			   ZOOS *order_opt       /* Ordering options, parsed by Zoltan_Order */
                           )
{

  static char *yo = "Zoltan_LocalHSFC_Order";

  int n, ierr=ZOLTAN_OK;

  double (*fhsfc)(ZZ*, double*);  /* space filling curve function */

  int wgt_dim=0; 
  float *obj_wgts=0;
  int *parts=0;

  int numGeomDims=0;
  double *geomArray=0;

  /* Variables for bounding box */
  double *minValInDim;
  double *maxValInDim;
  double *widthDim;


  double *hsfcKey=0;
  int *coordIndx=0;

  /* Counters */
  int objNum;
  int dimNum;

  int offset=0;

  int myrank;
  MPI_Comm_rank(MPI_COMM_WORLD,&myrank);

  ZOLTAN_TRACE_ENTER(zz, yo);

  /******************************************************************/
  /* If for some reason order_opt is NULL, allocate a new ZOOS here. */
  /* This should really never happen. */
  /******************************************************************/
  if (!order_opt)
  {
    order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
    strcpy(order_opt->method,"LOCAL_HSFC");
  }
  /******************************************************************/

  /* local HSFC only computes the rank vector */
  order_opt->return_args = RETURN_RANK; 


  /******************************************************************/
  /* Check that num_obj equals the number of objects on this proc. */
  /* This constraint may be removed in the future. */
  /******************************************************************/
  n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
  if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
  {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
    return(ZOLTAN_FATAL);
  }
  if (n != num_obj)
  {
    /* Currently this is a fatal error. */
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the number of objects.");
    return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Get lists of objects                                           */
  /******************************************************************/
  ierr = Zoltan_Get_Obj_List(zz, &n, &gids, &lids, wgt_dim, &obj_wgts, &parts);
  if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN))
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Obj_List returned error.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Get geometry for objects*/
  /******************************************************************/
  ierr = Zoltan_Get_Coordinates(zz, n, gids, lids, &numGeomDims,
			       &geomArray);
  if (ierr != 0)
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Zoltan_Get_Coordinates returned error.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Place coords in bounding box                                   */
  /******************************************************************/
  minValInDim =  (double *) malloc(numGeomDims * sizeof (double));
  maxValInDim =  (double *) malloc(numGeomDims * sizeof (double));
  widthDim =  (double *) malloc(numGeomDims * sizeof (double));

  for(dimNum=0; dimNum<numGeomDims; dimNum++)
  {
    minValInDim[dimNum] = HUGE_VAL;
    maxValInDim[dimNum] = -HUGE_VAL;
  }

  /*************************************************************/
  /* Determine min, max, and width for each dimension          */
  /*************************************************************/
  for (objNum=0; objNum<n; objNum++)
  {
    for(dimNum=0; dimNum<numGeomDims; dimNum++)
    {
      if (geomArray[objNum * numGeomDims + dimNum] < minValInDim[dimNum])
      {
        minValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
      }
      if (geomArray[objNum * numGeomDims + dimNum] > maxValInDim[dimNum])
      {
        maxValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum];
      }
    }
  }

  for(dimNum=0; dimNum<numGeomDims; dimNum++)
  {
    widthDim[dimNum] = maxValInDim[dimNum] - minValInDim[dimNum]; 
  }
  /*************************************************************/

  /*************************************************************/
  /* Rescale values to fit in bounding box                     */
  /*************************************************************/
  for (objNum=0; objNum<n; objNum++)
  {
    for(dimNum=0; dimNum<numGeomDims; dimNum++)
    {
      geomArray[objNum * numGeomDims + dimNum] -= minValInDim[dimNum];
      geomArray[objNum * numGeomDims + dimNum] /= widthDim[dimNum];
    }
  }
  /*************************************************************/

  free(minValInDim); minValInDim=0;
  free(maxValInDim); maxValInDim=0;
  free(widthDim); widthDim=0;
  /******************************************************************/

  /******************************************************************/   
  /* Specify which HSFC function to use (based on dim) */
  /******************************************************************/
  if (numGeomDims==1)
  {
    fhsfc = Zoltan_HSFC_InvHilbert1d;
  }
  else if (numGeomDims==2)
  {
    fhsfc = Zoltan_HSFC_InvHilbert2d;
  }
  else if (numGeomDims==3)
  {
    fhsfc = Zoltan_HSFC_InvHilbert3d;
  }
  else /* this error should have been previously caught */
  {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Geometry should be of dimension 1, 2, or 3.");
      return(ZOLTAN_FATAL);
  }
  /******************************************************************/

  /******************************************************************/
  /* Generate hsfc keys and indices to be sorted                    */
  /******************************************************************/
  hsfcKey = (double *) malloc(n * sizeof (double));
  coordIndx = (int *) malloc(n *sizeof(int));
  for (objNum=0; objNum<n; objNum++)
  {
    hsfcKey[objNum] = fhsfc(zz, &(geomArray[objNum * numGeomDims]) );
    coordIndx[objNum] = objNum;
  }
  /******************************************************************/

  /******************************************************************/
  /* Sort indices based on keys                                     */
  /******************************************************************/
  Zoltan_quicksort_pointer_dec_double (coordIndx, hsfcKey, 0, n-1);
  /******************************************************************/


  /******************************************************************/
  /* get ranks                                                      */
  /******************************************************************/

  /******************************************************/
  /* Determine offsets                                  */
  /******************************************************/
  MPI_Scan(&n, &offset, 1, MPI_INT, MPI_SUM, zz->Communicator);
  offset -= n; /* MPI_Scan is inclusive, so subtract off local size */
  /******************************************************/

  for(objNum=0; objNum<n; objNum++)
  {
    /*MMW temporary hack to make Cedric's interface give me want I need */
    /*rank[coordIndx[objNum]] = objNum + offset; */
    rank[objNum] = coordIndx[objNum] + offset; 
  }

  /******************************************************************/

  /* iperm is to be deprecated so not calculated*/

  free(hsfcKey);
  free(coordIndx);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ZOLTAN_OK);

}
Ejemplo n.º 15
0
int Zoltan_Preprocess_Graph(
  ZZ *zz,                               /* Zoltan structure */
  ZOLTAN_ID_PTR *global_ids,
  ZOLTAN_ID_PTR *local_ids,
  ZOLTAN_Third_Graph *gr,              /* Graph for third part libs */
  ZOLTAN_Third_Geom  *geo,
  ZOLTAN_Third_Part  *prt,
  ZOLTAN_Third_Vsize *vsp
)
{
  static char *yo = "Zoltan_Preprocess_Graph";
  int ierr;
  float *float_vwgt, *float_ewgts;
  char msg[256];


  char add_obj_weight[MAX_PARAM_STRING_LEN+1];

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Initialize all local pointers to NULL. This is necessary
   * because we free all non-NULL pointers upon errors.
   */
  gr->vtxdist = gr->xadj = gr->adjncy = NULL;
  gr->vwgt = gr->ewgts = NULL;
  float_vwgt = float_ewgts = NULL;

  if (gr->obj_wgt_dim >= 0) {
    /* Check weight dimensions */
    if (zz->Obj_Weight_Dim<0){
      sprintf(msg, "Object weight dimension is %d, "
	      "but should be >= 0. Using Obj_Weight_Dim = 0.",
	      zz->Obj_Weight_Dim);
      ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
      gr->obj_wgt_dim = 0;
    }
    else {
      gr->obj_wgt_dim = zz->Obj_Weight_Dim;
    }
  }
  else
    gr->obj_wgt_dim = 0;
  if (gr->edge_wgt_dim >= 0) {
    if (zz->Edge_Weight_Dim<0){
      sprintf(msg, "Edge weight dimension is %d, "
	      "but should be >= 0. Using Edge_Weight_Dim = 0.",
	      zz->Edge_Weight_Dim);
      ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
      gr->edge_wgt_dim = 0;
    }
    else if (zz->Edge_Weight_Dim>1){
      ZOLTAN_PRINT_WARN(zz->Proc, yo, "This method does not support "
			"multidimensional edge weights. Using Edge_Weight_Dim = 1.");
      gr->edge_wgt_dim = 1;
    }
    else {
      gr->edge_wgt_dim = zz->Edge_Weight_Dim;
    }
  }
  else
      gr->edge_wgt_dim = 0;

  if (gr->graph_type >= 0)
    /* Default graph type is GLOBAL. */
    gr->graph_type = GLOBAL_GRAPH;
  else
    gr->graph_type = - gr->graph_type;

  /* Get parameter options shared by ParMetis and Jostle */
  gr->check_graph = 1;          /* default */
  gr->scatter = 1;              /* default */
  gr->final_output = 0;
  strcpy(add_obj_weight, "NONE");  /* default */
  Zoltan_Bind_Param(Graph_params, "CHECK_GRAPH", (void *) &gr->check_graph);
  Zoltan_Bind_Param(Graph_params, "SCATTER_GRAPH", (void *) &gr->scatter);
  Zoltan_Bind_Param(Graph_params, "FINAL_OUTPUT", (void *) &gr->final_output);
  Zoltan_Bind_Param(Graph_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight);
  Zoltan_Assign_Param_Vals(zz->Params, Graph_params, zz->Debug_Level, zz->Proc,
			   zz->Debug_Proc);

  /* If reorder is true, we already have the id lists. Ignore weights. */
  if ((*global_ids == NULL) || (!gr->id_known)){
    int * input_part;
    ierr = Zoltan_Get_Obj_List(zz, &gr->num_obj, global_ids, local_ids,
			       gr->obj_wgt_dim, &float_vwgt, &input_part);
    if (prt) {
      prt->input_part = input_part;
    }
    else { /* Ordering, dont need part */
      ZOLTAN_FREE(&input_part);
    }
    if (ierr){
      /* Return error */
      ZOLTAN_PARMETIS_ERROR(ierr, "Get_Obj_List returned error.");
    }
  }


  /* Build Graph for third party library data structures, or just get vtxdist. */
  ierr = Zoltan_Build_Graph(zz, gr->graph_type, gr->check_graph, gr->num_obj,
			    *global_ids, *local_ids, gr->obj_wgt_dim, gr->edge_wgt_dim,
			    &gr->vtxdist, &gr->xadj, &gr->adjncy, &float_ewgts, &gr->adjproc);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      ZOLTAN_PARMETIS_ERROR(ierr, "Zoltan_Build_Graph returned error.");
  }

  if (prt) {
    prt->part_sizes = prt->input_part_sizes;
    if (gr->num_obj >0) {
      prt->part = (indextype *)ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
      if (!prt->part){
	/* Not enough memory */
	ZOLTAN_PARMETIS_ERROR(ZOLTAN_MEMERR, "Out of memory.");
      }
      memcpy (prt->part, prt->input_part, (gr->num_obj) * sizeof(indextype));
    }
    else {
      prt->input_part = prt->part = NULL;
    }
  }

  /* Convert from float. */

  /* Get vertex weights if needed */
  if (gr->obj_wgt_dim){
    ierr = Zoltan_Preprocess_Scale_Weights (gr, float_vwgt, &gr->vwgt,
					    gr->num_obj, gr->obj_wgt_dim, 1, zz,
					    "vertex", gr->vtxdist[zz->Proc]);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      /* Return error code */
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
    }
    ZOLTAN_FREE(&float_vwgt);
  }

  if (strcasecmp(add_obj_weight, "NONE")){
    if (Zoltan_Preprocess_Add_Weight(zz, gr, prt, add_obj_weight) != ZOLTAN_OK)
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in adding  vertex weights.");
  }

  /* Get edge weights if needed */
  if (gr->get_data)
    gr->num_edges = gr->xadj[gr->num_obj];
  else {
    gr->num_edges = 0;
    gr->edge_wgt_dim = 0;
  }

  if (gr->edge_wgt_dim){
    ierr = Zoltan_Preprocess_Scale_Weights (gr, float_ewgts, &gr->ewgts,
					    gr->num_edges, gr->edge_wgt_dim, 1, zz,
					    "edge", 0);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
      /* Return error code */
      ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
    }
    if (!gr->final_output)
      ZOLTAN_FREE(&float_ewgts);
    else
      gr->float_ewgts = float_ewgts;
  }
  else
    ZOLTAN_FREE(&float_ewgts);

  if (geo){
    ierr = Zoltan_Preprocess_Extract_Geom (zz, global_ids, local_ids, gr, geo);
    if (ierr) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			    "Error returned from Zoltan_Preprocess_Extract_Geom");
    }
  }


  if (vsp) {
    ierr = Zoltan_Preprocess_Extract_Vsize (zz, global_ids, local_ids, gr, vsp);
    if (ierr) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			  "Error returned from Zoltan_Preprocess_Extract_Vsize");
    }
  }

  /* Scatter graph?
   * If data distribution is highly imbalanced, it is better to
   * redistribute the graph data structure before calling ParMetis.
   * After partitioning, the results must be mapped back.
   */
  if (gr->scatter < gr->scatter_min) gr->scatter = gr->scatter_min;

  if (gr->scatter>0) {
    ierr = Zoltan_Preprocess_Scatter_Graph (zz, gr, prt, geo, vsp);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
			    "Error returned from Zoltan_Preprocess_Scatter_Graph");
    }
  }


  /* Verify that graph is correct */
  if (gr->get_data){
    int flag;

    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
      flag = 2; /* Maximum output level */
    else
      flag = 1; /* Medium output level */
    ierr = Zoltan_Verify_Graph(zz->Communicator, gr->vtxdist, gr->xadj, gr->adjncy, gr->vwgt,
	      gr->ewgts, gr->obj_wgt_dim, gr->edge_wgt_dim, gr->graph_type, gr->check_graph, flag);

  }

 End:

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 16
0
int Zoltan_Migrate(
  ZZ *zz,                      /* Zoltan structure.                  */
  int num_import,              /* Number of non-local objects assigned to the 
                                  processor in the new decomposition.        */
  ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects 
                                  assigned to this processor in the new
                                  decomposition; this field can be NULL if 
                                  the application doesn't provide import IDs.*/
  ZOLTAN_ID_PTR import_local_ids,  /* Array of local IDs for non-local objects
                                  assigned to the processor in the new
                                  decomposition; this field can be NULL if the 
                                  application does not provide import IDs.   */
  int *import_procs,           /* Array of processor IDs of processors owning
                                  the non-local objects that are assigned to
                                  this processor in the new decomposition; this
                                  field can be NULL if the application does
                                  not provide import IDs.                    */
  int *import_to_part,         /* Array of partition numbers to which imported
                                  objects should be assigned.                */
  int num_export,              /* Number of objs to be exported
                                  to other processors to establish the new
                                  decomposition.                             */
  ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of
                                  objects to be exported to other processors
                                  to establish the new decomposition.        */
  ZOLTAN_ID_PTR export_local_ids,  /* Array of local IDs of
                                  objects to be exported to other processors
                                  to establish the new decomposition.        */
  int *export_procs,           /* Array of processor IDs
                                  to which objects will be exported 
                                  to establish the new decomposition.        */
  int *export_to_part          /* Array of partition numbers to which exported
                                  objects should be assigned.                */
)
{
/*
 *  Routine to help perform migration.  If migration pre-processing routine
 *  (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn.
 *  It then calls a function to obtain the size of the migrating objects
 *  (ZOLTAN_OBJ_SIZE_FN).  The routine next calls an application-specified
 *  object packing routine (ZOLTAN_PACK_OBJ_FN) for each object
 *  to be exported.  It develops the needed communication map to move the
 *  objects to other processors.  It performs the communication according
 *  to the map, and then calls an application-specified object unpacking
 *  routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported.
 */

char *yo = "Zoltan_Migrate";
int num_gid_entries, num_lid_entries;  /* lengths of global & local ids */
int *sizes = NULL;       /* sizes (in bytes) of the object data for export. */
int id_size;             /* size (in bytes) of ZOLTAN_GID + padding for 
                            alignment                                       */
int tag_size;            /* size (in bytes) of ZOLTAN_GID + one int 
                            (for message size) */
char *export_buf = NULL; /* buffer for packing export data.                 */
char *import_buf = NULL; /* buffer for receiving imported data.             */
char *tmp;               /* temporary pointer into buffers.                 */
int i;                   /* loop counter.                                   */
int tmp_size;            /* size of a single object's data.                 */
int *idx = NULL;         /* index used for multi-fn packs and unpacks.      */
int idx_cnt = 0;         /* index counter for idx array.                    */
ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm  
                                buf  */
ZOLTAN_ID_PTR lid;       /* temporary pointer to a local ID; used to pass
                            NULL to query functions when NUM_LID_ENTRIES=0. */
ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */
ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */
int msgtag, msgtag2;     /* Tags for communication routines                 */
int total_send_size;     /* Total size of outcoming message (in #items)     */
int total_recv_size;     /* Total size of incoming message (in #items)      */
int aligned_int;         /* size of an int padded for alignment             */
int dest;                /* temporary destination partition.                */
int include_parts = 0;   /* flag indicating whether partition info is
                            provided */
int ierr = ZOLTAN_OK;
int actual_num_exp = 0;
int actual_exp_allocated = 0;
ZOLTAN_ID_PTR actual_exp_gids = NULL;    /* Arrays containing only objs to  */
ZOLTAN_ID_PTR actual_exp_lids = NULL;    /* actually be packed.  Objs that  */
int *actual_exp_procs = NULL;            /* are changing partition but not  */
int *actual_exp_to_part = NULL;          /* processor may not be included.  */
int actual_num_imp = 0;
int actual_imp_allocated = 0;
ZOLTAN_ID_PTR actual_imp_gids = NULL;    /* Arrays containing only objs to  */
ZOLTAN_ID_PTR actual_imp_lids = NULL;    /* actually be imported. Objs that  */
int *actual_imp_procs = NULL;            /* are changing partition but not  */
int *actual_imp_to_part = NULL;          /* processor may not be included.  */

  ZOLTAN_TRACE_ENTER(zz, yo);

  /*
   *  Return if this processor is not in the Zoltan structure's
   *  communicator.
   */

  if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
    goto End;
  }

  /*
   *  Check that all procs use the same id types.
   */

  ierr = check_input(zz, 
                    ((num_export >= 0 && export_to_part) || 
                     (num_import >= 0 && import_to_part)),
                     &include_parts);
  if (ierr != ZOLTAN_OK) 
    goto End;

  num_gid_entries = zz->Num_GID;
  num_lid_entries = zz->Num_LID;

  /*
   *  Check that all necessary query functions are available.
   */

  if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
           "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function "
           "to use the migration-help tools.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
           "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function "
           "to use the migration-help tools.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a "
         "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function "
         "to use the migration-help tools.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }


  if (num_export >= 0) {

    /* Build the actual export arrays */
    ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
                         num_export, export_global_ids, export_local_ids, 
                         export_procs, export_to_part, 
                         &actual_num_exp, &actual_exp_gids, &actual_exp_lids,
                         &actual_exp_procs, &actual_exp_to_part,
                         &actual_exp_allocated);
    if (ierr < 0) 
      goto End;

    /* Compute communication map based on actual exports.  */

    msgtag = 32767;
    ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs,
                              zz->Communicator, msgtag, &actual_num_imp);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
      goto End;
    }
  }

  else if (num_import >= 0) {

    /* Build the actual import arrays */
    ierr = actual_arrays(zz, num_gid_entries, num_lid_entries,
                         num_import, import_global_ids, import_local_ids, 
                         import_procs, import_to_part, 
                         &actual_num_imp, &actual_imp_gids, &actual_imp_lids,
                         &actual_imp_procs, &actual_imp_to_part,
                         &actual_imp_allocated);
    if (ierr < 0) 
      goto End;
    
    /* Compute communication map based on imports.  */
    msgtag = 32767;
    ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs,
                              zz->Communicator, msgtag, &actual_num_exp);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create.");
      goto End;
    }

    /* Compute actual export lists for packing objects */
    if (actual_num_exp > 0) {
      actual_exp_allocated = 1;
      actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp);
      actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp);
      actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp);
      if (include_parts)
        actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp);
      if (actual_exp_gids == NULL ||
          (num_lid_entries && actual_exp_lids == NULL) ||
          actual_exp_procs == NULL || 
          (import_to_part != NULL && actual_exp_to_part == NULL)) {
        Zoltan_Multifree(__FILE__, __LINE__, 4, 
                         &actual_exp_gids, &actual_exp_lids, 
                         &actual_exp_procs, &actual_exp_to_part);
        ierr = ZOLTAN_MEMERR;
        goto End;
      }
    }

    msgtag2 = 32766;
    ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids,
                    (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)),
                    (char *) actual_exp_gids);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
      goto End;
    }

    if (num_lid_entries) {
      msgtag2--;
      ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids,
                      (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries),
                      (char *) actual_exp_lids);
      if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
        goto End;
      }
    }

    Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL,
                     NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL);
 
    if (include_parts) {
      msgtag2--;
      ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part,
                      (int) sizeof(int), (char *) actual_exp_to_part);
      if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
        goto End;
      }
    }
   
    /* Create inverse plan (i.e., plan based on exports) so can set 
     * variable sizes. 
     * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable 
     * but does not tell how large to allocate receive buffer.
     */
    ierr = Zoltan_Comm_Invert_Plan(&imp_plan);
    exp_plan = imp_plan;
    imp_plan = NULL;
  }
  else {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  if (zz->Migrate.Pre_Migrate_PP != NULL) {
    zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data,
                            num_gid_entries, num_lid_entries,
                            num_import, import_global_ids,
                            import_local_ids, import_procs, import_to_part,
                            num_export, export_global_ids,
                            export_local_ids, export_procs, export_to_part,
                            &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_PRE_MIGRATE_PP_FN function.");
      goto End;
    }
  }

  if (zz->Migrate.Pre_Migrate != NULL) {
    zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data,
                            num_gid_entries, num_lid_entries,
                            num_import, import_global_ids,
                            import_local_ids, import_procs, 
                            num_export, export_global_ids,
                            export_local_ids, export_procs,
                            &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_PRE_MIGRATE_FN function.");
      goto End;
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing");

  id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE));
  /* Note that alignment is not strictly necessary 
     when ZOLTAN_ID_TYPE is int or unsigned int. */
  aligned_int = Zoltan_Align(sizeof(int));
  tag_size = id_size + aligned_int;

  /*
   * For each object, allow space for its global ID and its data plus 
   * one int (for the object data size).
   * Zoltan will pack the global IDs; the application must pack the data
   * through the pack routine.  Zoltan needs the global IDs for unpacking,
   * as the order of the data received during communication is not 
   * necessarily the same order as import_global_ids[].
   * Zoltan also needs to communicate the sizes of the objects because
   * only the sender knows the size of each object.
   */
  if (actual_num_exp > 0) {
    sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
    if (!sizes) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }

    if (zz->Get_Obj_Size_Multi != NULL) {
      zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, 
                             num_gid_entries, num_lid_entries, actual_num_exp,
                             actual_exp_gids, actual_exp_lids, sizes, &ierr);
      if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                        "ZOLTAN_OBJ_SIZE_MULTI function.");
        goto End;
      }
    }
    else {
      for (i = 0; i < actual_num_exp; i++){
        lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
        sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, 
                       num_gid_entries, num_lid_entries,
                       &(actual_exp_gids[i*num_gid_entries]), 
                       lid, &ierr);
        if (ierr < 0) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                          "ZOLTAN_OBJ_SIZE function.");
          goto End;
        }
      }
    }

    total_send_size = 0;

    for (i = 0; i < actual_num_exp; i++) {
      sizes[i] = Zoltan_Align(sizes[i]);
      total_send_size += sizes[i] + tag_size;
    }
    export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char));
    if (!export_buf) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }

    if (zz->Pack_Obj_Multi != NULL) {
      /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */
      idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int));
      if (!idx) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
        ierr = ZOLTAN_MEMERR;
        goto End;
      }
    }

    /*
     *  Pack the objects for export.
     */
  
    idx_cnt = 0;
    tmp = export_buf;
    for (i = 0; i < actual_num_exp; i++) {

      /* Pack the object's global ID */
      tmp_id = (ZOLTAN_ID_PTR) tmp;
      ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries]));
      tmp += id_size;
    
      /* Pack the object's size */
      *((int *)tmp) = sizes[i];
      tmp += aligned_int;

      /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */
      idx_cnt += tag_size;
      if (idx != NULL) {
        idx[i] = idx_cnt;
      }
      tmp += sizes[i];
      idx_cnt += sizes[i];
    }

    if (zz->Pack_Obj_Multi != NULL) {
      if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
        printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", 
               zz->Proc, yo);
      }
      zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data,
                         num_gid_entries, num_lid_entries, actual_num_exp,
                         actual_exp_gids, actual_exp_lids, 
                         (actual_exp_to_part!=NULL ? actual_exp_to_part 
                                                   : actual_exp_procs),
                         sizes, idx, export_buf, &ierr);
      if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                        "ZOLTAN_PACK_OBJ_MULTI function.");
        goto End;
      }
    }
    else {
      tmp = export_buf + tag_size;
      for (i = 0; i < actual_num_exp; i++) {
        if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
          printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo);
          ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries]));
          printf("size = %d bytes\n", sizes[i]); 
        }

        /* Pack the object's data */
        lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL);
        dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] 
                                           : actual_exp_procs[i]);
        zz->Pack_Obj(zz->Pack_Obj_Data, 
                           num_gid_entries, num_lid_entries,
                           &(actual_exp_gids[i*num_gid_entries]), lid, dest,
                           sizes[i], tmp, &ierr);
        if (ierr < 0) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                          "ZOLTAN_PACK_OBJ function.");
          goto End;
        }
        tmp += sizes[i] + tag_size;
      }
    }
    ZOLTAN_FREE(&idx);
    tmp_id = NULL;
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects");


  /* Modify sizes[] to contain message sizes, not object sizes */
  for (i=0; i<actual_num_exp; i++) {
    sizes[i] += tag_size;
  }

  msgtag--;
  ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size);
  if (ierr < 0) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize.");
    goto End;
  }

  if (actual_num_imp > 0) {
    import_buf = (char *) ZOLTAN_MALLOC(total_recv_size);
    if (!import_buf) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }
  }

  /*
   *  Send the export data using the communication plan.
   */

  msgtag2 = 32765;
  ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf);
  if (ierr < 0) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    goto End;
  }

  /*
   *  Free whatever memory we can.
   */

  Zoltan_Comm_Destroy(&exp_plan);
  ZOLTAN_FREE(&export_buf);
  ZOLTAN_FREE(&sizes);

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication");

  /* 
   *  Perform application-specified processing before unpacking the data.
   */
  if (zz->Migrate.Mid_Migrate_PP != NULL) {
    zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data,
                            num_gid_entries, num_lid_entries,
                            num_import, import_global_ids,
                            import_local_ids, import_procs, import_to_part,
                            num_export, export_global_ids,
                            export_local_ids, export_procs, export_to_part,
                            &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_MID_MIGRATE_PP_FN function.");
      goto End;
    }
  }

  if (zz->Migrate.Mid_Migrate != NULL) {
    zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data,
                            num_gid_entries, num_lid_entries,
                            num_import, import_global_ids,
                            import_local_ids, import_procs,
                            num_export, export_global_ids,
                            export_local_ids, export_procs,
                            &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_MID_MIGRATE_FN function.");
      goto End;
    }
  }

  /*
   *  Unpack the object data.
   */

  if (actual_num_imp > 0) {

    if (zz->Unpack_Obj_Multi != NULL) {

      /* Allocate and fill input arrays for Unpack_Obj_Multi. */
      sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
      tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp);
      idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int));
      if (!sizes || !tmp_id || !idx) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
        ierr = ZOLTAN_MEMERR;
        goto End;
      }

      tmp = import_buf;
      idx_cnt = 0;
      for (i = 0; i < actual_num_imp; i++) {

        /* Unpack the object's global ID */
        ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp);
        tmp += id_size;

        /* Unpack the object's size */
        sizes[i] = *((int *)tmp);
        tmp += aligned_int;

        /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */
        idx_cnt += tag_size;
        if (idx != NULL) {
          idx[i] = idx_cnt;
        }

        tmp += sizes[i];
        idx_cnt += sizes[i];
      }

      if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
        printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n",
               zz->Proc,yo);
      }
      zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries,
                         actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr);
      ZOLTAN_FREE(&import_buf);
      ZOLTAN_FREE(&sizes);
      ZOLTAN_FREE(&tmp_id);
      ZOLTAN_FREE(&idx);
      if (ierr < 0) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                        "ZOLTAN_UNPACK_OBJ_MULTI_FN.");
        goto End;
      }
    }
    else {
      tmp = import_buf;
      for (i = 0; i < actual_num_imp; i++) {
        tmp_size = *((int *)(tmp + id_size));
        if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
          printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo);
          ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp);
          printf("size = %d bytes\n", tmp_size);
        }

        /* Unpack the object's data */
       
        zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries,
                       (ZOLTAN_ID_PTR) tmp, tmp_size,
                       tmp + tag_size, &ierr);
        if (ierr < 0) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                          "ZOLTAN_UNPACK_OBJ_FN.");
          goto End;
        }
        tmp += (tmp_size + tag_size);
      }
      ZOLTAN_FREE(&import_buf);
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects");

  if (zz->Migrate.Post_Migrate_PP != NULL) {
    zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data,
                             num_gid_entries, num_lid_entries,
                             num_import, import_global_ids,
                             import_local_ids, import_procs, import_to_part,
                             num_export, export_global_ids,
                             export_local_ids, export_procs, export_to_part,
                             &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_POST_MIGRATE_PP_FN function.");
      goto End;
    }
  }

  if (zz->Migrate.Post_Migrate != NULL) {
    zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data,
                             num_gid_entries, num_lid_entries,
                             num_import, import_global_ids,
                             import_local_ids, import_procs,
                             num_export, export_global_ids,
                             export_local_ids, export_procs,
                             &ierr);
    if (ierr < 0) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from "
                      "ZOLTAN_POST_MIGRATE_FN function.");
      goto End;
    }
  }

End:

  if (actual_exp_allocated) {
    Zoltan_Multifree(__FILE__, __LINE__, 4, 
                     &actual_exp_gids, &actual_exp_lids, 
                     &actual_exp_procs, &actual_exp_to_part);
  }
  if (actual_imp_allocated) {
    Zoltan_Multifree(__FILE__, __LINE__, 4, 
                     &actual_imp_gids, &actual_imp_lids, 
                     &actual_imp_procs, &actual_imp_to_part);
  }

  if (ierr < 0) {
    if (exp_plan) Zoltan_Comm_Destroy(&exp_plan);
    Zoltan_Multifree(__FILE__, __LINE__, 5,
                     &import_buf, &tmp_id, &sizes, &idx, &export_buf);
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
  return (ierr);
}
Ejemplo n.º 17
0
int Zoltan_LB_Balance(
  ZZ *zz, 
  int *changes,
  int *num_gid_entries,
  int *num_lid_entries,
  int *num_import_objs,
  ZOLTAN_ID_PTR *import_global_ids,
  ZOLTAN_ID_PTR *import_local_ids,
  int **import_procs,
  int *num_export_objs,
  ZOLTAN_ID_PTR *export_global_ids,
  ZOLTAN_ID_PTR *export_local_ids,
  int **export_procs
)
{
/*
 * Wrapper around Zoltan_LB for backward compatibility with
 * previous Zoltan versions.  
 * Appropriate only when (# requested partitions == # processors), uniformly
 * distributed.
 * Arguments correspond directly with arguments of Zoltan_LB.
 */

char *yo = "Zoltan_LB_Balance";
int ierr = ZOLTAN_OK;    /* Error code */
int *import_to_part = NULL;    /* Array used as dummy arg in partitioning. */
int *export_to_part = NULL;    /* Array used as dummy arg in partitioning. */

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Determine whether partition parameters were set.  Report error if
   * values are unreasonable. */
  if ((zz->LB.Num_Global_Parts_Param != -1 && 
       zz->LB.Num_Global_Parts_Param != zz->Num_Proc) ||
      (zz->LB.Num_Local_Parts_Param != -1 &&
       zz->LB.Num_Local_Parts_Param != 1)) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
      "Non-uniform distribution of partitions over processors is specified; "
      "use Zoltan_LB_Partition.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }
  
  ierr = Zoltan_LB(zz, 0, changes, num_gid_entries, num_lid_entries,
           num_import_objs, import_global_ids, import_local_ids,
           import_procs, &import_to_part, 
           num_export_objs, export_global_ids, 
           export_local_ids, export_procs, &export_to_part);


End:
  /* Not returning import/export partition information; free it if allocated. */
  if (import_to_part != NULL) 
    Zoltan_Special_Free(zz, (void **) &import_to_part, 
                        ZOLTAN_SPECIAL_MALLOC_INT);
  if (export_to_part != NULL) 
    Zoltan_Special_Free(zz, (void **) &export_to_part, 
                        ZOLTAN_SPECIAL_MALLOC_INT);
  ZOLTAN_TRACE_EXIT(zz, yo);
  return(ierr);
}
Ejemplo n.º 18
0
int Zoltan_Random(
  ZZ *zz,                       /* The Zoltan structure.                     */
  float *part_sizes,            /* Input:  Array of size zz->LB.Num_Global_Parts
                                   * zz->Obj_Weight_Dim
                                   containing the percentage of work to be
                                   assigned to each partition.               */
  int *num_import,              /* Return -1. Random uses only export lists. */
  ZOLTAN_ID_PTR *import_global_ids, /* Not used. */
  ZOLTAN_ID_PTR *import_local_ids,  /* Not used. */
  int **import_procs,           /* Not used. */
  int **import_to_part,         /* Not used. */
  int *num_export,              /* Output: Number of objects to export. */
  ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */
  ZOLTAN_ID_PTR *export_local_ids,  /* Output: LIDs to export. */
  int **export_procs,           /* Output: Processsors to export to. */
  int **export_to_part          /* Output: Partitions to export to. */
)
{
  int ierr = ZOLTAN_OK;
  int i, count, num_obj;
  int max_export;
  double rand_frac = 1.0;       /* Default is to move all objects. */
  ZOLTAN_ID_PTR global_ids = NULL;
  ZOLTAN_ID_PTR local_ids = NULL; 
  int *parts = NULL;
  float *dummy = NULL;
  static char *yo = "Zoltan_Random";
  static int first_time = 1;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Synchronize the random number generator. 
   * This synchronization is needed only for sanity in our nightly testing. 
   * If some other operation (eg., Zoltan_LB_Eval) changes the status of 
   * the random number generator, the answers here will change.  They won't
   * be wrong, but they will be different from our accepted answers.
   */
  if (first_time) {
    Zoltan_Srand(zz->Seed, NULL);
    Zoltan_Rand(NULL);
    first_time=0;
  }

  /* No import lists computed. */
  *num_import = -1;

  /* Get parameter values. */
  Zoltan_Bind_Param(Random_params, "RANDOM_MOVE_FRACTION", (void *) &rand_frac);
  Zoltan_Assign_Param_Vals(zz->Params, Random_params, zz->Debug_Level, 
                           zz->Proc, zz->Debug_Proc);

  /* Get list of local objects. */
  ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, 0,
                             &dummy, &parts);

  /* Bound number of objects to export. */
  max_export = 1.5*rand_frac*num_obj;

  /* Allocate export lists. */
  *export_global_ids = *export_local_ids = NULL;
  *export_procs = *export_to_part = NULL;
  if (max_export > 0) {
    if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, max_export,
                               ZOLTAN_SPECIAL_MALLOC_GID)
     || !Zoltan_Special_Malloc(zz, (void **)export_local_ids, max_export,
                               ZOLTAN_SPECIAL_MALLOC_LID)
     || !Zoltan_Special_Malloc(zz, (void **)export_procs, max_export,
                               ZOLTAN_SPECIAL_MALLOC_INT)
     || !Zoltan_Special_Malloc(zz, (void **)export_to_part, max_export,
                               ZOLTAN_SPECIAL_MALLOC_INT)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }
  }

  /* Randomly assign ids to procs. */
  count=0;
  for (i=0; i<num_obj; i++){
    /* Randomly select some objects to move (export) */
    if ((count<max_export) && (Zoltan_Rand(NULL)<rand_frac*ZOLTAN_RAND_MAX)){
      /* export_global_ids[count] = global_ids[i]; */
      ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]),
                     &global_ids[i*zz->Num_GID]);
      if (local_ids)
        /* export_local_ids[count] = local_ids[i]; */
        ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]),
                       &local_ids[i*zz->Num_LID]);
      /* Randomly pick new partition number. */
      (*export_to_part)[count] = Zoltan_Rand_InRange(NULL, zz->LB.Num_Global_Parts);
      /* Processor number is derived from partition number. */
      (*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz, 
                     (*export_to_part)[count], &global_ids[i*zz->Num_GID]);

      /* printf("Debug: Export gid %u to part %d and proc %d.\n", (*export_global_ids)[count], (*export_to_part)[count], (*export_procs)[count]); */

      ++count;
    }
  }
  (*num_export) = count;

End:
  /* Free local memory, but not export lists. */
  ZOLTAN_FREE(&global_ids);
  ZOLTAN_FREE(&local_ids);
  ZOLTAN_FREE(&parts);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ierr;
}
Ejemplo n.º 19
0
/*  Main partitioning function for hypergraph partitioning. */
int Zoltan_PHG_Partition (
  ZZ *zz,               /* Zoltan data structure */
  HGraph *hg,           /* Input hypergraph to be partitioned */
  int p,                /* Input:  number partitions to be generated */
  float *part_sizes,    /* Input:  array of length p containing percentages
                           of work to be assigned to each partition */
  Partition parts,      /* Input:  initial partition #s; aligned with vtx 
                           arrays. 
                           Output:  computed partition #s */
  PHGPartParams *hgp)   /* Input:  parameters for hgraph partitioning. */
{

  PHGComm *hgc = hg->comm;
  VCycle  *vcycle=NULL, *del=NULL;
  int  i, err = ZOLTAN_OK, middle;
  ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */
  ZOLTAN_GNO_TYPE prevVcnt     = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */
  ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered
				 		               before any coarsening */
  ZOLTAN_GNO_TYPE tot_nPins, local_nPins;
  MPI_Datatype zoltan_gno_mpi_type;
  char *yo = "Zoltan_PHG_Partition";
  int do_timing = (hgp->use_timers > 1);
  int fine_timing = (hgp->use_timers > 2);
  int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0);
  short refine = 0;
  struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz);
  int reset_geometric_matching = 0;
  char reset_geometric_string[4];

  ZOLTAN_TRACE_ENTER(zz, yo);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
    
  if (do_timing) {
    if (timer->vcycle < 0) 
      timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle");
    if (timer->procred < 0) 
      timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction");
    if (timer->match < 0) 
      timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching");
    if (timer->coarse < 0) 
      timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening");
    if (timer->coarsepart < 0)
      timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1,
                                           "Coarse_Partition");
    if (timer->refine < 0) 
      timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement");
    if (timer->project < 0) 
      timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up");

    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
  }

  local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;

  MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator);

  origVpincnt = tot_nPins;

  if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) {
    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_MEMERR;
  }

  /* For geometric coarsening, hgp->matching pointer and string are reset
   * after geometric_levels of coarsening.  Will need to reset them after
   * this vcycle is completed.  Capture that fact now!  */
  if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) {
    reset_geometric_matching = 1;
    strcpy(reset_geometric_string, hgp->redm_str);
  }

  /****** Coarsening ******/    
#define COARSEN_FRACTION_LIMIT 0.9  /* Stop if we don't make much progress */
  while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl)
	 && ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */
	     || (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */
    && hg->dist_y[hgc->nProc_y] && hgp->matching) {
      ZOLTAN_GNO_TYPE *match = NULL;
      VCycle *coarser=NULL, *redistributed=NULL;
        
      prevVcnt     = hg->dist_x[hgc->nProc_x];
      prevVedgecnt = hg->dist_y[hgc->nProc_y];

#ifdef _DEBUG      
      /* UVC: load balance stats */
      Zoltan_PHG_LoadBalStat(zz, hg);
#endif
      
      if (hgp->output_level >= PHG_DEBUG_LIST) {
          uprintf(hgc,
                  "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
                  hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
                  hgp->redm_str,
                  hgp->coarsepartition_str, hgp->refinement_str, p);
          if (hgp->output_level > PHG_DEBUG_LIST) {
              err = Zoltan_HG_Info(zz, hg);
              if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
                  goto End;
          }
      }
      if (hgp->output_level >= PHG_DEBUG_PLOT)
        Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
         "coarsening plot");

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator);
      }
      if (vcycle_timing) {
        if (vcycle->timer_match < 0) {
          char str[80];
          sprintf(str, "VC Matching %d", hg->info);
          vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match,
                           hgc->Communicator);
      }

      /* Allocate and initialize Matching Array */
      if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array");
        ZOLTAN_TRACE_EXIT(zz, yo);
        return ZOLTAN_MEMERR;
      }
      for (i = 0; i < hg->nVtx; i++)
        match[i] = i;
        
      /* Calculate matching (packing or grouping) */

      err = Zoltan_PHG_Matching (zz, hg, match, hgp);

      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
        ZOLTAN_FREE (&match);
        goto End;
      }
      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match,
                          hgc->Communicator);

      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator);
      }

      if (vcycle_timing) {
        if (vcycle->timer_coarse < 0) {
          char str[80];
          sprintf(str, "VC Coarsening %d", hg->info);
          vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str);
        }
        ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse,
                           hgc->Communicator);
      }
            
      if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) {
        ZOLTAN_FREE (&match);
        ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL.");
        goto End;
      }

      /* Construct coarse hypergraph and LevelMap */
      err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap,
       &vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData, 
       &vcycle->comm_plan, hgp);

      if (err != ZOLTAN_OK && err != ZOLTAN_WARN) 
        goto End;

      if (vcycle_timing)
        ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse,
                          hgc->Communicator);
        
      if (do_timing) {
        ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator);
        ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
      }

      ZOLTAN_FREE (&match);

      if ((err=allocVCycle(coarser))!= ZOLTAN_OK)
        goto End;
      vcycle = coarser;
      hg = vcycle->hg;

      if (hgc->nProc > 1 && hgp->ProRedL > 0) {
        local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins;
	MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM,
		      hgc->Communicator);

	if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) {
	  if (do_timing) {
	    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	    ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator);
	  }
	  /* redistribute to half the processors */
	  origVpincnt = tot_nPins; /* update for processor reduction test */

	  if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  for (i = 0; i < hg->nVtx; i++)
	    hg->vmap[i] = i;

	  middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL);

	  if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */
	    if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1))
	      --middle; /* if it was prime just use one less #procs (since
			   it should be bigger than SMALL_PRIME it is safe to
			   decrement) */
	  }

	  if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) {
	    ZOLTAN_FREE (&hgc);
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL.");
	    goto End;
	  }

	  Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg,
				  &vcycle->vlno,&vcycle->vdest);
	  if (hgp->UseFixedVtx || hgp->UsePrefPart)
            redistributed->hg->bisec_split = hg->bisec_split;

	  if ((err=allocVCycle(redistributed))!= ZOLTAN_OK)
	    goto End;
	  vcycle = redistributed;

	  if (hgc->myProc < 0)
	    /* I'm not in the redistributed part so I should go to uncoarsening
	       refinement and wait */ {
	    if (fine_timing) {
	      if (timer->cpgather < 0)
		timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather");
	      if (timer->cprefine < 0)
		timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine");
	      if (timer->cpart < 0)
		timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part");
	    }
	    if (do_timing) {
	      ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
	      ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	    }
	    goto Refine;
	  }

	  hg = vcycle->hg;
	  hg->redl = hgp->redl; /* not set with hg creation */
	  if (do_timing) {
	    ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator);
	    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	  }
	}
      }
  }

  if (hgp->output_level >= PHG_DEBUG_LIST) {
    uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n",
     hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
     hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p);
    if (hgp->output_level > PHG_DEBUG_LIST) {
      err = Zoltan_HG_Info(zz, hg);
      if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
        goto End;
    }
  }
  if (hgp->output_level >= PHG_DEBUG_PLOT)
    Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL,
     "coarsening plot");

  /* free array that may have been allocated in matching */
  if (hgp->vtx_scal) {
    hgp->vtx_scal_size = 0;
    ZOLTAN_FREE(&(hgp->vtx_scal));
  }

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator);
  }

  /****** Coarse Partitioning ******/

  err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp);

  if (err != ZOLTAN_OK && err != ZOLTAN_WARN)
    goto End;

  if (do_timing) {
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator);
    ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
  }

Refine:
  del = vcycle;
  refine = 1;

  /****** Uncoarsening/Refinement ******/
  while (vcycle) {
    VCycle *finer = vcycle->finer;
    hg = vcycle->hg;

    if (refine && hgc->myProc >= 0) {
      if (do_timing) {
	ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator);
      }
      if (vcycle_timing) {
	if (vcycle->timer_refine < 0) {
	  char str[80];
	  sprintf(str, "VC Refinement %d", hg->info);
	  vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str);
	}
	ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine,
			   hgc->Communicator);
      }

      err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp);
        
      if (do_timing) {
	ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator);
	ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
      }
      if (vcycle_timing)
	ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine,
			  hgc->Communicator);

                          
      if (hgp->output_level >= PHG_DEBUG_LIST)     
	uprintf(hgc, 
		"FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n",
		hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, 
		hgp->redm_str,
		hgp->coarsepartition_str, hgp->refinement_str, p,
		Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p, 
                                           vcycle->Part),
		Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err));

      if (hgp->output_level >= PHG_DEBUG_PLOT)
	Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part,
			"partitioned plot");
    }

    if (finer)  {
      int *rbuffer;
            
      /* Project coarse partition to fine partition */
      if (finer->comm_plan) {
	refine = 1;
	if (do_timing) {
	  ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
	  ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator);
	}
	if (vcycle_timing) {
	  if (vcycle->timer_project < 0) {
	    char str[80];
	    sprintf(str, "VC Project Up %d", hg->info);
	    vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str);
	  }
	  ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project,
			     hgc->Communicator);
	}
        
	/* easy to assign partitions to internal matches */
	for (i = 0; i < finer->hg->nVtx; i++)
	  if (finer->LevelMap[i] >= 0)   /* if considers only the local vertices */
	    finer->Part[i] = vcycle->Part[finer->LevelMap[i]];
          
	/* now that the course partition assignments have been propagated */
	/* upward to the finer level for the local vertices, we need to  */    
	/* fill the LevelData (matched pairs of a local vertex with a    */
	/* off processor vertex) with the partition assignment of the    */
	/* local vertex - can be done totally in the finer level!        */    
	for (i = 0; i < finer->LevelCnt; i++)  {
	  ++i;          /* skip over off processor lno */
	  finer->LevelData[i] = finer->Part[finer->LevelData[i]]; 
	}
            
	/* allocate rec buffer to exchange LevelData information */
	rbuffer = NULL;
	if (finer->LevelSndCnt > 0)  {
	  rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int));
	  if (!rbuffer)    {
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }
	}       
      
	/* get partition assignments from owners of externally matched vtxs */  
	Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i);
	Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1, 
         (char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer);

	/* process data to assign partitions to expernal matches */
	for (i = 0; i < 2 * finer->LevelSndCnt;)  {
	  int lno, partition;
	  lno       = rbuffer[i++];
	  partition = rbuffer[i++];      
	  finer->Part[lno] = partition;         
	}

	ZOLTAN_FREE (&rbuffer);                  
	Zoltan_Comm_Destroy (&finer->comm_plan);                   

	if (do_timing) {
	  ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator);
	  ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator);
	}
	if (vcycle_timing)
	  ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project,
			    hgc->Communicator);
      } else {
	int *sendbuf = NULL, size;
	refine = 0;
	/* ints local and partition numbers */
	if (finer->vlno) {
	  sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int));
	  if (!sendbuf) {
	    ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }

	  for (i = 0; i < hg->nVtx; ++i) {
	    sendbuf[2 * i] = finer->vlno[i];     /* assign local numbers */
	    sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */
	  }
	}

	ZOLTAN_FREE (&hgc);
	hgc = finer->hg->comm; /* updating hgc is required when the processors
				   change */
	/* Create comm plan to unredistributed processors */
	err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0,
				 finer->vdest, hgc->Communicator, COMM_TAG+2,
				 &size);

	if (err != ZOLTAN_OK && err != ZOLTAN_WARN) {
	  ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed.");
	  goto End;
	}

	/* allocate rec buffer to exchange sendbuf information */
	rbuffer = NULL;
	if (finer->hg->nVtx) {
	  rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int));

	  if (!rbuffer) {
	    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
            ZOLTAN_TRACE_EXIT(zz, yo);
	    return ZOLTAN_MEMERR;
	  }
	}

	/* Use plan to send partitions to the unredistributed processors */

	Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf,
		       2*sizeof(int), (char *) rbuffer);

	MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm);
	
	/* process data to assign partitions to unredistributed processors */
	for (i = 0; i < 2 * finer->hg->nVtx;) {
	  int lno, partition;
	  lno       = rbuffer[i++];
	  partition = rbuffer[i++];
	  finer->Part[lno] = partition;
	}

	if (finer->vlno)
	  ZOLTAN_FREE (&sendbuf);

	ZOLTAN_FREE (&rbuffer);
	Zoltan_Comm_Destroy (&finer->comm_plan);
      }
    }

    vcycle = finer;
  }       /* while (vcycle) */
    
End:
  vcycle = del;
  while (vcycle) {
    if (vcycle_timing) {
      Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout);
      Zoltan_Timer_Destroy(&vcycle->timer);
    }
    if (vcycle->finer) {   /* cleanup by level */
      Zoltan_HG_HGraph_Free (vcycle->hg);

      if (vcycle->LevelData)
	Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part,
			  &vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg);
      else if (vcycle->vlno)
	Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest,
			  &vcycle->vlno, &vcycle->LevelMap, &vcycle->hg);
      else
	Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part,
			  &vcycle->LevelMap, &vcycle->hg);
    }
    else                   /* cleanup top level */
      Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap,
                        &vcycle->LevelData);
    del = vcycle;
    vcycle = vcycle->finer;
    ZOLTAN_FREE(&del);
  }

  if (reset_geometric_matching) {
    strcpy(hgp->redm_str, reset_geometric_string);
    Zoltan_PHG_Set_Matching_Fn(hgp);
  }

  if (do_timing)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator);
  ZOLTAN_TRACE_EXIT(zz, yo) ;
  return err;
}
Ejemplo n.º 20
0
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions,
                               int level, Map *array, int *c1, int *c2) {
  int     i, j, k;                    /* index counters */
  pRegion ptr;                        /* region in the mesh */
  COORD   origin;                     /* centroid coordinate information */
  pRegion *regions = NULL;            /* an array of regions */
  int     *npids = NULL;
  Region  *regions2 = NULL;           /* an array of regions */
  int     *npids2 = NULL;
  int     nreg;                       /* number of regions */
  COORD   min,                        /* minimum bounds of an octant */
          max;                        /* maximum bounds of an octant */
  COORD   cmin,                       /* minimum bounds of a child octant */
          cmax;                       /* maximum bounds of a child octant */
  COORD   rmin,                       /* minimum bounds of a remote octant */
          rmax;                       /* maximum bounds of a remote octant */
  int     new_num;
  int     n;
  int     dir = 0;
  pRList  RootList;              
  pOctant RootOct;
  OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure);
  char *yo = "Zoltan_Oct_migreg_migrate_orphans_static";
  int ierr = ZOLTAN_OK;
  ZOLTAN_ID_PTR gids2, lids2;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;

  if(nregions > 0) {
    /* create the array of messages to be sent to other processors */
    /* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */
    
    if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
    if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&regions);
      return ZOLTAN_MEMERR;
    }
  }
  ptr = RegionList;
  n = nreg = 0;
  while((ptr != NULL) && (nregions > 0)) {
    if(ptr->attached == 1) {
      /* if region already attached to an octant, then skip to next region */
      ptr = ptr->next;
      continue;
    }

    /* region not attached, have to find which processor to send to */
    j=0;
    dir = 0;
    vector_set(min, OCT_info->OCT_gmin);
    vector_set(max, OCT_info->OCT_gmax);
    /* 
     * for each level of refinement, find which child region belongs to.
     * translate which child to which entry in map array.
     */
    for(i=0; i<level; i++) {
      Zoltan_Oct_bounds_to_origin(min, max, origin);
      if(OCT_info->OCT_dimension == 2)
	j = j * 4;
      else
	j = j * 8;
      k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord);
      new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k);
      dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num);
      j += new_num;
      Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax);
      vector_set(min, cmin);
      vector_set(max, cmax);
    }
    /* inform message which processor to send to */
    npids[n] = array[j].npid;
    RootList = array[j].list;
    while((RootOct = RL_nextRootOctant(&RootList))) {
      Zoltan_Oct_bounds(RootOct,rmin,rmax);
      if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) {
	npids[n] = RootOct->npid;
	break;
      }
    }
    if((npids[n] != -1) && (npids[n] != zz->Proc)) {
      Zoltan_Oct_copy_info(zz, ptr, &(regions[n++]));
    }
    else {
      Zoltan_Oct_insert_orphan(zz, *ptr);
    }
    nreg++;                                      /* increment region counter */
    ptr = ptr->next;                                  /* look at next region */
  }

  /*
   * if regions looked at != number of regions in region list, 
   * then there is an error
   */
  if (nreg!=nregions) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions");
    return ZOLTAN_FATAL;
  }

  regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region));
  gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n);
  lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n);
  npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int));
  
  for(i=0; i<n; i++) {
    npids2[i] = npids[i];
    vector_set(regions2[i].Coord, regions[i]->Coord);
    regions2[i].Weight = regions[i]->Weight;
    regions2[i].Global_ID = &(gids2[i*num_gid_entries]);
    regions2[i].Local_ID = (num_lid_entries 
                              ? &(lids2[i*num_lid_entries]) 
                              : NULL);
    ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID);
    ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID);
    regions2[i].Proc = regions[i]->Proc;
    regions2[i].attached = 0;
  }

  *c1 = n;
  /* migrate the orphan regions according to the message array */
  Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2);
  
  for (i=0; i < n; i++) {
    ZOLTAN_FREE(&(regions[i]->Global_ID));
    ZOLTAN_FREE(&(regions[i]->Local_ID));
    ZOLTAN_FREE(&(regions[i]));
  }
  ZOLTAN_FREE(&regions);
  ZOLTAN_FREE(&npids);
  ZOLTAN_FREE(&regions2);
  ZOLTAN_FREE(&gids2);
  ZOLTAN_FREE(&lids2);
  ZOLTAN_FREE(&npids2);

  return ierr;
}
Ejemplo n.º 21
0
int
Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix,
  int request_GNOs,                /* Input:  Flag indicating calling code 
                                              needs translation of extra GIDs
                                              to GNOs; partial 2D coloring
                                              needs this feature. */
  int num_requested,               /* Input:  Local # of GIDs needing 
                                              translation to GNOs. */
  ZOLTAN_ID_PTR requested_GIDs,    /* Input:  Calling code requests the 
                                              GNOs for these GIDs */
  ZOLTAN_GNO_TYPE *requested_GNOs  /* Output: Return GNOs of 
                                              the requested GIDs.  */
)  
{
  static char *yo = "Zoltan_Matrix_Build";
  int ierr = ZOLTAN_OK;
  int nX;
  ZOLTAN_GNO_TYPE tmp;
  ZOLTAN_GNO_TYPE *xGNO = NULL;
  ZOLTAN_ID_PTR xLID=NULL;
  ZOLTAN_ID_PTR xGID=NULL;
  ZOLTAN_ID_PTR yGID=NULL;
  ZOLTAN_ID_PTR pinID=NULL;
  float *xwgt = NULL;
  int * Input_Parts=NULL;
  struct Zoltan_DD_Struct *dd = NULL;
  int *proclist = NULL;
  int *xpid = NULL;
  int i;
  int gno_size_for_dd;
  MPI_Datatype zoltan_gno_mpi_type;
  int use_full_dd = (opt->speed == MATRIX_FULL_DD);
  int fast_build_base = opt->fast_build_base;
  matrix->opts.speed = opt->speed;  
  matrix->opts.fast_build_base = opt->fast_build_base;

  ZOLTAN_TRACE_ENTER(zz, yo);

  if (num_requested && (!requested_GIDs || !requested_GNOs)) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                       "Error in requested input; needed arrays are NULL.\n");
  }

  /* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */
  gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();

  memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */
  memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options));

  /**************************************************/
  /* Obtain vertex information from the application */
  /**************************************************/

  ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID,
			     zz->Obj_Weight_Dim, &xwgt,
			     &Input_Parts);

  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data");
    goto End;
  }
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&xwgt);

  /*******************************************************************/
  /* Assign vertex consecutive numbers (gnos)                        */
  /*******************************************************************/

  if (use_full_dd) {
    /* Zoltan computes a translation */
    /* Have to use Data Directory if request_GNOs is true. */
    if (nX) {
      xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
      if (xGNO == NULL)
	MEMORY_ERROR;
    }
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX);

    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices");
      goto End;
    }

    ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0);
    CHECK_IERR;

    /* Make our new numbering public */
    Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL,  NULL, nX);
    if (request_GNOs) {
      Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs,
                     NULL, NULL, num_requested, NULL);
    }
  }
  else { /* We don't want to use the DD */
     /*
     * KDDKDD 2/10/11  This code cannot work when NUM_GID_ENTRIES>1.
     * KDDKDD 2/10/11  The assumption is that, if a user sets the
     * KDDKDD 2/10/11  appropriate parameter to enable this code, the user
     * KDDKDD 2/10/11  knows that his GIDs are compatible with integers.
     */
    if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
      xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE));
      if (nX && xGNO == NULL)
        MEMORY_ERROR;
      for (i=0; i < nX; i++)
        xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base;
    }
    else {
      xGNO = (ZOLTAN_GNO_TYPE *)xGID;
      if (fast_build_base)
        for (i = 0; i < nX; i++)
          xGNO[i] -= fast_build_base;
    }

    for (i = 0; i < num_requested; i++)
      requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i]
                        - fast_build_base;
     
    tmp = (ZOLTAN_GNO_TYPE)nX; 
    MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator);
  }

  /* I store : xGNO, xGID, xpid,  */

  ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID,
			   sizeof(int), matrix->globalX/zz->Num_Proc, 0);
  CHECK_IERR;

  /* Hope a linear assignment will help a little */
  if (matrix->globalX/zz->Num_Proc)
    Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc);
  /* Associate all the data with our xGNO */
  xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int));
  if (nX >0 && xpid == NULL) MEMORY_ERROR;
  for (i = 0 ; i < nX ; ++i)
    xpid[i] = zz->Proc;

  Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX);
  ZOLTAN_FREE(&xpid);

  if (matrix->opts.pinwgt)
    matrix->pinwgtdim = zz->Edge_Weight_Dim;
  else
    matrix->pinwgtdim = 0;

  ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd);
  CHECK_IERR;
  matrix->nY_ori = matrix->nY;

  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){
    goto End;
  }

  if (matrix->opts.enforceSquare && matrix->redist) {
    /* Convert yGID to yGNO using the same translation as x */
    /* Needed for graph : rowID = colID */
    /* y and x may have different distributions */
    matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE));
    if (matrix->nY && matrix->yGNO == NULL) {
      ZOLTAN_FREE(&pinID);
      MEMORY_ERROR;
    }
    ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL,
		    matrix->nY, NULL);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n");
      ierr = ZOLTAN_FATAL;
      ZOLTAN_FREE(&pinID);
      goto End;
    }
  }

  if (matrix->opts.local) { /* keep only local edges */
    proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int));
    if (matrix->nPins && proclist == NULL) { 
      ZOLTAN_FREE(&pinID);
      MEMORY_ERROR;
    }
  }
  else
    proclist = NULL;

  /* Convert pinID to pinGNO using the same translation as x */
  if (use_full_dd) {
    matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE));
    if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) {
        ZOLTAN_FREE(&pinID);
        MEMORY_ERROR;
    }

    ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL,
			   matrix->nPins, proclist);
    if (ierr != ZOLTAN_OK) {
      ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n");
      ierr = ZOLTAN_FATAL;
      goto End;
    }
    ZOLTAN_FREE(&pinID);
    Zoltan_DD_Destroy(&dd);
    dd = NULL;
  }
  else {
    if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){
      matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE));
      if (matrix->nPins && !matrix->pinGNO){
        ZOLTAN_FREE(&pinID);
        MEMORY_ERROR;
      }
      for (i=0; i < matrix->nPins; i++)
        matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base;
      
      ZOLTAN_FREE(&pinID);
    }
    else{
      matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID;
      if (fast_build_base)
        for (i=0; i < matrix->nPins; i++)
          matrix->pinGNO[i] -= fast_build_base;
      pinID = NULL;
    }
  }

/*   if (matrix->opts.local) {  /\* keep only local edges *\/ */
/*     int *nnz_list; /\* nnz offset to delete *\/ */
/*     int nnz;       /\* number of nnz to delete *\/ */
/*     int i; */

/*     nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */
/*     if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */
/*     for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */
/*       if (proclist[i] == zz->Proc) continue; */
/*       nnz_list[nnz++] = i; */
/*     } */
/*     ZOLTAN_FREE(&proclist); */
/*     Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */
/*   } */

  if (!matrix->opts.enforceSquare) {
    /* Hyperedges name translation is different from the one of vertices */
    matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE));
    if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR;

    /*     int nGlobalEdges = 0; */
    ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY,
					     matrix->opts.randomize, &matrix->globalY);
    CHECK_IERR;

/*     /\**************************************************************************************** */
/*      * If it is desired to remove dense edges, divide the list of edges into */
/*      * two lists.  The ZHG structure will contain the removed edges (if final_output is true), */
/*      * and the kept edges will be returned. */
/*      ****************************************************************************************\/ */
/*     totalNumEdges = zhg->globalHedges; */

/*     ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */
/*				     &nLocalEdges, &nGlobalEdges, &nPins, */
/*				     &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */

/*     if (nGlobalEdges < totalNumEdges){ */
/*       /\* re-assign edge global numbers if any edges were removed *\/ */
/*       ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */
/*					       randomizeInitDist, &totalNumEdges); */
/*       if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */
/*	ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */
/*	goto End; */
/*       } */
/*     } */

      /* We have to define ddY : yGNO, yGID, ywgt */
      ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID,
			       0, matrix->globalY/zz->Num_Proc, 0);
      /* Hope a linear assignment will help a little */
      if (matrix->globalY/zz->Num_Proc)
        Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc);
      /* Associate all the data with our yGNO */
      Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY);
  }

 End:
  ZOLTAN_FREE(&xpid);
  ZOLTAN_FREE(&xLID);
  ZOLTAN_FREE(&xGNO);
  ZOLTAN_FREE(&xGID);
  ZOLTAN_FREE(&xwgt);
  ZOLTAN_FREE(&Input_Parts);
  ZOLTAN_FREE(&proclist);
  if (dd != NULL)
    Zoltan_DD_Destroy(&dd);
  /* Already stored in the DD */
  ZOLTAN_FREE(&yGID);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 22
0
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, 
                              ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
                              int *npids, int nregions, int *c2) 
{
  char *yo = "Zoltan_Oct_migreg_migrate_regions";
  int i;                         /* index counter */
  int ierr = ZOLTAN_OK;
  int n_import;
  ZOLTAN_COMM_OBJ *comm_plan;           /* Object returned by communication routines */
  Region *import_objs = NULL;    /* Array of import objects used to request 
				    the objs from other processors. */
  ZOLTAN_ID_PTR import_gids = NULL;  /* Array of global IDs of import_objs. */
  ZOLTAN_ID_PTR import_lids = NULL;  /* Array of local IDs of import_objs. */
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;

  ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, 
                        MIGMIGREGCommCreate, &n_import);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create");
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }
  *c2 = n_import;
  if (n_import > 0) {
    import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region));
    import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import);
    import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import);

    if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_MEMERR;
    }
  }
  ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), 
                   (char *) import_objs);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&import_objs);
    ZOLTAN_FREE(&import_gids);
    ZOLTAN_FREE(&import_lids);
    return (ierr);
  }

  ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, 
                    sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
    ZOLTAN_TRACE_EXIT(zz, yo);
    ZOLTAN_FREE(&import_objs);
    ZOLTAN_FREE(&import_gids);
    ZOLTAN_FREE(&import_lids);
    return (ierr);
  }

  if (num_lid_entries > 0) {
    ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, 
                      sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do.");
      ZOLTAN_TRACE_EXIT(zz, yo);
      ZOLTAN_FREE(&import_objs);
      ZOLTAN_FREE(&import_gids);
      ZOLTAN_FREE(&import_lids);
      return (ierr);
    }
  }
  for (i=0; i<n_import; i++) {
    import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]);
    import_objs[i].Local_ID = (num_lid_entries 
                                 ? &(import_lids[i*num_lid_entries]) 
                                 : NULL);
    Zoltan_Oct_insert_orphan(zz, import_objs[i]);
  }
  
  ZOLTAN_FREE(&import_objs);
  ZOLTAN_FREE(&import_gids);
  ZOLTAN_FREE(&import_lids);

  ierr = Zoltan_Comm_Destroy(&comm_plan);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }
  return ierr;
}
Ejemplo n.º 23
0
static int _Zoltan_Get_Obj_List(
  ZZ *zz, 
  int *num_obj,
  ZOLTAN_ID_PTR *global_ids, 
  ZOLTAN_ID_PTR *local_ids, 
  int wdim, 
  float **objwgts,
  int **parts,
  int special_malloc
)
{
  char *yo = "_Zoltan_Get_Obj_List";
  int i, n;
  int num_gid_entries = zz->Num_GID;
  int num_lid_entries = zz->Num_LID;
  int alloced_gids = 0, alloced_lids = 0;
  int gid_off, lid_off;
  ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass 
                                  NULL to query functions when 
                                  NUM_LID_ENTRIES == 0. */
  float *next_objwgt;          /* Temporarry pointer to an object weight; used
                                  to pass NULL to query functions when wdim=0 */
  int ierr = ZOLTAN_OK;

  ZOLTAN_TRACE_ENTER(zz, yo);

  *num_obj = 0;
  *objwgts = NULL;

  if (zz->Get_Num_Obj != NULL) {
    *num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
      goto End;
    }
  }
  else {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
    ierr = ZOLTAN_FATAL;
    goto End;
  }

  if (*num_obj > 0) {

    /* 
     * Test global_ids and local_ids for NULL.  
     * Should be NULL for doing partitioning.
     * Should not be NULL if doing ordering.
     */
    if (special_malloc){
      if (*global_ids == NULL) {
        Zoltan_Special_Malloc(zz, (void **)global_ids,
                          *num_obj, ZOLTAN_SPECIAL_MALLOC_GID);
        alloced_gids = 1;
      }
      if (*local_ids == NULL) {
        Zoltan_Special_Malloc(zz, (void **)local_ids,
                          *num_obj, ZOLTAN_SPECIAL_MALLOC_LID);
        alloced_lids = 1;
      }

      Zoltan_Special_Malloc(zz, (void **)parts,
                          *num_obj, ZOLTAN_SPECIAL_MALLOC_INT);
    }
    else{
      if (*global_ids == NULL) {
        *global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj);
        alloced_gids = 1;
      }
      if (*local_ids == NULL) {
        *local_ids  = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj);
        alloced_lids = 1;
      }
      *parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int));
    }

    if (wdim > 0)
      *objwgts  = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim);

    if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) ||
        (*parts == NULL) ||
        (wdim > 0 && *objwgts == NULL)) {
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error.");
      ierr = ZOLTAN_MEMERR;
      goto End;
    }

    if (zz->Get_Obj_List != NULL){
      /* Get object list directly */
      zz->Get_Obj_List(zz->Get_Obj_List_Data, 
                       num_gid_entries, num_lid_entries,
                       *global_ids, *local_ids, 
                       wdim, *objwgts, &ierr);
    }
    else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){
      /* Use iterator functions to loop through object list */
      if (zz->Get_First_Obj(zz->Get_First_Obj_Data, 
                            num_gid_entries, num_lid_entries, 
                            *global_ids, *local_ids, 
                            wdim, *objwgts, &ierr)){
        n = *num_obj;
        i = 0;
        while (!ierr && (i<n-1)){ 
          gid_off = i * num_gid_entries;
          lid_off = i * num_lid_entries;
          lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL);
          next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries]) 
                                      : NULL);
          next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL);
          zz->Get_Next_Obj(zz->Get_Next_Obj_Data, 
                           num_gid_entries, num_lid_entries, 
                           &((*global_ids)[gid_off]), lid, 
                           &((*global_ids)[gid_off+num_gid_entries]),
                           next_lid,
                           wdim, next_objwgt, &ierr);
          i++;
        }
      }
    }
    else { /* No way to get objects */
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or "
                         "ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN.");
      ierr = ZOLTAN_FATAL;
      goto End;
    }

    /* Get partition information for objects. */
    /* Call user-callback if provided; otherwise, all parts == zz->Proc */
    
    if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) {
      for (i = 0; i < *num_obj; i++) 
        (*parts)[i] = zz->Proc;
    }
    else if (zz->Get_Part_Multi) {
      zz->Get_Part_Multi(zz->Get_Part_Multi_Data, 
                              num_gid_entries, num_lid_entries, *num_obj,
                              *global_ids, *local_ids, *parts, &ierr);
      if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                           "Error returned from ZOLTAN_PART_MULTI_FN");
        goto End;
      }
    }
    else {
      for (i = 0; i < *num_obj; i++) {
        lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL);
        (*parts)[i] = zz->Get_Part(zz->Get_Part_Data,
                              num_gid_entries, num_lid_entries,
                              &((*global_ids)[i*num_gid_entries]), lid, &ierr);
        if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
          ZOLTAN_PRINT_ERROR(zz->Proc, yo, 
                             "Error returned from ZOLTAN_PART_FN");
          goto End;
        }
      }
    }
  }

End:
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned.");
    if (special_malloc){
      if (alloced_gids) 
        Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID);
      if (alloced_lids) 
        Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID);
      Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT);
    }
    else{
      if (alloced_gids) ZOLTAN_FREE(global_ids);
      if (alloced_lids) ZOLTAN_FREE(local_ids);
      ZOLTAN_FREE(parts);
    }
    ZOLTAN_FREE(objwgts);
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
  return ierr;
}
Ejemplo n.º 24
0
int Zoltan_Invert_Lists(
  ZZ *zz,                       /* Zoltan structure.                  */
  int num_in,                   /* Number of objects in the input lists. */ 
  ZOLTAN_ID_PTR in_global_ids,  /* Array of input global IDs. */
  ZOLTAN_ID_PTR in_local_ids,   /* Array of input local IDs. */
  int *in_procs,                /* Array of processor IDs of processors owning
                                   the input objects. */
  int *in_to_part,              /* Optional:  Array of partition numbers to 
                                   which input objects should be assigned. */
  int *num_out,                 /* Returned value:  Number of output objs. */
  ZOLTAN_ID_PTR *out_global_ids,/* Returned value:  Array of global IDs of
                                   output objects. */
  ZOLTAN_ID_PTR *out_local_ids, /* Returned value:  Array of local IDs of
                                   output objects. */
  int **out_procs,              /* Returned value:  Array of processor IDs
                                   to which output objects are assigned. */
  int **out_to_part             /* Optional:  Returned value:  Array of 
                                   partition numbers to which output
                                   objects should be assigned. */
)
{
/*
 *  Routine to compute the inverse map.  Can be used in two ways:
 *  1.  Given, for each processor, a list of objects to be received by the 
 *  processor, compute the list of objects that the processor needs to send 
 *  to other processors to satisfy their needs.
 *  2.  Given, for each processor, a list of objects to be sent to other 
 *  processors, compute the list of objects that the processor needs to receive
 *  to satisfy its needs.
 */

char *yo = "Zoltan_Invert_Lists";
char msg[256];
ZOLTAN_COMM_OBJ *comm_plan;        /* Object returned communication routines  */
int msgtag, msgtag2;               /* Message tags for communication routines */
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int include_parts;                 /* Flag indicating whether to compute
                                      inverse list for partitions. */
int ierr, ret_ierr = ZOLTAN_OK;

  ZOLTAN_TRACE_ENTER(zz, yo);
  /*
   *  Return if this processor is not in the Zoltan structure's
   *  communicator.
   */

  if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ZOLTAN_OK);
  }

  /*
   *  Check that all procs use the same id types.
   */

  ierr = check_invert_input(zz, num_in, in_procs, in_to_part, 
                            &num_gid_entries, &num_lid_entries, &include_parts);
  if (ierr != ZOLTAN_OK) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ierr;
  }

  /* Initialize returned arrays. */
  *out_global_ids = NULL;
  *out_local_ids = NULL;
  *out_procs = NULL;
  if (include_parts) *out_to_part = NULL;


  /*
   *  Compute communication map and num_out, the number of objs this
   *  processor has to out to establish the new decomposition.
   */

  msgtag = 32767;
  ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, 
                        msgtag, num_out);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
            (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    ret_ierr = ierr;
    goto End;
  }
  

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create");

  /*
   *  Allocate space for the object tags that need to be outed.  Communicate
   *  to get the list of objects to be outed.
   */

  if (*num_out > 0) {
    if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_GID)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_LID)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out,
                           ZOLTAN_SPECIAL_MALLOC_INT)) {
      ret_ierr = ZOLTAN_MEMERR;
      goto End;
    }
    if (include_parts) {
      if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out,
                             ZOLTAN_SPECIAL_MALLOC_INT)) {
        ret_ierr = ZOLTAN_MEMERR;
        goto End;
      }
    }
  }

  /*
   *  Use the communication plan to send global IDs, local IDs, and processor
   *  numbers.  Do in separate communications to avoid a memory copy and to
   *  simplify implementation when a data type is added to the comm. package
   *  (to support heterogeneous computing).
   */

  msgtag2 = 32766;
  ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, 
                    (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), 
                    (char *) *out_global_ids);
  if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
            (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    ret_ierr = ierr;
  }

  if (num_lid_entries) {
    msgtag2--;
    ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, 
                      (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), 
                      (char *) *out_local_ids);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
              (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      ret_ierr = ierr;
    }
  }

  Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL,
                   NULL, NULL, NULL, NULL, NULL, *out_procs, NULL);
  
  if (include_parts) {
    msgtag2--;
    ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, 
                      (int) sizeof(int), (char *) *out_to_part);
    if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
      sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", 
              (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      ret_ierr = ierr;
    }
  }

  ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do");

End:

  Zoltan_Comm_Destroy(&comm_plan);

  if (ret_ierr == ZOLTAN_MEMERR) {
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
    Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
    Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
    Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT);
    if (include_parts)
      Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT);
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
  return (ret_ierr);
}
Ejemplo n.º 25
0
/*
 * void Zoltan_Oct_gen_tree_from_input_data()
 *
 * This function will create a root node of on each processor which will
 * then be used to create an octree with regions associated with it. The
 * tree will then be balanced and the output used to balance "mesh regions"
 * on several processors.
 */
static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag,
						int *c1, int *c2, int *c3, 
						float *c0, int createpartree) 
{
  char *yo = "Zoltan_Oct_gen_tree_from_input_data";
  pRList  RootList;       /* list of all local roots */
  pOctant RootOct;        /* root octree octant */
  COORD min,              /* min coord bounds of objects */
        max;              /* max coord bounds of objects */
  int num_extra;          /* number of orphaned objects */
  int num_objs;           /* total number of local objects */
  pRegion ptr,            /* pointer to iterate trough region list */
          ptr1;           /* pointer to iterate trough region list */
  pOctant root;           /* root of the partition tree */
  int     i;              /* index counter */
  int     count,          /* count for leaf nodes in partition tree */
          proc,           /* proc leaf node of parition tree belongs to */
          extra,          /* extra leaf node flag, if not evenly divisible */
          remainder;      /* remainder of node, or processors to fill */
  pOctant cursor,         /* cursor to iterate through octant list */
          cursor2,        /* another cursor to iterate through octant list */
          parent;         /* parent of an octant */
  int level,              /* number of levels of refinement */
      n,                  /* index counter */
      part;               /* partition counter */
  Map *array;             /* map of which processors own which octants */
  int hold;               /* used for calculating partition divisions */
  int ierr = 0;

#ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
  double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX};
  COORD global_min, global_max;
#endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */
  int nroots = 0;
  /*test*/
  /* COORD gmin,gmax; */

  OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure);

  ZOLTAN_TRACE_ENTER(zz, yo);
  /*
   * If there are no objects on this processor, do not create a root octant.
   * The partitioner will probably assign objects to this processor
   */
  if(zz->Get_Num_Obj == NULL) {
    fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
	    "Must register ZOLTAN_NUM_OBJ_FN function");
    abort();
  }
  *c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
  if (ierr) {
    fprintf(stderr, "OCT [%d] %s: Error returned from user defined "
                    "Get_Num_Obj function.\n", zz->Proc, yo);
    exit (-1);
  }
  ptr1 = NULL;

  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds");
  /* Need A Function To Get The Bounds Of The Local Objects */
  Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0);
  
#ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN
  /* For now, don't want to add the new query function to Zoltan. */
  /* Zoltan_Oct_get_bounds appears to compute the global min and max from */
  /* the object input. */
  vector_set(OCT_info->OCT_gmin, min);
  vector_set(OCT_info->OCT_gmax, max);
#else
  /*test*/
  /*getMaxBounds(&gmin, &gmax);*/
  if(zz->Get_Bounds_Geom == NULL) {
    fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:",
	    "Must register Get_Bounds_Geom function");
    abort();
  }
  zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr); 
  
  MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3, 
		MPI_DOUBLE, MPI_MIN, zz->Communicator);
  MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3,
		MPI_DOUBLE, MPI_MAX, zz->Communicator);
  vector_set(OCT_info->OCT_gmin, global_min);
  vector_set(OCT_info->OCT_gmax, global_max);
#endif
  /* 
   * the following code segment was added to create a pseudo global octree
   * needed for the partitioner. The basic idea is to regroup all the
   * regions into something close to an octree partitioning and build the
   * tree from that.
   * NOTE: This way of doing things is very costly, especially when calling
   * this for the first time on a mesh not partitioned in an octree style
   * partitioning.
   */

    level = 0;                                    /* initialize level count */

  /* 
   * if more than 1 processor, need to find what level of refinement needed
   * to initially partition bounding box among the processors 
   */

  
    if(zz->Num_Proc > 1) {
      n = zz->Num_Proc;
      if(OCT_info->OCT_dimension == 2)
	hold = 4;
      else
	hold = 8;
      remainder = hold;
      for(; remainder > 0; level++) {
	int pr = (int)POW(hold, level);
	remainder = n - pr;
      }
      level--;
    }
  ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree");

  if(createpartree) {
    /* create the global root octant */
    root = Zoltan_Oct_POct_new(OCT_info);
    Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax);
    /* Zoltan_Oct_setOrientation(root, 0); */
  
    /* subdivide to as many levels as calculated */
    for(i=0; i<level; i++) {
      cursor = root;
      while(cursor != NULL) {
	if(Zoltan_Oct_isTerminal(cursor)) {
	  cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
	  Zoltan_Oct_terminal_refine(zz, cursor, 0);
	  cursor = cursor2;
	}
	else 
	  cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
      }
    }
    
#if 0
    if(zz->Proc == 0)
      for(i=0; i<8; i++)
	if(Zoltan_Oct_child(root, i) == NULL)
	  fprintf(stderr,"NULL child pointer\n");
	else
	  fprintf(stderr, "child %d exists\n", i);
#endif

  ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array");
    /* this part creates the map array */
    if(OCT_info->OCT_dimension == 2) {
      hold = (int)POW(4, level);                 /* ignoring the z+ octants */
      if(hold == 0)
	hold = 1;
    }
    else
      hold = (int)POW(8, level);

    part = hold / zz->Num_Proc;          /* how many octants per partition */
    remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */
    extra = zz->Num_Proc - remainder;/* where to start adding extra octants */
    array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map));   /* alloc map array */
    if(array == NULL) {
      fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n",
	      zz->Proc);
      abort();
    }
    /* initialize variables */
    proc = 0;
    count = 0;
    i = 0;
    cursor = root; 
    while(cursor != NULL) {
      cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor);
      if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) {
	if(proc == extra) {
	  part++;
	  extra = -1;
	}
	if(count != part) {
	  array[i].npid = proc;
	  array[i].list = RL_initRootList();
	  Zoltan_Oct_bounds(cursor, min, max);
	  vector_set(array[i].min, min);
	  vector_set(array[i].max, max);
	  count++;
	}
	else {
	  count = 1;
	  proc++;
	  array[i].npid = proc;
	  array[i].list = RL_initRootList();
	  Zoltan_Oct_bounds(cursor, min, max);
	  vector_set(array[i].min, min);
	  vector_set(array[i].max, max);
	}
	if(proc == zz->Proc) {
	  array[i].npid = -1;
          /* KDDKDD Added RL_freeList below.  The 
           * KDDKDD implementation from RPI leaked memory because the 
           * KDDKDD test cases for setting array[i].list were not mutually 
           * KDDKDD exclusive.  Freeing the list produces the result we got
           * KDDKDD before, without the memory leak.
           */
	  /* LGG --  it seems to me that this array[i].list assignment is
	   * not really necessary. It looks as though it has already been
	   * assigned with the same information from the prev if-else
	   * commented out RL_freeList(), and RL_initRootList()
	   */
          /*RL_freeList(&(array[i].list));*/
          /* KDDKDD End addition */
	  /*array[i].list = RL_initRootList();*/
	  parent = Zoltan_Oct_parent(cursor);
	  if(parent != NULL)
	    Zoltan_Oct_setchild(parent, cursor->which, NULL);
	  /* octant into local root list */
 	  Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1);
	  Zoltan_Oct_setMapIdx(cursor, i);
	  nroots++;
	  /* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc);     
             octant into local root list */
	}
	i++;
      }
      cursor = cursor2;
    } 
    RootList = Zoltan_Oct_POct_localroots(OCT_info); 
    RootOct = RL_nextRootOctant(&RootList);
    if(RootOct != root) {
      /* KDDKDDFREE changed root to &root to allow root to be reset to NULL */
      Zoltan_Oct_POct_delTree(OCT_info,&root);
    }
    
    OCT_info->map = array;
    OCT_info->mapsize = hold;
  }

  /* 
   * attach the regions to the root... Zoltan_Oct_fix will create the octree
   * starting with the root and subdividing as needed 
   */    
  num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs);
 
  ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans");
  Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map,
				    c1, c2);

  /* ZOLTAN_FREE(&array); */
  while(ptr1 != NULL) {
    ptr = ptr1->next;
    ZOLTAN_FREE(&(ptr1->Global_ID));
    ZOLTAN_FREE(&(ptr1->Local_ID));
    ZOLTAN_FREE(&ptr1);
    ptr1 = ptr;
  }
  ZOLTAN_TRACE_EXIT(zz, yo);
}
Ejemplo n.º 26
0
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
			    Zoltan_matrix_2d *outmat, int copy)
{
  static char *yo = "Zoltan_Matrix2d_Distribute";
  int ierr = ZOLTAN_OK;
  int nProc_x, nProc_y;
  int myProc_x, myProc_y;
  int i, j, cnt;
  int *proclist = NULL;
  Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
  ZOLTAN_GNO_TYPE *perm_y = NULL;
  float *wgtarray = NULL;
  float *tmpwgtarray = NULL;
  int msg_tag = 1021982;
  ZOLTAN_COMM_OBJ *plan;
  MPI_Comm communicator = MPI_COMM_NULL;
  int nProc;
  ZOLTAN_GNO_TYPE *yGNO = NULL;
  ZOLTAN_GNO_TYPE *pinGNO = NULL;
  ZOLTAN_GNO_TYPE tmp_gno;
  void *partdata = NULL;
  MPI_Datatype zoltan_gno_mpi_type;

  ZOLTAN_TRACE_ENTER(zz, yo);

  zoltan_gno_mpi_type = Zoltan_mpi_gno_type();

  memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
  if(copy) {
    /* TODO: We need to copy the arrays also */
    Zoltan_Matrix_Reset (&outmat->mtx);
    /* Copy also directories */
    outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
    if (inmat.ddY == inmat.ddX)
      outmat->mtx.ddY = outmat->mtx.ddX;
    else
      outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
  }

  communicator = outmat->comm->Communicator;
  nProc = outmat->comm->nProc;

  nProc_x = outmat->comm->nProc_x;
  nProc_y = outmat->comm->nProc_y;
  myProc_x = outmat->comm->myProc_x;
  myProc_y = outmat->comm->myProc_y;

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Remove_Duplicates");
  ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);

/* KDDKDDKDD  FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
  if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
    /* I need to know the original distribution */
    if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
      outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
    }
    else {
      int *cmember = NULL;

      cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
      if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
      Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
		      outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, "    Zoltan_Distribute_Partition_Register");
      partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
						      cmember, zz->Num_Proc, zz->Num_Proc);
      ZOLTAN_FREE(&cmember);
      Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
    }
  }

  /*
   * Build comm plan for sending non-zeros to their target processors in
   * 2D data distribution.
   */
  /* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
  proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
  sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));

  if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;

  wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));

  if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;

  yGNO = outmat->mtx.yGNO;
  pinGNO = outmat->mtx.pinGNO;

KDDKDDKDD(zz->Proc, "    CommPlan Hash");
  cnt = 0;
  for (i = 0; i < outmat->mtx.nY; i++) {
    ZOLTAN_GNO_TYPE edge_gno=-1;
    /* processor row for the edge */
    edge_gno = yGNO[i];

    for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
      ZOLTAN_GNO_TYPE vtx_gno=-1;
      /* processor column for the vertex */
      vtx_gno = pinGNO[j];

      proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
					  &sendbuf[cnt].part_y);
      if (proclist[cnt] < 0) /* Discard this nnz */
        continue;
      sendbuf[cnt].GNO[0] = edge_gno;
      sendbuf[cnt].GNO[1] = vtx_gno;
      memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
               outmat->mtx.pinwgtdim*sizeof(float));
      cnt++;
    }
    if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
      proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
					  &sendbuf[cnt].part_y);
      if (proclist[cnt] < 0) /* Discard this nnz */
        continue;
      sendbuf[cnt].GNO[0] = edge_gno;
      sendbuf[cnt].GNO[1] = -1;
      memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
      cnt++;
    }
  }

  if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
    Zoltan_Distribute_Partition_Free(&outmat->hashDistData);

  if (outmat->mtx.yend != outmat->mtx.ystart + 1)
    ZOLTAN_FREE(&outmat->mtx.yend);
  outmat->mtx.yend = NULL;
  ZOLTAN_FREE(&outmat->mtx.ystart);
  ZOLTAN_FREE(&outmat->mtx.yGNO);
  ZOLTAN_FREE(&outmat->mtx.pinGNO);
  ZOLTAN_FREE(&outmat->mtx.pinwgt);
  ZOLTAN_FREE(&outmat->mtx.yGID);


  /*
   * Send pins to their target processors.
   * They become non-zeros in the 2D data distribution.
   */

KDDKDDKDD(zz->Proc, "    CommPlan Create");
  msg_tag--;
  ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
  ZOLTAN_FREE(&proclist);

  nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
  if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;

  msg_tag--;
  Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
  ZOLTAN_FREE(&sendbuf);

  if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
    tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
    if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;

    msg_tag--;
    Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
		   (char *) tmpwgtarray);
    ZOLTAN_FREE(&wgtarray);
  }
  Zoltan_Comm_Destroy(&plan);

  /* Unpack the non-zeros received. */

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Remove_DupArcs");
  /* TODO: do take care about singletons */
  Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
			       &outmat->mtx);
}

  /* Now we just have to change numbering */
  outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
  outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
  if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;

  /* FIXME: Work only in 1D */
  tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
  MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
  for (i = 1 ; i <= nProc_y ; i ++) {
    outmat->dist_y[i] += outmat->dist_y[i-1];
  }
  outmat->dist_x[1] = outmat->mtx.globalX;

  perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
  if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
  for (i = 0 ; i < outmat->mtx.nY ; ++i){
    perm_y[i] = i + outmat->dist_y[myProc_y];
  }

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Permute");
  Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);

KDDKDDKDD(zz->Proc, "    Zoltan_Matrix_Permute done");
 End:
  ZOLTAN_FREE(&perm_y);
  ZOLTAN_FREE(&proclist);
  ZOLTAN_FREE(&sendbuf);
  ZOLTAN_FREE(&nonzeros);
  ZOLTAN_FREE(&tmpwgtarray);


  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 27
0
int Zoltan_ParMetis(
  ZZ *zz,               /* Zoltan structure */
  float *part_sizes,    /* Input:  Array of size zz->Num_Global_Parts
                           containing the percentage of work to be
                           assigned to each partition.               */
  int *num_imp,         /* number of objects to be imported */
  ZOLTAN_ID_PTR *imp_gids,  /* global ids of objects to be imported */
  ZOLTAN_ID_PTR *imp_lids,  /* local  ids of objects to be imported */
  int **imp_procs,      /* list of processors to import from */
  int **imp_to_part,    /* list of partitions to which imported objects are
                           assigned.  */
  int *num_exp,         /* number of objects to be exported */
  ZOLTAN_ID_PTR *exp_gids,  /* global ids of objects to be exported */
  ZOLTAN_ID_PTR *exp_lids,  /* local  ids of objects to be exported */
  int **exp_procs,      /* list of processors to export to */
  int **exp_to_part     /* list of partitions to which exported objects are
                           assigned. */
)
{
  char *yo = "Zoltan_ParMetis";
  int ierr;
  ZOLTAN_Third_Graph gr;
  ZOLTAN_Third_Geom  *geo = NULL;
  ZOLTAN_Third_Vsize vsp;
  ZOLTAN_Third_Part  prt;
  ZOLTAN_Output_Part part;

  ZOLTAN_ID_PTR global_ids = NULL;
  ZOLTAN_ID_PTR local_ids = NULL;

  int use_timers = 0;
  int timer_p = -1;
  int get_times = 0;
  double times[5];

  double pmv3_itr = 0.0;
  realtype itr = 0.0;
  indextype options[MAX_PARMETIS_OPTIONS];
  char alg[MAX_PARAM_STRING_LEN];

#ifdef ZOLTAN_PARMETIS
  MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
                                   /* change our zz struct.                  */
#endif

  indextype i;
  realtype *imb_tols;
  indextype ncon;
  indextype edgecut;
  indextype wgtflag;
  indextype numflag = 0;
  indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */

  ZOLTAN_TRACE_ENTER(zz, yo);

  Zoltan_Third_Init(&gr, &prt, &vsp, &part,
                    imp_gids, imp_lids, imp_procs, imp_to_part,
                    exp_gids, exp_lids, exp_procs, exp_to_part);

  if (sizeof(realtype) != sizeof(float)) {
    int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
    prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));

    for (i = 0; i < tmp; i++) 
      prt.input_part_sizes[i] = (realtype) part_sizes[i];

    /* KDD 2/2014:  removed re-scaling part sizes so they sum to one.  
     *              part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
     *              plus, the code here was wrong for multiple object weights.
     *              similar scaling code did not exist in the Scotch interface.
     */
    prt.part_sizes = prt.input_part_sizes;
  }
  else
    prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;


  ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
    Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
    return (ierr);
  }

  gr.graph_type = 0;

#ifdef ZOLTAN_PARMETIS
  SET_GLOBAL_GRAPH(&gr.graph_type);
  /* Select type of graph, negative because we impose them */
  /* TODO: add a parameter to select the type, shared with Scotch */
/*   if (strcmp (graph_type, "GLOBAL") != 0) { */
/*     gr.graph_type = - LOCAL_GRAPH; */
/*     if (zz->Num_Proc > 1) { */
/*       ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/*       gr.graph_type = - GLOBAL_GRAPH; */
/*       retval = ZOLTAN_WARN; */
/*     } */
/*   } */
#else /* graph is local */
  SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */


  /* Some algorithms use geometry data */
  if (strncmp(alg, "PARTGEOM", 8) == 0){          /* PARTGEOM & PARTGEOMKWAY */
    geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
    memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
    /* ParMETIS will crash if geometric method and some procs have no nodes. */
    /* Avoid fatal crash by setting scatter to level 2 or higher. */
    gr.scatter_min = 2;
    if (geo == NULL) {
      ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
      return (ZOLTAN_MEMERR);
    }
    if (strcmp(alg, "PARTGEOM") == 0) {
      gr.get_data = 0;
    }
  }

  timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);

  /* Start timer */
  get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
  if (get_times){
    MPI_Barrier(zz->Communicator);
    times[0] = Zoltan_Time(zz->Timer);
  }

  vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
  if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
    /* ParMETIS will free this memory; use malloc to allocate so
       ZOLTAN_MALLOC counters don't show an error. */
    vsp.vsize_malloc = 1 ;
  }
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */


  ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids,  &gr, 
                                 geo, &prt, &vsp);
  if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
    Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
    return (ierr);
  }

  /* Get object sizes if requested */
  if (options[PMV3_OPT_USE_OBJ_SIZE] &&
      (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
      (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
    gr.showMoveVol = 1;


  /* Get a time here */
  if (get_times) times[1] = Zoltan_Time(zz->Timer);

  /* Get ready to call ParMETIS */
  edgecut = -1;
  wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
  numflag = 0;
  ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);

  if (!prt.part_sizes){
    ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
  }
  if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
    for (i=0; i<num_part; i++){
      indextype j;

      printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
      for (j=0; j<ncon; j++)
        printf("%f ", prt.part_sizes[i*ncon+j]);
      printf("\n");
    }
  }

  /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
  for (i = 0; i < num_part*ncon; i++)
    if (prt.part_sizes[i] == 0) 
      ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
                            "ParMETIS 3.x will likely fail. Please use a "
                            "different method, or remove the zero-sized "
                            "parts from the problem.");


  /* Set Imbalance Tolerance for each weight component. */
  imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
  if (!imb_tols){
    /* Not enough memory */
    ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
  }
  for (i=0; i<ncon; i++)
    imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);

  /* Now we can call ParMetis */

  /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */


#ifdef ZOLTAN_PARMETIS
  if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */

    /* First check for ParMetis 3 routines */
    if (strcmp(alg, "PARTKWAY") == 0){
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
                                  "ParMETIS_V3_PartKway");
      ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
                           &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
                           imb_tols, options, &edgecut, prt.part, &comm);
      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
    }
    else if (strcmp(alg, "PARTGEOMKWAY") == 0){
      indextype ndims = geo->ndims;
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
                                  "ParMETIS_V3_PartGeomKway");
      ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
                               &wgtflag, &numflag, &ndims, geo->xyz, &ncon,
                               &num_part, prt.part_sizes,
                               imb_tols, options, &edgecut, prt.part, &comm);
      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
    }
    else if (strcmp(alg, "PARTGEOM") == 0){
      indextype ndims = geo->ndims;
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
                                  "ParMETIS_V3_PartGeom");
      ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
    }
    else if (strcmp(alg, "ADAPTIVEREPART") == 0){
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
                                  "ParMETIS_V3_AdaptiveRepart");
      ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
                                 vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
                                 &num_part, prt.part_sizes, imb_tols,
                                 &itr, options, &edgecut, prt.part, &comm);
      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
    }
    else if (strcmp(alg, "REFINEKWAY") == 0){
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
                                  "ParMETIS_V3_RefineKway");
      ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
                             &wgtflag, &numflag, &ncon, &num_part,
                             prt.part_sizes, imb_tols,
                             options, &edgecut, prt.part, &comm);
      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
    }
    else {
      /* Sanity check: This should never happen! */
      char msg[256];
      sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
      ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
    }
  }
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
  /* TODO: I don't know how to set balance ! */
  if (IS_LOCAL_GRAPH(gr.graph_type)) {
    /* Check for Metis routines */
    if (strcmp(alg, "PARTKWAY") == 0){
      ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
      /* Use default options for METIS */
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
      options[0] = 0;
      METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, 
                            gr.vwgt, gr.ewgts, &wgtflag,
                            &numflag, &num_part, prt.part_sizes, 
                            options, &edgecut, prt.part);
#else
      METIS_SetDefaultOptions(options);
      METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
                           gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
                           prt.part_sizes, imb_tols, options,
                           &edgecut, prt.part);
#endif

      ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
    }
    else {
      /* Sanity check: This should never happen! */
      char msg[256];
      sprintf(msg, "Unknown Metis algorithm %s.", alg);
      ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
    }
  }
#endif /* ZOLTAN_METIS */


  /* Get a time here */
  if (get_times) times[2] = Zoltan_Time(zz->Timer);


  if (gr.final_output) { 
    /* Do final output now because after the data will not be coherent:
       unscatter only unscatter part data, not graph */
    ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
  }
  /* Ignore the timings of Final Ouput */
  if (get_times) times[3] = Zoltan_Time(zz->Timer);

  ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, 
                                  geo, &prt, &vsp, NULL, &part);

  Zoltan_Third_Export_User(&part, 
                           num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
                           num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);

  /* Get a time here */
  if (get_times) times[4] = Zoltan_Time(zz->Timer);

  if (get_times) Zoltan_Third_DisplayTime(zz, times);

  if (use_timers && timer_p >= 0)
    ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);

  Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
  if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
  if (geo != NULL) ZOLTAN_FREE(&geo);
  ZOLTAN_FREE(&global_ids);
  ZOLTAN_FREE(&local_ids);

  ZOLTAN_TRACE_EXIT(zz, yo);

  return (ierr);
}
Ejemplo n.º 28
0
int Zoltan_Order(
    ZZ *zz,               /* Zoltan structure */
    int num_gid_entries, /* # of entries for a global id */
    int num_obj,		/* Number of objects to order */
    ZOLTAN_ID_PTR gids,   /* List of global ids (local to this proc) */
    /* The application must allocate enough space */
    int *rank,            /* rank[i] is the rank of gids[i] */
    int *iperm            /* iperm[rank[i]]=i, only for sequential ordering */
)
{
    /*
     * Main user-call for ordering.
     * Input:
     *   zz, a Zoltan structure with appropriate function pointers set.
     *   gids, a list of global ids or enough space to store such a list
     *   lids, a list of local ids or enough space to store such a list
     * Output:
     *   num_gid_entries
     *   num_lid_entries
     *   gids, a list of global ids (filled in if empty on entry)
     *   lids, a list of local ids (filled in if empty on entry)
     *   rank, rank[i] is the global rank of gids[i]
     * Return values:
     *   Zoltan error code.
     */

    char *yo = "Zoltan_Order";
    int ierr;
    double start_time, end_time;
    double order_time[2] = {0.0,0.0};
    char msg[256];
    int comm[2],gcomm[2];
    ZOLTAN_ORDER_FN *Order_fn;
    struct Zoltan_Order_Options opt;
    int * vtxdist = NULL;
    ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
    int local_num_obj;
    int *local_rank = NULL, *local_iperm=NULL;
    struct Zoltan_DD_Struct *dd = NULL;


    ZOLTAN_TRACE_ENTER(zz, yo);

    if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
        Zoltan_Print_Key_Params(zz);

    start_time = Zoltan_Time(zz->Timer);

    /*
     * Compute Max number of array entries per ID over all processors.
     * This is a sanity-maintaining step; we don't want different
     * processors to have different values for these numbers.
     */
    comm[0] = zz->Num_GID;
    comm[1] = zz->Num_LID;
    MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
    zz->Num_GID = gcomm[0];

    if (num_gid_entries != zz->Num_GID) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
        return (ZOLTAN_FATAL);
    }


    zz->Order.nbr_objects = num_obj;
    zz->Order.rank = rank;
    zz->Order.iperm = iperm;
    zz->Order.gids = gids;
    zz->Order.lids = lids;
    zz->Order.start = NULL;
    zz->Order.ancestor = NULL;
    zz->Order.leaves = NULL;
    zz->Order.nbr_leaves = 0;
    zz->Order.nbr_blocks = 0;

    /*
     *  Return if this processor is not in the Zoltan structure's
     *  communicator.
     */

    if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
        ZOLTAN_TRACE_EXIT(zz, yo);
        return (ZOLTAN_OK);
    }

    /*
     *  Get ordering options from parameter list.
     */

    /* Set default parameter values */
    strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
    strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
    strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */

    opt.use_order_info = 0;
    opt.start_index = 0;
    opt.reorder = 0;

    Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
    Zoltan_Bind_Param(Order_params, "ORDER_TYPE",   (void *) opt.order_type);
    Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
    Zoltan_Bind_Param(Order_params, "REORDER",      (void *) &opt.reorder);
    Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);

    Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
                             zz->Proc, zz->Debug_Proc);

    zz->Order.start_index = opt.start_index;

    /*
     *  Check that the user has allocated space for the return args.
     */
    if (!(gids && rank)) {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
        ZOLTAN_TRACE_EXIT(zz, yo);
        return (ZOLTAN_FATAL);
    }

    /*
     *  Find the selected method.
     */

    if (!strcmp(opt.method, "NONE")) {
        if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
            ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");

        ZOLTAN_TRACE_EXIT(zz, yo);
        return (ZOLTAN_WARN);
    }
#ifdef ZOLTAN_PARMETIS
    else if (!strcmp(opt.method, "NODEND")) {
        Order_fn = Zoltan_ParMetis_Order;
    }
    else if (!strcmp(opt.method, "METIS")) {
        Order_fn = Zoltan_ParMetis_Order;
        /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
        strcpy(opt.method, "NODEND");
        strcpy(opt.order_type, "LOCAL");
    }
    else if (!strcmp(opt.method, "PARMETIS")) {
        Order_fn = Zoltan_ParMetis_Order;
        /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
        strcpy(opt.method, "NODEND");
        strcpy(opt.order_type, "GLOBAL");
    }
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
    else if (!strcmp(opt.method, "SCOTCH")) {
        Order_fn = Zoltan_Scotch_Order;
        /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
        strcpy(opt.method, "NODEND");
        /*     strcpy(opt.order_type, "GLOBAL"); */
    }
#endif /* ZOLTAN_SCOTCH */
    else {
        fprintf(stderr, "%s\n", opt.method);
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
        ZOLTAN_TRACE_EXIT(zz, yo);
        return (ZOLTAN_FATAL);
    }

    if (!strcmp(opt.order_type, "GLOBAL"))
        strcpy (opt.order_type, "DIST");
    if (!strcmp(opt.order_type, "LOCAL"))
        strcpy (opt.order_type, "SERIAL");

    strcpy(zz->Order.order_type, opt.order_type);


    /*
     *  Construct the heterogenous machine description.
     */

    ierr = Zoltan_Build_Machine_Desc(zz);

    if (ierr == ZOLTAN_FATAL) {
        ZOLTAN_TRACE_EXIT(zz, yo);
        return (ierr);
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");

    /*
     * Call the actual ordering function.
     * Compute gid according to the local graph.
     */


    if (zz->Get_Num_Obj != NULL) {
        local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
        if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
            return (ierr);
        }
    }
    else {
        ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
        return (ZOLTAN_FATAL);
    }

    local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
    local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
    local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));

    lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);

    ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
    ZOLTAN_FREE(&lids);

    if (ierr) {
        sprintf(msg, "Ordering routine returned error code %d.", ierr);
        if (ierr == ZOLTAN_WARN) {
            ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
        } else {
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
            Zoltan_Multifree(__FILE__, __LINE__, 3,
                             &local_gids, &local_rank, &local_iperm);
            ZOLTAN_TRACE_EXIT(zz, yo);
            return (ierr);
        }
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");

    /*   Compute inverse permutation if necessary */
    if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
            || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
        ierr = Zoltan_Get_Distribution(zz, &vtxdist);
        if (ierr) {
            /* Error */
            ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
            return (ierr);
        }

        if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
            /* Compute rank from iperm */
            ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
            Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
        }
        else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
            /* Compute iperm from rank */
            ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
            Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
        }
        ZOLTAN_FREE(&vtxdist);
    }

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");


    /* TODO: Use directly the "graph" structure to avoid to duplicate things. */
    /* I store : GNO, rank, iperm */
    ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
    /* Hope a linear assignment will help a little */
    Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
    /* Associate all the data with our xGNO */
    Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);

    ZOLTAN_FREE(&local_gids);
    ZOLTAN_FREE(&local_rank);
    ZOLTAN_FREE(&local_iperm);

    Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
                    num_obj, NULL);
    Zoltan_DD_Destroy(&dd);

    ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");


    end_time = Zoltan_Time(zz->Timer);
    order_time[0] = end_time - start_time;

    if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
        int i;
        Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
        printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
        for (i = 0; i < num_obj; i++) {
            printf("GID = ");
            ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
            printf(", rank = %3d\n", rank[i]);
        }
        printf("\n");
        Zoltan_Print_Sync_End(zz->Communicator, TRUE);
    }


    /* Print timing info */
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
        if (zz->Proc == zz->Debug_Proc) {
            printf("ZOLTAN Times:  \n");
        }
        Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
                            "ZOLTAN     Balance:     ");
    }

    ZOLTAN_TRACE_EXIT(zz, yo);
    if (ierr)
        return (ierr);
    else
        return (ZOLTAN_OK);
}
Ejemplo n.º 29
0
int Zoltan_Scatter_Graph(
  indextype **vtxdist,
  indextype **xadj,
  indextype **adjncy,
  weighttype **vwgt,
  indextype **vsize,
  weighttype **adjwgt,
  realtype   **xyz,
  int     ndims,		/* # dimensions of xyz geometry data */
  int     vwgt_dim,
  ZZ      *zz,
  ZOLTAN_COMM_OBJ **plan
)
{
  static char *yo = "Zoltan_Scatter_Graph";
  char     msg[256];
  indextype *old_vtxdist, *old_adjncy;
  indextype *old_xadj;
  indextype *old_vsize;
  weighttype *old_vwgt, *old_adjwgt;
  realtype   *old_xyz;
  int *ptr, *proclist = NULL, *proclist2 = NULL;
  int i, j, num_obj, old_num_obj, num_edges, nrecv;
  int use_graph;	/* do we use graph data, or only the geometry? */
  int use_vsize;	/* do we use the vsize array? */
  int ewgt_dim= zz->Edge_Weight_Dim;
  ZOLTAN_COMM_OBJ *plan2;

  ZOLTAN_TRACE_ENTER(zz, yo);

  /* Save pointers to "old" data distribution */
  old_adjncy = NULL;
  old_xadj = NULL;
  old_vwgt = old_adjwgt = NULL;
  old_vsize = NULL;
  old_xyz = NULL;

  old_vtxdist = *vtxdist;

  if (xadj)
    old_xadj = *xadj;
  if (adjncy)
    old_adjncy = *adjncy;
  if (vwgt)
    old_vwgt = *vwgt;
  if (vsize)
    old_vsize = *vsize;
  if (adjwgt)
    old_adjwgt = *adjwgt;
  if (xyz)
    old_xyz = *xyz;

  old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]); 
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj);

  /* Compute new distribution, *vtxdist */
  (*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype));
  for (i=0; i<=zz->Num_Proc; i++){
    (*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc;
  }

  /* Check if any proc has graph data */
  i = (old_xadj != NULL);
  MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator);
  j = (old_vsize != NULL);
  MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator);
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc, 
          use_graph, use_vsize);

  /* Reset all data pointers to NULL for now */
  *xadj = NULL;
  *adjncy = NULL;
  *vwgt = *adjwgt = NULL;
  *xyz = NULL;
  if (use_vsize) *vsize = NULL;

  /* Convert the xdj array so that it contains the degree of each vertex */
  if (use_graph){
    for (i=0; i<old_num_obj; i++){
      old_xadj[i] = old_xadj[i+1] - old_xadj[i];
    }
  }

  /* Allocate new space for vertex data */
  num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]);
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj);
  if (use_graph)
    *xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype));
  if (vwgt_dim)
    *vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype));
  if (use_vsize)
    *vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype));
  if (ndims)
    *xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype));

  if (old_num_obj > 0) {
    /* Set up the communication plan for the vertex data */
    proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int));
    /* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */
    j = 0;
    while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++;
    for (i=0; i<old_num_obj; i++){
      if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++;
      proclist[i] = j;
    }
  }

  Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv);

  if (nrecv != num_obj){
    sprintf(msg,"Proc %d received %d object but expected %d.",
      zz->Proc, nrecv, num_obj);
    ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
    /* Free data */
    ZOLTAN_FREE(&proclist);
    ZOLTAN_TRACE_EXIT(zz, yo);
    return ZOLTAN_FATAL;
  }

  /* Do the communication. To save memory, we do not pack all the data into
   * a buffer, but send directly from the old arrays to the new arrays. 
   * We use the vertex communication plan for all the vertex-based arrays
   * and the edge communication plan for all the edge-based arrays.
   */

  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc);

  if (use_graph){
    Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj);
  }
  if (vwgt_dim){
    Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt);
  }
  if (use_vsize){
    Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize);
  }
  if (ndims){
    Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz);
  }
  if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
    printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc);

  if (use_graph){

    /* Rebuild xadj from degrees */
    for (i=1; i<num_obj; i++)
      (*xadj)[i] += (*xadj)[i-1];
    for (i=num_obj; i>0; i--)
      (*xadj)[i] = (*xadj)[i-1];
    (*xadj)[0] = 0;
  
    /* Allocate space for new edge data structures */
    num_edges = (*xadj)[num_obj];
    *adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype));
  
    if (ewgt_dim)
      *adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype));
  
    /* Set up the communication plan for the edge data. */
    ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int));
    for (i=0; i<old_num_obj; i++)
      for (j=0; j<old_xadj[i]; j++)
        *ptr++ = proclist[i];
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
      printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n", 
             zz->Proc, old_xadj[old_num_obj]);
    }

    Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator, 
                   TAG1, &nrecv);
  
    if (nrecv != num_edges){
      sprintf(msg,"Proc %d received %d edges but expected %d.",
        zz->Proc, nrecv, num_edges);
      ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
      /* Free data */
      ZOLTAN_FREE(&proclist);
      ZOLTAN_FREE(&proclist2);
      ZOLTAN_FREE(&old_vtxdist);
      ZOLTAN_FREE(&old_xadj);
      ZOLTAN_FREE(&old_adjncy);
      ZOLTAN_FREE(&old_vwgt);
      ZOLTAN_FREE(&old_vsize);
      ZOLTAN_FREE(&old_adjwgt);
      ZOLTAN_FREE(&old_xyz);
      ZOLTAN_TRACE_EXIT(zz, yo);
      return ZOLTAN_FATAL;
    }
  
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
      printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc);
  
    /* Do the communication. */
    Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy);
    if (ewgt_dim){
      Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt);
    }
  
    if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) 
      printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc);
  
    /* Free the comm. plan for edge data */
    Zoltan_Comm_Destroy(&plan2);

  } /* end of use_graph */

  /* Free data structures */
  ZOLTAN_FREE(&proclist);
  ZOLTAN_FREE(&proclist2);
  ZOLTAN_FREE(&old_vtxdist);
  ZOLTAN_FREE(&old_xadj);
  ZOLTAN_FREE(&old_adjncy);
  ZOLTAN_FREE(&old_vwgt);
  ZOLTAN_FREE(&old_vsize);
  ZOLTAN_FREE(&old_adjwgt);
  ZOLTAN_FREE(&old_xyz);

  ZOLTAN_TRACE_EXIT(zz, yo);
  return ZOLTAN_OK;
}
Ejemplo n.º 30
0
/* 
 * void Zoltan_Oct_migrate_objects()
 *
 * sets up the export_tags, and import_tags for the application that called
 * this load balancing routine 
 */
int Zoltan_Oct_migrate_objects(ZZ *zz, pOctant *octs, int *newpids, int nocts,
			       int *nsenregs, pRegion *import_regions,
			       int *nrecregs, float *c2, float *c3,
			       int *counter3, int *counter4)
{
  int i;                   /* index counter */
  int *tag_pids;           /* array of which processors to send information */
  int np_regs;            /* number of regions previously imported */
  Region *p_reg;          /* previously imported regions */
  ZOLTAN_ID_PTR p_gids;      /* global IDs of previously imported regions */
  ZOLTAN_ID_PTR p_lids;      /* local IDs of previously imported regions */
  ZOLTAN_ID_PTR exported_gids, exported_lids;
  Region *exported_regions;
  int max_objs;
  int ierr = ZOLTAN_OK;
  char *yo = "Zoltan_Oct_migrate_objects";
  p_reg = *import_regions = exported_regions = NULL;
  exported_gids = p_gids = NULL;
  exported_lids = p_lids = NULL;

  /* tag all the regions to be exported */
  ierr = tag_regions(zz, octs, newpids, nocts, &exported_regions, 
                     &exported_gids, &exported_lids, nsenregs, &tag_pids, 
		     &p_reg, &p_gids, &p_lids, &np_regs, c2, &max_objs);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }

  /* get all the region tags that are being imported */
  ierr = malloc_new_objects(zz, *nsenregs, exported_regions, 
			    exported_gids, exported_lids, tag_pids, nrecregs, 
			    import_regions, p_reg, p_gids, p_lids,
			    np_regs, c3);
  if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
    ZOLTAN_TRACE_EXIT(zz, yo);
    return (ierr);
  }

  if(np_regs > 0){
    ZOLTAN_FREE(&p_reg);
    ZOLTAN_FREE(&p_gids);
    ZOLTAN_FREE(&p_lids);
  }
  ZOLTAN_FREE(&exported_regions);
  ZOLTAN_FREE(&exported_gids);
  ZOLTAN_FREE(&exported_lids);

  if(max_objs > (*counter3))
    (*counter3) = max_objs;
  i = (max_objs - (*nsenregs) + (*nrecregs) - np_regs);
  if(i > (*counter3))
    (*counter3) = i;
  (*counter4) = (*nrecregs) - np_regs;

  /*  fprintf(stderr,"(%d) nrectags = %d\n", zz->Proc, (*nrecregs));
   *  for(i=0; i<(*nrecregs); i++)
   *    fprintf(stderr,"%d\n", (*import_regions)[i].Proc);
   */
  ZOLTAN_FREE(&tag_pids);
  return ierr;
}