/* * void Zoltan_Oct_fix_tags() * * fixes the import tags so that region tags that were previously * exported aren't counted when imported back. */ int Zoltan_Oct_fix_tags(ZZ *zz, ZOLTAN_ID_PTR *import_global_ids, ZOLTAN_ID_PTR *import_local_ids, int **import_procs, int **import_to_part, int nrectags, pRegion import_regs) { char *yo = "Zoltan_Oct_fix_tags"; int i; /* index counter */ int ierr = ZOLTAN_OK; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; /* allocate memory */ if (!Zoltan_Special_Malloc(zz,(void **)import_global_ids,nrectags, ZOLTAN_SPECIAL_MALLOC_GID)) { ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if (!Zoltan_Special_Malloc(zz,(void **)import_local_ids,nrectags, ZOLTAN_SPECIAL_MALLOC_LID)) { Zoltan_Special_Free(zz,(void **)import_global_ids, ZOLTAN_SPECIAL_MALLOC_GID); ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if (!Zoltan_Special_Malloc(zz,(void **)import_procs,nrectags, ZOLTAN_SPECIAL_MALLOC_INT)) { Zoltan_Special_Free(zz,(void **)import_global_ids, ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void **)import_local_ids, ZOLTAN_SPECIAL_MALLOC_LID); ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if (!Zoltan_Special_Malloc(zz,(void **)import_to_part,nrectags, ZOLTAN_SPECIAL_MALLOC_INT)) { Zoltan_Special_Free(zz,(void **)import_global_ids, ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void **)import_local_ids, ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz,(void **)import_procs,ZOLTAN_SPECIAL_MALLOC_INT); ZOLTAN_PRINT_ERROR(zz->Proc,yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } /* for each region imported, look at its originating processor */ for(i=0; i<nrectags; i++) { ZOLTAN_SET_GID(zz, &((*import_global_ids)[i*num_gid_entries]), import_regs[i].Global_ID); ZOLTAN_SET_LID(zz, &((*import_local_ids)[i*num_lid_entries]), import_regs[i].Local_ID); (*import_procs)[i] = import_regs[i].newProc; /*(*import_to_part)[i] = zz->Proc;*/ (*import_to_part)[i] = import_regs[i].newProc; } return ierr; }
int Zoltan_PHG( ZZ *zz, /* The Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work assigned to each partition. */ int *num_imp, /* not computed */ ZOLTAN_ID_PTR *imp_gids, /* not computed */ ZOLTAN_ID_PTR *imp_lids, /* not computed */ int **imp_procs, /* not computed */ int **imp_to_part, /* not computed */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part ) /* list of partitions to which exported objs are assigned. */ { char *yo = "Zoltan_PHG"; ZHG *zoltan_hg = NULL; PHGPartParams hgp; /* Hypergraph parameters. */ HGraph *hg = NULL; /* Hypergraph itself */ Partition parts = NULL; /* Partition assignments in 2D distribution. */ int err = ZOLTAN_OK, p=0; struct phg_timer_indices *timer = NULL; int do_timing = 0; ZOLTAN_TRACE_ENTER(zz, yo); /* Initialization of return arguments. */ *num_imp = *num_exp = -1; *imp_gids = *exp_gids = NULL; *imp_lids = *exp_lids = NULL; *imp_procs = *exp_procs = NULL; /* Initialize HG parameters. */ err = Zoltan_PHG_Initialize_Params(zz, part_sizes, &hgp); if (err != ZOLTAN_OK) goto End; if (hgp.use_timers) { if (!zz->LB.Data_Structure) { zz->LB.Data_Structure = (struct phg_timer_indices *) ZOLTAN_MALLOC(sizeof(struct phg_timer_indices)); initialize_timer_indices((struct phg_timer_indices *)zz->LB.Data_Structure); } timer = zz->LB.Data_Structure; if (timer->all < 0) timer->all = Zoltan_Timer_Init(zz->ZTime, 1, "Zoltan_PHG"); } if (hgp.use_timers > 1) { do_timing = 1; if (timer->build < 0) timer->build = Zoltan_Timer_Init(zz->ZTime, 1, "Build"); if (timer->setupvmap < 0) timer->setupvmap = Zoltan_Timer_Init(zz->ZTime, 0, "Vmaps"); } if (hgp.use_timers) ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator); if (do_timing) ZOLTAN_TIMER_START(zz->ZTime, timer->build, zz->Communicator); /* build initial Zoltan hypergraph from callback functions. */ err = Zoltan_PHG_Build_Hypergraph (zz, &zoltan_hg, &parts, &hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph."); goto End; } if (zoltan_hg->GnObj == 0){ /* degenerate case - no objects to partition */ hgp.final_output = 0; goto End; } hg = &zoltan_hg->HG; p = zz->LB.Num_Global_Parts; zoltan_hg->HG.redl = MAX(hgp.redl, p); /* redl needs to be dynamic */ /* RTHRTH -- redl may need to be scaled by number of procs */ /* EBEB -- at least make sure redl > #procs */ if (hgp.UseFixedVtx) hg->bisec_split = 1; /* this will be used only #parts=2 otherwise rdivide will set to appropriate value */ if (hgp.UsePrefPart || hgp.UseFixedVtx) { /* allocate memory for pref_part */ if (hg->nVtx && !(hg->pref_part = (int*) ZOLTAN_MALLOC (sizeof(int) * hg->nVtx))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error building hypergraph."); goto End; } } if (hgp.UsePrefPart) /* copy input parts as pref_part; UVCUVC: TODO: this code (and alloc of pref_part) should go to Build_Hypergraph later */ memcpy(hg->pref_part, parts, sizeof(int) * hg->nVtx); if (hgp.UseFixedVtx) { int i; for (i=0; i<hg->nVtx; ++i) if (hg->fixed_part[i]>=0) hg->pref_part[i] = hg->fixed_part[i]; else if (!hgp.UsePrefPart) hg->pref_part[i] = -1; } hgp.UsePrefPart |= hgp.UseFixedVtx; if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->build, zz->Communicator); /* UVCUVC DEBUG PRINT uprintf(hg->comm, "Zoltan_PHG kway=%d #parts=%d\n", hgp.kway, zz->LB.Num_Global_Parts); */ if (!strcasecmp(hgp.hgraph_pkg, "PARKWAY")){ if (do_timing) { if (timer->parkway < 0) timer->parkway = Zoltan_Timer_Init(zz->ZTime, 0, "PHG_ParKway"); ZOLTAN_TIMER_START(zz->ZTime, timer->parkway, zz->Communicator); } err = Zoltan_PHG_ParKway(zz, hg, p, parts, &hgp); if (err != ZOLTAN_OK) goto End; if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->parkway, zz->Communicator); } else if (!strcasecmp(hgp.hgraph_pkg, "PATOH")){ if (hgp.use_timers > 1) { if (timer->patoh < 0) timer->patoh = Zoltan_Timer_Init(zz->ZTime, 0, "HG_PaToH"); ZOLTAN_TIMER_START(zz->ZTime, timer->patoh, zz->Communicator); } err = Zoltan_PHG_PaToH(zz, hg, p, parts, &hgp); if (err != ZOLTAN_OK) goto End; if (hgp.use_timers > 1) ZOLTAN_TIMER_STOP(zz->ZTime, timer->patoh, zz->Communicator); } else { /* it must be PHG */ /* UVC: if it is bisection anyways; no need to create vmap etc; rdivide is going to call Zoltan_PHG_Partition anyways... */ if (hgp.globalcomm.Communicator != MPI_COMM_NULL) { /* This processor is part of the 2D data distribution; it should participate in partitioning. */ if (hgp.kway || zz->LB.Num_Global_Parts == 2) { /* call main V cycle routine */ err = Zoltan_PHG_Partition(zz, hg, p, hgp.part_sizes, parts, &hgp); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph."); goto End; } } else { int i; if (do_timing) ZOLTAN_TIMER_START(zz->ZTime, timer->setupvmap, zz->Communicator); /* vmap associates original vertices to sub hypergraphs */ if (hg->nVtx && !(hg->vmap = (int*) ZOLTAN_MALLOC(hg->nVtx*sizeof (int)))) { err = ZOLTAN_MEMERR; ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); goto End; } for (i = 0; i < hg->nVtx; ++i) hg->vmap[i] = i; if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->setupvmap, zz->Communicator); /* partition hypergraph */ err = Zoltan_PHG_rdivide (0, p-1, parts, zz, hg, &hgp, 0); if (hgp.output_level >= PHG_DEBUG_LIST) uprintf(hg->comm, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %s/%s/%s/%s p=%d " "bal=%.2f cutl=%.2f\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hgp.convert_str, hgp.redm_str, hgp.coarsepartition_str, hgp.refinement_str, p, Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0, p, parts), Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err)); if (err != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error partitioning hypergraph."); goto End; } ZOLTAN_FREE (&hg->vmap); } #ifdef CHECK_LEFTALONE_VERTICES findAndSaveLeftAloneVertices(zz, hg, p, parts, &hgp); #endif } } if (!strcasecmp(hgp.hgraph_method, "REPARTITION")) { Zoltan_PHG_Remove_Repart_Data(zz, zoltan_hg, hg, &hgp); } /* UVC DEBUG PRINT if (!strcasecmp(hgp->hgraph_method, "REFINE")){ uprintf(hg->comm, "UVC ATTHEEND |V|=%6d |E|=%6d #pins=%6d p=%d bal=%.2f cutl=%.2f\n", hg->nVtx, hg->nEdge, hg->nPins, p, Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, p, parts), Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, p, &err)); detailed_balance_info(zz, hg, part_sizes, p, parts); } */ /* Initialize these timers here so their output is near end of printout */ if (do_timing) if (timer->retlist < 0) timer->retlist = Zoltan_Timer_Init(zz->ZTime, 1, "Return_Lists"); if (hgp.use_timers) if (timer->finaloutput < 0) timer->finaloutput = Zoltan_Timer_Init(zz->ZTime, 1, "Final_Output"); if (do_timing) ZOLTAN_TIMER_START(zz->ZTime, timer->retlist, zz->Communicator); /* Build Zoltan's Output_Parts, mapped from 2D distribution to input distribution. */ Zoltan_PHG_Output_Parts(zz, zoltan_hg, parts); /* Build Zoltan's return arguments. */ Zoltan_PHG_Return_Lists(zz, zoltan_hg, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->retlist, zz->Communicator); End: if (err == ZOLTAN_MEMERR) ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Memory error.") else if (err != ZOLTAN_OK) ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Error partitioning hypergraph.") /* KDDKDD The following code prints a final quality result even when * KDDKDD phg_output_level is zero. It is useful for our tests and * KDDKDD data collection. */ if ((err == ZOLTAN_OK) && hgp.final_output) { static int nRuns=0; static double balsum = 0.0, cutlsum = 0.0, cutnsum = 0.0, movesum = 0.0, repartsum = 0.0; static double balmax = 0.0, cutlmax = 0.0, cutnmax = 0.0, movemax = 0.0, repartmax = 0.0; static double balmin = 1e100, cutlmin = 1e100, cutnmin = 1e100, movemin = 1e100, repartmin = 1e100; double bal = 0.; double cutl = 0.; /* Connnectivity cuts: sum_over_edges((npart-1)*ewgt) */ double cutn = 0.; /* Net cuts: sum_over_edges((nparts>1)*ewgt) */ double rlocal[2]; /* local cut stats for removed edges */ double rglobal[2]; /* global cut stats for removed edges */ int gnremove, i; double move=0.0, gmove; /* local and global migration costs */ double repart=0.0; /* total repartitioning cost: comcost x multiplier + migration_cost */ if (hgp.use_timers) { /* Do not include final output time in partitioning time */ ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->finaloutput, zz->Communicator); } if (hgp.globalcomm.Communicator != MPI_COMM_NULL) { /* Processor participated in partitioning */ bal = Zoltan_PHG_Compute_Balance(zz, hg, hgp.part_sizes, 0, zz->LB.Num_Global_Parts, parts); cutl= Zoltan_PHG_Compute_ConCut(hg->comm, hg, parts, zz->LB.Num_Global_Parts, &err); cutn = Zoltan_PHG_Compute_NetCut(hg->comm, hg, parts, zz->LB.Num_Global_Parts); for (i = 0; i < zoltan_hg->nObj; ++i) { /* uprintf(hg->comm, " obj[%d] = %d in=%d out=%d\n", i, zoltan_hg->AppObjSizes[i], zoltan_hg->Input_Parts[i], zoltan_hg->Output_Parts[i]); */ if (zoltan_hg->Input_Parts[i] != zoltan_hg->Output_Parts[i]) move += (double) ((zoltan_hg->AppObjSizes) ? zoltan_hg->AppObjSizes[i] : 1.0); } } if (!err) { /* Add in cut contributions from removed edges */ MPI_Allreduce(&(zoltan_hg->nRemove), &gnremove, 1, MPI_INT, MPI_SUM, zz->Communicator); if (gnremove) { err = Zoltan_PHG_Removed_Cuts(zz, zoltan_hg, rlocal); MPI_Allreduce(rlocal, rglobal, 2, MPI_DOUBLE,MPI_SUM,zz->Communicator); cutl += rglobal[0]; cutn += rglobal[1]; } MPI_Allreduce(&move, &gmove, 1, MPI_DOUBLE, MPI_SUM, zz->Communicator); repart = cutl*hgp.RepartMultiplier + gmove; repartsum += repart; if (repart > repartmax) repartmax = repart; if (repart < repartmin) repartmin = repart; movesum += gmove; if (gmove > movemax) movemax = gmove; if (gmove < movemin) movemin = gmove; cutlsum += cutl; if (cutl > cutlmax) cutlmax = cutl; if (cutl < cutlmin) cutlmin = cutl; cutnsum += cutn; if (cutn > cutnmax) cutnmax = cutn; if (cutn < cutnmin) cutnmin = cutn; balsum += bal; if (bal > balmax) balmax = bal; if (bal < balmin) balmin = bal; nRuns++; if (zz->Proc == 0) { uprintf(hg->comm, "STATS Runs %d bal CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, bal, balmax, balmin, balsum/nRuns); uprintf(hg->comm, "STATS Runs %d cutl CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, cutl, cutlmax, cutlmin, cutlsum/nRuns); uprintf(hg->comm, "STATS Runs %d cutn CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, cutn, cutnmax, cutnmin, cutnsum/nRuns); uprintf(hg->comm, "STATS Runs %d %s CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, (zoltan_hg->showMoveVol) ? "moveVol" : "moveCnt", gmove, movemax, movemin, movesum/nRuns); if (zoltan_hg->showMoveVol) uprintf(hg->comm, "STATS Runs %d repart CURRENT %f MAX %f MIN %f AVG %f\n", nRuns, repart, repartmax, repartmin, repartsum/nRuns); } } if (hgp.use_timers) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->finaloutput, zz->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->all, zz->Communicator); } } /* KDDKDD End of printing section. */ ZOLTAN_FREE(&parts); if (zoltan_hg != NULL) { Zoltan_PHG_Free_Hypergraph_Data(zoltan_hg); ZOLTAN_FREE (&zoltan_hg); } if (hgp.use_timers) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->all, zz->Communicator); if (hgp.globalcomm.Communicator != MPI_COMM_NULL) Zoltan_Timer_PrintAll(zz->ZTime, 0, hgp.globalcomm.Communicator, stdout); } if (hgp.globalcomm.row_comm != MPI_COMM_NULL) MPI_Comm_free(&(hgp.globalcomm.row_comm)); if (hgp.globalcomm.col_comm != MPI_COMM_NULL) MPI_Comm_free(&(hgp.globalcomm.col_comm)); if (hgp.globalcomm.Communicator != MPI_COMM_NULL) MPI_Comm_free(&(hgp.globalcomm.Communicator)); /* Free part_sizes if created new due to ADD_OBJ_WEIGHT */ if (hgp.part_sizes != part_sizes) ZOLTAN_FREE(&hgp.part_sizes); ZOLTAN_TRACE_EXIT(zz, yo); return err; }
int Zoltan_ParMetis_Order( ZZ *zz, /* Zoltan structure */ int num_obj, /* Number of (local) objects to order. */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */ int *iperm, ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */ ) { static char *yo = "Zoltan_ParMetis_Order"; int i, n, ierr; ZOLTAN_Output_Order ord; ZOLTAN_Third_Graph gr; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't want to risk letting external packages changing our communicator */ #endif indextype numflag = 0; int timer_p = 0; int get_times = 0; int use_timers = 0; double times[5]; ZOLTAN_ID_PTR l_gids = NULL; ZOLTAN_ID_PTR l_lids = NULL; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; ZOLTAN_TRACE_ENTER(zz, yo); #ifdef ZOLTAN_PARMETIS #if TPL_USE_DATATYPE != TPL_METIS_DATATYPES #ifdef TPL_FLOAT_WEIGHT i = 1; #else i = 0; #endif if ((sizeof(indextype) != sizeof(idxtype)) || (sizeof(weighttype) != sizeof(idxtype)) || i){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Not supported: Multiple 3rd party libraries with incompatible " "data types."); return ZOLTAN_FATAL; } #endif #endif memset(&gr, 0, sizeof(ZOLTAN_Third_Graph)); memset(&ord, 0, sizeof(ZOLTAN_Output_Order)); memset(times, 0, sizeof(times)); ord.order_opt = order_opt; if (!order_opt){ /* If for some reason order_opt is NULL, allocate a new ZOOS here. */ /* This should really never happen. */ order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS)); strcpy(order_opt->method,"PARMETIS"); } ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord); /* ParMetis only computes the rank vector */ order_opt->return_args = RETURN_RANK; /* Check that num_obj equals the number of objects on this proc. */ /* This constraint may be removed in the future. */ n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){ /* Return error code */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error."); return(ZOLTAN_FATAL); } if (n != num_obj){ /* Currently this is a fatal error. */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the " "number of objects."); return(ZOLTAN_FATAL); } /* Do not use weights for ordering */ gr.obj_wgt_dim = -1; gr.edge_wgt_dim = -1; gr.num_obj = num_obj; /* Check what ordering type is requested */ if (order_opt){ SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */ #ifdef ZOLTAN_PARMETIS if ((strcmp(order_opt->method, "METIS") == 0)) #endif /* ZOLTAN_PARMETIS */ SET_LOCAL_GRAPH(&gr.graph_type); } gr.get_data = 1; if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: " "set ParMetis instead."); return(ZOLTAN_FATAL); } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); return (ierr); } /* Allocate space for separator sizes */ if (IS_GLOBAL_GRAPH(gr.graph_type)) { if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) { /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype)); if (ord.sep_sizes == NULL) { Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */ } /* Allocate space for direct perm */ ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.rank){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } if (IS_LOCAL_GRAPH(gr.graph_type)){ /* Allocate space for inverse perm */ ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype)); if (!ord.iperm){ /* Not enough memory */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord); ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } } else ord.iperm = NULL; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_PARMETIS if (IS_GLOBAL_GRAPH(gr.graph_type)){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library"); ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy, &numflag, options, ord.rank, ord.sep_sizes, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else #endif /* ZOLTAN_PARMETIS */ #if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS) if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */ indextype numobj = gr.num_obj; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library"); order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; /* Use default options for METIS. */ METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options, ord.iperm, ord.rank); #else METIS_SetDefaultOptions(options); METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options, ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */ #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */ int numbloc; int start; int leaf; int *converttab; int levelmax; levelmax = mylog2(zz->Num_Proc) + 1; converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int)); memset(converttab, 0, zz->Num_Proc*2*sizeof(int)); /* Determine the first node in each separator, store it in zz->TPL_Order.start */ for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) { int father; father = zz->Num_Proc + numbloc; converttab[start] = 2*numbloc; zz->TPL_Order.leaves[leaf++]=start; zz->TPL_Order.ancestor[start] = start + 2; converttab[start+1] = 2*numbloc+1; zz->TPL_Order.leaves[leaf++]=start+1; zz->TPL_Order.ancestor[start+1] = start + 2; start+=2; do { converttab[start] = father; if (father %2 == 0) { int nextoffset; int level; level = mylog2(2*zz->Num_Proc - 1 - father); nextoffset = (1<<(levelmax-level)); zz->TPL_Order.ancestor[start] = start+nextoffset; start++; break; } else { zz->TPL_Order.ancestor[start] = start+1; start++; father = zz->Num_Proc + father/2; } } while (father < 2*zz->Num_Proc - 1); } zz->TPL_Order.start[0] = 0; zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1; for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) { int oldblock=converttab[numbloc-1]; zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock]; } ZOLTAN_FREE(&converttab); ZOLTAN_FREE(&ord.sep_sizes); zz->TPL_Order.leaves[zz->Num_Proc] = -1; zz->TPL_Order.nbr_leaves = zz->Num_Proc; zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1; } else { /* No tree */ zz->TPL_Order.nbr_blocks = 0; zz->TPL_Order.start = NULL; zz->TPL_Order.ancestor = NULL; zz->TPL_Order.leaves = NULL; } /* Correct because no redistribution */ memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE)); memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE)); ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL); ZOLTAN_FREE(&l_gids); ZOLTAN_FREE(&l_lids); /* Get a time here */ if (get_times) times[3] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){ memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i]; } } if ((ord.iperm != NULL) && (iperm != NULL)){ if (sizeof(indextype) == sizeof(int)){ memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype)); } else{ for (i=0; i < gr.num_obj; i++){ iperm[i] = (int)ord.iperm[i]; } } } if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm); ZOLTAN_FREE(&ord.rank); /* Free all other "graph" stuff */ Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); }
int Zoltan_PHG_2ways_hyperedge_partition ( ZZ *zz, /* Input : Zoltan data structure */ HGraph *hg, Partition parts, Zoltan_PHG_Tree *tree, struct Zoltan_DD_Struct * gnoToGID, struct Zoltan_DD_Struct **dd, int *numParts, int **sizeParts ) { int ierr = ZOLTAN_OK; char *yo = "Zoltan_PHG_2ways_hyperedge_partition"; int nEdge, hEdge; int *interval; int *partnumber = NULL; int tree_size; ZOLTAN_ID_TYPE *rowpart =NULL; /* ZOLTAN_ID_TYPE because it's used in Zoltan_DD_* */ ZOLTAN_GNO_TYPE *rowGNO = NULL; ZOLTAN_ID_PTR rowGID=NULL; int index; int offset; ZOLTAN_TRACE_ENTER(zz, yo); nEdge = hg->nEdge; fprintf (stderr, "HG (%d %d x %d) : %d %d\n", hg->comm->myProc, hg->comm->myProc_x, hg->comm->myProc_y, hg->nVtx, nEdge); interval = (int*)ZOLTAN_MALLOC(nEdge*2*sizeof(int)); if ((nEdge > 0 ) && (interval == NULL)) MEMORY_ERROR; tree_size = get_tree_size(tree) + 1; for (index = 0 ; index < nEdge ; ++index){ SET_MIN_NODE(interval, index, tree_size); SET_MAX_NODE(interval, index, -1); } /* Update interval with the local knowledge */ /* XXX: I loop on the hyperedges, as I think it's more cache friendly * and it allows me to discoupled the computation if a non blocking MPI_Reduce is * available */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge){ int part; int max = -1; /* Trick : we use the initialized values */ int min = tree_size + 1; for (index = hg->hindex[hEdge] ; index < hg->hindex[hEdge+1] ; ++ index) { part = parts[hg->hvertex[index]]; max = MAX(max, part); min = MIN(min, part); } SET_MIN_NODE(interval, hEdge, min); SET_MAX_NODE(interval, hEdge, max); } /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace (interval, 2*nEdge, MPI_INT, MPI_MAX, hg->comm->row_comm); /* Now I have to compute the partition of hyperedges according to the "interval" * and the tree */ /* First, compute the partition number corresponding to the nodes in the tree */ partnumber = compute_part_number(tree); if (partnumber == NULL) { ierr = ZOLTAN_FATAL; goto End; } (*numParts) = get_tree_size(tree); rowpart = (ZOLTAN_ID_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_ID_TYPE)); if ((nEdge > 0) && (rowpart == NULL)) MEMORY_ERROR; rowGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nEdge*sizeof(ZOLTAN_GNO_TYPE)); if ((nEdge > 0) && (rowGNO == NULL)) MEMORY_ERROR; (*sizeParts) = (int*)ZOLTAN_CALLOC((*numParts), sizeof(int)); if (*numParts && (*sizeParts) == NULL) MEMORY_ERROR; offset = hg->dist_y[hg->comm->myProc_y]; /* Then we search we is the hyperedge in the tree */ for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { int node; node = find_interval_in_tree(tree, interval+2*hEdge); rowpart[hEdge] = partnumber[node]; (*sizeParts)[rowpart[hEdge]] ++; rowGNO[hEdge] = EDGE_LNO_TO_GNO(hg, hEdge); #if 0 fprintf (stderr, "%zd : " ZOLTAN_ID_SPEC " (%d : %d - %d)\n", rowGNO[hEdge], rowpart[hEdge], node, -interval[2*hEdge], interval[2*hEdge+1]); #endif } partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); /* Compute number of elements per parts */ /* TODO: support processor which are not part of the distribution */ /* Update results to view the complete hyperedges */ Zoltan_AllReduceInPlace ((*sizeParts), (*numParts), MPI_INT, MPI_SUM, hg->comm->col_comm); /* Export results to data directory */ /* First, get the GIDs of our edges */ rowGID = ZOLTAN_MALLOC_GID_ARRAY(zz, nEdge); if (nEdge && rowGID == NULL) MEMORY_ERROR; ierr = Zoltan_DD_Find (gnoToGID , (ZOLTAN_ID_PTR)rowGNO, rowGID, NULL, NULL, nEdge, NULL); ZOLTAN_FREE(&rowGNO); ierr = Zoltan_DD_Create (dd, zz->Communicator, zz->Num_GID, 1, 0, nEdge, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (*dd, (ZOLTAN_ID_PTR)rowGID, rowpart, NULL, NULL, nEdge); #ifdef CEDRIC_PRINT for (hEdge = 0 ; hEdge < nEdge ; ++hEdge) { fprintf (stderr, "%d : %d\n", rowGID[hEdge], rowpart[hEdge]); } #endif End: ZOLTAN_FREE(&rowGID); ZOLTAN_FREE(&rowGNO); ZOLTAN_FREE(&rowpart); if (partnumber != NULL) partnumber += 1; ZOLTAN_FREE(&partnumber); ZOLTAN_FREE(&interval); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
/* * void lb_oct_init(); * * initialize the calls needed to start the octree load balancing rounties */ static int lb_oct_init( ZZ *zz, /* The Zoltan structure with info for the OCTPART balancer. */ int *num_export, /* Number of non-local objs assigned to this processor in the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids, /* Returned value: array of global IDs for non-local objects in this processor's new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Returned value: array of local IDs for non-local objects in this processor's new decomposition. */ int **export_procs, /* Returned value: array of processor IDs for processors owning the non-local objects in this processor's new decomposition. */ int **export_to_part, /* Returned value: array of partitions to which objects are imported. KDDKDD Assume #parts==#procs. */ int oct_dim, /* Dimension of method (2D or 3D) */ int oct_method, /* Flag specifying curve to be used. */ int oct_maxoctregions, /* max # of objects in leaves of octree. */ int oct_minoctregions, /* min # of objects in leaves of octree. */ int oct_output_level, /* Flag specifying amount of output. */ int oct_wgtflag, /* Flag specifying use of object weights. */ float *part_sizes /* Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ ) { char *yo = "lb_oct_init"; OCT_Global_Info *OCT_info; int nsentags; /* number of tags being sent */ pRegion export_regs; /* */ int nrectags; /* number of tags received */ int kk; double time1,time2; /* timers */ double timestart,timestop; /* timers */ double timers[4]; /* diagnostic timers 0 = start-up time before recursion 1 = time before median iterations 2 = time in median iterations 3 = communication time */ int counters[6]; /* diagnostic counts 0 = # of median iterations 1 = # of objects sent 2 = # of objects received 3 = most objects this proc ever owns */ float c[4]; int createpartree = 0; /*int num_gid_entries = zz->Num_GID;*/ /*int num_lid_entries = zz->Num_LID;*/ ZOLTAN_TRACE_ENTER(zz, yo); MPI_Barrier(zz->Communicator); timestart = MPI_Wtime(); /* initialize timers and counters */ counters[0] = 0; counters[1] = 0; counters[2] = 0; counters[3] = 0; counters[4] = 0; counters[5] = 0; c[0] = 0; c[1] = 0; c[2] = 0; c[3] = 0; timers[1] = 0.0; timers[2] = 0.0; timers[3] = 0.0; nsentags = nrectags = 0; if(zz->LB.Data_Structure == NULL) { OCT_info = Zoltan_Oct_POct_init(zz, zz->Proc, oct_dim); Zoltan_Oct_set_method(OCT_info, oct_method); Zoltan_Oct_set_maxregions(oct_maxoctregions); Zoltan_Oct_set_minregions(oct_minoctregions); createpartree = 1; } else { OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); } /* create the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_gen_tree_from_input_data"); Zoltan_Oct_gen_tree_from_input_data(zz, oct_wgtflag, &counters[1], &counters[2], &counters[3], &c[0], createpartree); time2 = MPI_Wtime(); timers[0] = time2 - time1; /* time took to create octree */ /* Zoltan_Oct_POct_printResults(OCT_info); */ /* partition the octree structure */ time1 = MPI_Wtime(); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_partition"); /* old call to dfs_paritition: */ #if 0 Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1]); #else /*************************** if(zz->Proc == 0) { int debug_i; for(debug_i=0; debug_i<zz->Num_Proc; debug_i++) { fprintf(stdout,"Part_size[%d] = %f\n", debug_i, part_sizes[debug_i]); } } ****************************/ Zoltan_Oct_dfs_partition(zz, &counters[0], &c[1], part_sizes); #endif time2 = MPI_Wtime(); timers[1] = time2 - time1; /* time took to partition octree */ if (oct_output_level > 2) { Zoltan_Oct_Plots(zz); } /* set up tags for migrations */ time1 = MPI_Wtime(); #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_dfs_migrate"); Zoltan_Oct_dfs_migrate(zz, &nsentags, &export_regs, &nrectags, &c[2], &c[3], &counters[3], &counters[5]); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_fix_tags"); if (zz->LB.Return_Lists) { *num_export = nrectags; if (nrectags > 0) Zoltan_Oct_fix_tags(zz, export_global_ids, export_local_ids, export_procs, export_to_part, nrectags, export_regs); } time2 = MPI_Wtime(); timers[2] = time2 - time1; /* time took to setup migration */ #if 0 /* KDDKDD -- Count is never used; why is it computed? */ { /* count the number of objects on this processor */ pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ int count = 0; RootList = Zoltan_Oct_POct_localroots(OCT_info); while((RootOct = RL_nextRootOctant(&RootList))) { while(RootOct) { if(Zoltan_Oct_isTerminal(RootOct)) { count += Zoltan_Oct_nRegions(RootOct); } RootOct = Zoltan_Oct_POct_nextDfs(OCT_info, RootOct); } } } #endif counters[4] = nsentags; MPI_Barrier(zz->Communicator); timestop = MPI_Wtime(); if (oct_output_level > 0) { ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_print_stats"); Zoltan_Oct_print_stats(zz, timestop-timestart, timers, counters, c, oct_output_level); } for (kk = 0; kk < nrectags; kk++) { ZOLTAN_FREE(&(export_regs[kk].Global_ID)); ZOLTAN_FREE(&(export_regs[kk].Local_ID)); } ZOLTAN_FREE(&export_regs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_global_clear"); Zoltan_Oct_global_clear(OCT_info); /* KDDKDD Don't understand how re-used octree will work, especially without * KDDKDD the Zoltan_Oct_Bounds_Geom function. For now, we'll delete everything; * KDDKDD we can move back to saving some of the tree later. */ Zoltan_Oct_Free_Structure(zz); /* KDDKDD END */ /* Temporary return value until error codes are fully implemented. */ ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_OK); }
static int Zoltan_LB( ZZ *zz, int include_parts, /* Flag indicating whether to generate partition informtion; 0 if called by Zoltan_LB_Balance, 1 if called by Zoltan_LB_Partition. */ int *changes, /* Set to zero or one depending on if Zoltan determines a new decomposition or not: zero - No changes to the decomposition were made by the load-balancing algorithm; migration is not needed. one - A new decomposition is suggested by the load-balancer; migration is needed to establish the new decomposition. */ int *num_gid_entries, /* The number of array entries in a global ID; set to be the max over all processors in zz->Communicator of the parameter Num_Global_ID_Entries. */ int *num_lid_entries, /* The number of array entries in a local ID; set to be the max over all processors in zz->Communicator of the parameter Num_Local_ID_Entries. */ int *num_import_objs, /* The number of non-local objects in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects (i.e., objs to be imported) in the processor's new decomposition. */ int **import_procs, /* Array of processor IDs for processors currently owning non-local objects (i.e., objs to be imported) in this processor's new decomposition. */ int **import_to_part, /* Partition to which the objects should be imported. */ int *num_export_objs, /* The number of local objects that need to be exported from the processor to establish the new decomposition. */ ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need to be exported (assigned and sent to other processors) to establish the new decomposition. */ int **export_procs, /* Array of destination processor IDs for objects that need to be exported to establish the new decomposition. */ int **export_to_part /* Partition to which objects should be exported. */ ) { /* * Main load-balancing routine. * Input: a Zoltan structure with appropriate function pointers set. * Output: * changes * num_import_objs * import_global_ids * import_local_ids * import_procs * import_to_part * num_export_objs * export_global_ids * export_local_ids * export_procs * export_to_part * Return values: * Zoltan error code. */ char *yo = "Zoltan_LB"; int gmax; /* Maximum number of imported/exported objects over all processors. */ int error = ZOLTAN_OK; /* Error code */ double start_time, end_time; double lb_time[2] = {0.0,0.0}; char msg[256]; int comm[3],gcomm[3]; float *part_sizes = NULL, *fdummy = NULL; int wgt_dim, part_dim; int all_num_obj, i, ts, idIdx; struct Hash_Node **ht; int *export_all_procs, *export_all_to_part, *parts=NULL; ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL; ZOLTAN_ID_PTR gid; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); #ifdef ZOLTAN_DRUM /* initialize DRUM if needed */ Zoltan_Drum_Create_Model(zz); /* stop DRUM monitors */ Zoltan_Drum_Stop_Monitors(zz); #endif /* * Compute Max number of array entries per ID over all processors. * Compute Max number of return arguments for Zoltan_LB_Balance. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; comm[2] = zz->LB.Return_Lists; MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = *num_gid_entries = gcomm[0]; zz->Num_LID = *num_lid_entries = gcomm[1]; zz->LB.Return_Lists = gcomm[2]; /* assume no changes */ *changes = 0; *num_import_objs = *num_export_objs = 0; *import_global_ids = NULL; *import_local_ids = NULL; *import_procs = NULL; *import_to_part = NULL; *export_global_ids = NULL; *export_local_ids = NULL; *export_procs = NULL; *export_to_part = NULL; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) goto End; if (zz->LB.Method == NONE) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s Balancing method selected == NONE; no balancing performed\n", yo); error = ZOLTAN_WARN; goto End; } /* * Sync the random number generator across processors. */ Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator); /* * Construct the heterogenous machine description. */ error = Zoltan_Build_Machine_Desc(zz); if (error == ZOLTAN_FATAL) goto End; ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* Since generating a new partition, need to free old mapping vector */ zz->LB.OldRemap = zz->LB.Remap; zz->LB.Remap = NULL; error = Zoltan_LB_Build_PartDist(zz); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) goto End; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { int i, np, fp; for (i = 0; i < zz->Num_Proc; i++) { Zoltan_LB_Proc_To_Part(zz, i, &np, &fp); printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n", zz->Proc, i, np, fp); } } /* * Generate partitions sizes. */ #ifdef ZOLTAN_DRUM /* set partition sizes computed by DRUM, if requested */ Zoltan_Drum_Set_Part_Sizes(zz); #endif wgt_dim = zz->Obj_Weight_Dim; part_dim = ((wgt_dim > 0) ? wgt_dim : 1); part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim * zz->LB.Num_Global_Parts); if (part_sizes == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); error = ZOLTAN_MEMERR; goto End; } /* Get partition sizes. */ Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim, part_sizes); /* * Call the actual load-balancing function. */ error = zz->LB.LB_Fn(zz, part_sizes, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_FREE(&part_sizes); if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){ sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } else if (error){ if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) { sprintf(msg, "Partitioning routine returned code %d.", error); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning"); #ifdef ZOLTAN_DRUM /* restart DRUM monitors -- should happen later but there are a lot of ways out of Zoltan_LB and we want to make sure they do start */ Zoltan_Drum_Start_Monitors(zz); #endif if (*num_import_objs >= 0) MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); else /* use export data */ MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX, zz->Communicator); if (gmax == 0) { /* * Decomposition was not changed by the load balancing; no migration * is needed. */ if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) printf("%s No changes to the decomposition due to partitioning; " "no migration is needed.\n", yo); /* * Reset num_import_objs and num_export_objs; don't want to return * -1 for the arrays that weren't returned by ZOLTAN_LB_FN. */ *num_import_objs = *num_export_objs = 0; if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){ /* * This parameter setting requires that all local objects * and their assignments appear in the export list. */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } } goto End; } /* * Check whether we know the import data, export data, or both. * * If we were given the import data, * we know what the new decomposition should look like on the * processor, but we don't know which of our local objects we have * to export to other processors to establish the new decomposition. * Reverse the argument if we were given the export data. * * Unless we were given both maps, compute the inverse map. */ if (*num_import_objs >= 0){ if (*num_export_objs >= 0) { /* Both maps already available; nothing to do. */; if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) { /* This condition should never happen!! */ /* Methods should not return arrays if no lists are requested. */ *num_import_objs = *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); ZOLTAN_PRINT_WARN(zz->Proc, yo, "Method returned lists, but no lists requested."); } } else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Export lists are requested; compute export map */ error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* Method returned import lists, but only export lists were desired. */ /* Import lists not needed; free them. */ *num_import_objs = -1; Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids, import_procs, import_to_part); } } } else { /* (*num_import_objs < 0) */ if (*num_export_objs >= 0) { /* Only export lists have been returned. */ if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS || zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Compute import map */ error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part, num_import_objs, import_global_ids, import_local_ids, import_procs, import_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error building return arguments; " "%d returned by Zoltan_Compute_Destinations\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) { /* Method returned export lists, but only import lists are desired. */ /* Export lists not needed; free them. */ *num_export_objs = -1; Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); } } } else { /* *num_export_objs < 0 && *num_import_objs < 0) */ if (zz->LB.Return_Lists) { /* No map at all available */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned " "neither import nor export data."); error = ZOLTAN_WARN; goto End; } } } if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) { /* * Normally, Zoltan_LB returns in the export lists all local * objects that are moving off processor, or that are assigned * to a partition on the local processor that is not the * default partition. This setting of Return_Lists requests * that all local objects be included in the export list. */ if (*num_export_objs == 0){ /* all local objects are remaining on processor */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs, export_global_ids, export_local_ids, wgt_dim, &fdummy, export_to_part); if (error == ZOLTAN_OK){ ZOLTAN_FREE(&fdummy); if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs, ZOLTAN_SPECIAL_MALLOC_INT)){ for (i=0; i<*num_export_objs; i++) (*export_procs)[i] = zz->Proc; } else{ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End; } else{ all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error); if (*num_export_objs < all_num_obj){ /* Create a lookup table for exported IDs */ if (*num_export_objs > 16){ /* could be 0, maybe only importing */ ts = (*num_export_objs) / 4; /* what's a good table size? */ } else{ ts = *num_export_objs; } ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts); /* Create a list of all gids, lids and partitions */ error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj, &all_global_ids, &all_local_ids, wgt_dim, &fdummy, &parts); if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){ ZOLTAN_FREE(&fdummy); if ((Zoltan_Special_Malloc(zz, (void **)&export_all_procs, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) || (Zoltan_Special_Malloc(zz, (void **)&export_all_to_part, all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){ error = ZOLTAN_MEMERR; } } if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){ sprintf(msg, "Error building complete export list; " "%d returned by Zoltan_Get_Obj_List\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } gid = all_global_ids; for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){ idIdx = search_hash_table(zz, gid, ht, ts); if (idIdx >= 0){ export_all_procs[i] = (*export_procs)[idIdx]; export_all_to_part[i] = (*export_to_part)[idIdx]; } else{ export_all_procs[i] = zz->Proc; export_all_to_part[i] = parts[i]; } } free_hash_table(ht, ts); Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids, export_procs, export_to_part); Zoltan_Special_Free(zz, (void **)&parts, ZOLTAN_SPECIAL_MALLOC_INT); *export_global_ids = all_global_ids; *export_local_ids = all_local_ids; *export_procs = export_all_procs; *export_to_part = export_all_to_part; *num_export_objs = all_num_obj; } } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments"); end_time = Zoltan_Time(zz->Timer); lb_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc); for (i = 0; i < *num_import_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID])); printf(" To partition: %4d", (*import_to_part != NULL ? (*import_to_part)[i] : zz->Proc)); printf(" From processor: %4d\n", (*import_procs)[i]); } printf("\n"); printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc); for (i = 0; i < *num_export_objs; i++) { printf(" Obj: "); ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID])); printf(" To partition: %4d", (*export_to_part != NULL ? (*export_to_part)[i] : (*export_procs)[i])); printf(" To processor: %4d\n", (*export_procs)[i]); } Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* * If the Help_Migrate flag is set, perform migration for the application. */ if (zz->Migrate.Auto_Migrate) { ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration"); start_time = Zoltan_Time(zz->Timer); error = Zoltan_Migrate(zz, *num_import_objs, *import_global_ids, *import_local_ids, *import_procs, *import_to_part, *num_export_objs, *export_global_ids, *export_local_ids, *export_procs, *export_to_part); if (error != ZOLTAN_OK && error != ZOLTAN_WARN) { sprintf(msg, "Error in auto-migration; %d returned from " "Zoltan_Help_Migrate\n", error); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); goto End; } end_time = Zoltan_Time(zz->Timer); lb_time[1] = end_time - start_time; ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration"); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0], "ZOLTAN Partition: "); if (zz->Migrate.Auto_Migrate) Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1], "ZOLTAN Migrate: "); } *changes = 1; End: ZOLTAN_TRACE_EXIT(zz, yo); return (error); }
/* * Each processor: * owns a set of pins (nonzeros) * may provide some edge weights * * We assume that no two processes will supply the same pin. * But more than one process may supply pins for the same edge. */ static int matrix_get_edges(ZZ *zz, Zoltan_matrix *matrix, ZOLTAN_ID_PTR *yGID, ZOLTAN_ID_PTR *pinID, int nX, ZOLTAN_ID_PTR *xGID, ZOLTAN_ID_PTR *xLID, ZOLTAN_GNO_TYPE **xGNO, float **xwgt, int use_full_dd) { static char *yo = "matrix_get_edges"; int ierr = ZOLTAN_OK; int hypergraph_callbacks = 0, graph_callbacks = 0; int *nbors_proc = NULL; /* Pointers are global for the function to ensure proper free */ int *edgeSize = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Get_HG_Size_CS && zz->Get_HG_CS) { hypergraph_callbacks = 1; } if ((zz->Get_Num_Edges != NULL || zz->Get_Num_Edges_Multi != NULL) && (zz->Get_Edge_List != NULL || zz->Get_Edge_List_Multi != NULL)) { graph_callbacks = 1; } /* TEMPORARY FIX */ if (!graph_callbacks){ fprintf(stderr,"Bug #5470: matrix_get_edges fails for hypergraph queries\n"); return ZOLTAN_FATAL; } hypergraph_callbacks=0; /* TEMPORARY FIX */ if (graph_callbacks && hypergraph_callbacks){ /* if (hgraph_model == GRAPH) */ /* hypergraph_callbacks = 0; */ graph_callbacks = 1; /* I prefer graph (allow to do "inplace") ! */ } if (hypergraph_callbacks) { matrix->redist = 1; if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID)) ZOLTAN_FREE(xGID); else *xGID = NULL; ZOLTAN_FREE(xLID); ZOLTAN_FREE(xGNO); ierr = Zoltan_Hypergraph_Queries(zz, &matrix->nY, &matrix->nPins, yGID, &matrix->ystart, pinID); CHECK_IERR; matrix->yend = matrix->ystart + 1; } else if (graph_callbacks) { int max_edges = 0; int vertex; int numGID, numLID; matrix->opts.enforceSquare = 1; matrix->nY = nX; /* It is square ! */ matrix->yGNO = *xGNO; *yGID = NULL; matrix->ywgtdim = zz->Obj_Weight_Dim; *xwgt = NULL; numGID = zz->Num_GID; numLID = zz->Num_LID; /* TODO : support local graphs */ /* TODO : support weights ! */ /* Get edge data */ Zoltan_Get_Num_Edges_Per_Obj(zz, matrix->nY, *xGID, *xLID, &edgeSize, &max_edges, &matrix->nPins); (*pinID) = ZOLTAN_MALLOC_GID_ARRAY(zz, matrix->nPins); nbors_proc = (int *)ZOLTAN_MALLOC(matrix->nPins * sizeof(int)); if (matrix->nPins && ((*pinID) == NULL || nbors_proc == NULL)) MEMORY_ERROR; matrix->pinwgt = (float*)ZOLTAN_MALLOC(matrix->nPins*matrix->pinwgtdim*sizeof(float)); if (matrix->nPins && matrix->pinwgtdim && matrix->pinwgt == NULL) MEMORY_ERROR; if (zz->Get_Edge_List_Multi) { zz->Get_Edge_List_Multi(zz->Get_Edge_List_Multi_Data, numGID, numLID, matrix->nY, *xGID, *xLID, edgeSize, (*pinID), nbors_proc, matrix->pinwgtdim, matrix->pinwgt, &ierr); } else { int edge; for (vertex = 0, edge = 0 ; vertex < matrix->nY ; ++vertex) { zz->Get_Edge_List(zz->Get_Edge_List_Data, numGID, numLID, (*xGID)+vertex*numGID, (*xLID)+vertex*numLID, (*pinID)+edge*numGID, nbors_proc+edge, matrix->pinwgtdim, matrix->pinwgt+edge*matrix->pinwgtdim, &ierr); edge += edgeSize[vertex]; } } CHECK_IERR; /* Not Useful anymore */ ZOLTAN_FREE(xLID); if (use_full_dd || ((ZOLTAN_ID_PTR) *xGNO != *xGID)) { ZOLTAN_FREE(xGID); *xGNO = NULL; } else { *xGID = NULL; *xGNO = NULL; } ZOLTAN_FREE(&nbors_proc); /* Now construct CSR indexing */ matrix->ystart = (int*) ZOLTAN_MALLOC((matrix->nY+1)*sizeof(int)); if (matrix->ystart == NULL) MEMORY_ERROR; matrix->ystart[0] = 0; matrix->yend = matrix->ystart + 1; for (vertex = 0 ; vertex < matrix->nY ; ++vertex) matrix->ystart[vertex+1] = matrix->ystart[vertex] + edgeSize[vertex]; } else { FATAL_ERROR ("You have to define Hypergraph or Graph queries"); } if (matrix->opts.enforceSquare) { matrix->globalY = matrix->globalX; matrix->ddY = matrix->ddX; matrix->ywgtdim = zz->Obj_Weight_Dim; } End: ZOLTAN_FREE(&edgeSize); ZOLTAN_FREE(&nbors_proc); ZOLTAN_FREE(xLID); ZOLTAN_FREE(xGID); ZOLTAN_FREE(xGNO); ZOLTAN_FREE(xwgt); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
void Zoltan_HG_Print( ZZ *zz, HGraph *hg, Partition parts, FILE *fp, char *str ) { /* Routine to print hypergraph weights and edges. Assumes serial execution; * put inside Zoltan_Print_Sync_Start/Zoltan_Print_Sync_End for parallel * programs. */ int i, j; int num_vwgt; int num_ewgt; float *sum; char *yo = "Zoltan_HG_Print"; if (hg == NULL) return; ZOLTAN_TRACE_ENTER(zz, yo); num_vwgt = hg->VtxWeightDim; num_ewgt = hg->EdgeWeightDim; sum = (float *) ZOLTAN_MALLOC(MAX(num_vwgt, num_ewgt) * sizeof(float)); fprintf(fp, "%s nVtx=%d nEdge=%d nPins=%d vWgt=%d eWgt=%d\n", str, hg->nVtx, hg->nEdge, hg->nPins, hg->VtxWeightDim, hg->EdgeWeightDim); /* Print Vertex Info */ fprintf(fp, "%s Vertices: (edges)\n", str); for (i = 0; i < hg->nVtx; i++) { fprintf(fp, "%d (" ZOLTAN_GNO_SPEC ") in part %d: ", i, VTX_LNO_TO_GNO(hg, i), (parts ? parts[i] : -1)); fprintf(fp, "("); for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++) fprintf(fp, "%d ", hg->vedge[j]); fprintf(fp, ")\n"); } if (hg->vwgt != NULL) { for (j = 0; j < num_vwgt; j++) sum[j] = 0; fprintf(fp, "%s Vertices: [weights])\n", str); for (i = 0; i < hg->nVtx; i++) { fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): [", i, VTX_LNO_TO_GNO(hg, i)); for (j = 0; j < num_vwgt; j++) { fprintf(fp, "%f ", hg->vwgt[i*num_vwgt + j]); sum[j] += hg->vwgt[i*num_vwgt + j]; } fprintf(fp, "])\n"); } fprintf(fp, "Total vertex weight = ["); for (j = 0; j < num_vwgt; j++) fprintf(fp, "%f ", sum[j]); fprintf(fp, "]\n"); } /* Print Hyperedge Info */ fprintf(fp, "%s Hyperedges: (vertices)\n", str); for (i = 0; i < hg->nEdge; i++) { fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): ", i, EDGE_LNO_TO_GNO(hg, i)); fprintf(fp, "("); for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++) fprintf(fp, "%d ", hg->hvertex[j]); fprintf(fp, ")\n"); } if (hg->ewgt != NULL) { for (j = 0; j < num_ewgt; j++) sum[j] = 0; fprintf(fp, "%s Hyperedge Weights: [weights]\n", str); for (i = 0; i < hg->nEdge; i++) { fprintf(fp, "%d (" ZOLTAN_GNO_SPEC "): ", i, EDGE_LNO_TO_GNO(hg, i)); fprintf(fp, "["); for (j = 0; j < num_ewgt; j++) { fprintf(fp, "%f ", hg->ewgt[i*num_ewgt + j]); sum[j] += hg->ewgt[i*num_ewgt + j]; } fprintf(fp, "])\n"); } fprintf(fp, "Total hyperedge weight = ["); for (j = 0; j < num_ewgt; j++) fprintf(fp, "%f ", sum[j]); fprintf(fp, "]\n"); } ZOLTAN_FREE(&sum); ZOLTAN_TRACE_EXIT(zz, yo); }
int Zoltan_HG_Info ( ZZ *zz, HGraph *hg ) { int i, dd, size, size_min, size_max, count; float wgt_min, wgt_max, wgt_tot; double mean, var, temp; char *yo = "Zoltan_HG_Info"; ZOLTAN_TRACE_ENTER(zz, yo); printf("--------- HGraph Information (min/ave/max/tot) ------------------\n"); printf("INFO MAY BE WRONG FOR 2D DATA DISTRIBUTION. KDDKDD \n"); printf("info:%d |V|=%d |E|=%d |P|=%d \n", hg->info, hg->nVtx, hg->nEdge, hg->nPins); /* print weights */ if (hg->nVtx && hg->vwgt) { for (dd = 0; dd < hg->VtxWeightDim; dd++) { wgt_tot = 0.0; wgt_min = FLT_MAX; wgt_max = FLT_MIN; for (i = 0; i < hg->nVtx; i++) { wgt_tot += hg->vwgt[i*hg->VtxWeightDim+dd]; wgt_min = MIN(wgt_min, hg->vwgt[i*hg->VtxWeightDim+dd]); wgt_max = MAX(wgt_max, hg->vwgt[i*hg->VtxWeightDim+dd]); } printf("Vertex weights : %9.2f %9.2f %9.2f %12.2f\n", wgt_min, wgt_tot/hg->nVtx, wgt_max, wgt_tot); mean = var = 0.0; if (hg->nVtx > 1) { mean = wgt_tot / hg->nVtx; for (i = 0; i < hg->nVtx; i++) { temp = hg->vwgt[i*hg->VtxWeightDim+dd] - mean; var += (temp*temp); } var = sqrt(var/(hg->nVtx-1)); printf ("Vertex Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean); } count=0; temp=0.0; for (i = 0; i < hg->nVtx; i++) if (hg->vwgt[i*hg->VtxWeightDim+dd] > (mean + 3.0 * var)) { count++; temp += hg->vwgt[i*hg->VtxWeightDim+dd]; } printf ("Vertex >3sigma: count %d, rel count x10000 %.1f, " "rel weight x100 %.1f\n", count, (float)10000*count/(float)hg->nVtx, 100.0*temp/wgt_tot); } } if (hg->nEdge && hg->ewgt) { wgt_tot = 0.0; wgt_min = FLT_MAX; wgt_max = FLT_MIN; for (i = 0; i < hg->nEdge; i++) { wgt_tot += hg->ewgt[i]; wgt_min = MIN(wgt_min, hg->ewgt[i]); wgt_max = MAX(wgt_max, hg->ewgt[i]); } printf("HEdge weights : %9.2f %9.2f %9.2f %12.2f\n", wgt_min, wgt_tot/hg->nEdge, wgt_max, wgt_tot); if (hg->nEdge > 1) { var = 0.0; mean = wgt_tot / hg->nEdge; for (i = 0; i < hg->nEdge; i++) { temp = hg->ewgt[i] - mean; var += (temp*temp); } var = sqrt(var/(hg->nEdge-1)); printf ("HEdge Stats: STDV %.2f, Coef of Var %.2f\n", var, var/mean); } } /* print sizes */ if (hg->nPins && hg->hindex) { size_min = INT_MAX; size_max = INT_MIN; for (i = 0; i < hg->nEdge; i++) { size = hg->hindex[i+1] - hg->hindex[i]; size_min = MIN(size_min, size); size_max = MAX(size_max, size); } printf("Edge sizes : %6d %9.2f %6d %9d\n", size_min, (float)hg->nPins / hg->nEdge, size_max, hg->nPins); if (hg->nEdge > 1) { var = 0.0; mean = (float)hg->nPins / hg->nEdge; for (i = 0; i < hg->nEdge; i++) { temp = (float)(hg->hindex[i+1]-hg->hindex[i]) - mean; var += (temp*temp); } var = sqrt(var/(hg->nEdge-1)); printf ("Edge Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean); } } if (hg->nPins && hg->vindex) { size_min = INT_MAX; size_max = INT_MIN; for (i = 0; i < hg->nVtx; i++) { size = hg->vindex[i+1] - hg->vindex[i]; size_min = MIN(size_min, size); size_max = MAX(size_max, size); } printf("Vertex sizes : %6d %9.2f %6d %9d\n", size_min, (float)hg->nPins / hg->nVtx, size_max, hg->nPins); if (hg->nVtx > 1) { var = 0.0; mean = (float)hg->nPins / hg->nVtx; for (i = 0; i < hg->nVtx; i++) { temp = (float)(hg->vindex[i+1]-hg->vindex[i]) - mean; var += (temp*temp); } var = sqrt(var/(hg->nVtx-1)); printf ("Vertex Stats: stdev %.2f, Coef of Var %.2f\n", var, var/mean); } } printf("-----------------------------------------------------------------\n"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_OK; }
int Zoltan_HG_Create_Mirror ( ZZ *zz, HGraph *hg ) { int inlength, outlength; /* input/output array lengths */ int *index; /* pointers to input information */ int *outindex; int *data, *outdata; char *yo = "Zoltan_HG_Create_Mirror"; ZOLTAN_TRACE_ENTER(zz, yo); /* determine which data to "mirror" and set corresponding data pointers. */ if (hg && (hg->nEdge == 0 || hg->hindex) && (hg->nPins == 0 || hg->hvertex) && !hg->vindex && !hg->vedge) { ZOLTAN_TRACE_DETAIL(zz, yo, "Have hindex; building vindex."); inlength = hg->nEdge; outlength = hg->nVtx; index = hg->hindex; data = hg->hvertex; outindex = hg->vindex = (int*) ZOLTAN_MALLOC((hg->nVtx+1) * sizeof(int)); outdata = hg->vedge = (int*) ZOLTAN_MALLOC (hg->nPins * sizeof(int)); if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) { Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->vindex, &hg->vedge); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } else if (hg && (hg->nVtx == 0 || hg->vindex) && (hg->nPins == 0 || hg->vedge) && !hg->hindex && !hg->hvertex) { ZOLTAN_TRACE_DETAIL(zz, yo, "Have vindex; building hindex."); inlength = hg->nVtx; outlength = hg->nEdge; index = hg->vindex; data = hg->vedge; outindex = hg->hindex = (int*) ZOLTAN_MALLOC((hg->nEdge+1) * sizeof(int)); outdata = hg->hvertex = (int*) ZOLTAN_MALLOC(hg->nPins * sizeof(int)); if (outindex == NULL || (hg->nPins > 0 && outdata == NULL)) { Zoltan_Multifree (__FILE__, __LINE__, 2, &hg->hindex, &hg->hvertex); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } else { ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input error."); return ZOLTAN_FATAL; /* unable to proceed */ } Zoltan_HG_Mirror(inlength, index, data, outlength, outindex, outdata); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_OK; }
int Zoltan_HG_Check ( ZZ *zz, HGraph *hg ) { int i; int iedge; /* iedge denotes an hyperedge */ int j; /* j is the index of a vertex */ int k; /* k is an index of hyperedge */ int *check; char str[256]; int err = ZOLTAN_OK; char *yo = "Zoltan_HG_Check"; ZOLTAN_TRACE_ENTER(zz, yo); if ((hg->nEdge && !hg->hindex) || (hg->nVtx && !hg->vindex) || (hg->nPins && (!hg->vedge || !hg->hvertex))) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "NULL arrays found"); err = ZOLTAN_WARN; goto End; } if (hg->nPins) { check = (int*) ZOLTAN_MALLOC(((hg->nEdge > hg->nVtx) ? hg->nEdge : hg->nVtx) * sizeof(int)); if (check == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to allocate memory."); err = ZOLTAN_MEMERR; goto End; } for (i = 0; i < hg->nEdge; i++) check[i] = -1; for (i = 0; i < hg->nVtx; i++) for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++) if (check[hg->vedge[j]] < i) check[hg->vedge[j]] = i; else { ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple hedge for same vertex."); err = ZOLTAN_WARN; } for (i = 0; i < hg->nVtx; i++) check[i] = -1; for (i = 0; i < hg->nEdge; i++) for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++) if (check[hg->hvertex[j]] < i) check[hg->hvertex[j]] = i; else { ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple vertex for same hedge."); err = ZOLTAN_WARN; } ZOLTAN_FREE (&check); } for (i = 0; i < hg->VtxWeightDim * hg->nVtx; i++) if (hg->vwgt[i] < 0.0) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative vertex weight."); err = ZOLTAN_WARN; } for (i = 0; i < hg->EdgeWeightDim * hg->nEdge; i++) if (hg->ewgt[i] < 0.0) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative edge weight."); err = ZOLTAN_WARN; } if (hg->comm && hg->comm->nProc_x == 1) { /* In 2D distribution, check makes sense only if proc has entire hedge */ for (i = 0; i < hg->nEdge; i++) if ((hg->hindex[i+1] == hg->hindex[i])) { sprintf (str, "Found hyperedge with no vertices: " "edge %d has %d vtxs\n", i, (hg->hindex[i+1] - hg->hindex[i])); ZOLTAN_PRINT_WARN(zz->Proc, yo, str); err = ZOLTAN_WARN; } } for (i = 0; i < hg->nEdge; i++) for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++) if (hg->hvertex[j] < 0 || hg->hvertex[j] >= hg->nVtx) { ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found vertex out of range in hvertex."); err = ZOLTAN_WARN; } for (i = 0; i < hg->nVtx; i++) for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++) if (hg->vedge[j] < 0 || hg->vedge[j] >= hg->nEdge) { ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found edge out of range in vedge."); err = ZOLTAN_WARN; } /* starting from (hindex,hvertex), for each edge determine each associated * vertex. Then for each vertex lookup associated edges using (vindex, vedge) */ for (iedge = 0; iedge < hg->nEdge; iedge++) for (j = hg->hindex[iedge]; j < hg->hindex[iedge+1]; j++) { /* for each hyperedge get index to vertices */ for (k=hg->vindex[hg->hvertex[j]]; k<hg->vindex[hg->hvertex[j]+1]; k++) /* for each vertex of current hyperedge get index to hyperedges */ if (hg->vedge[k] == iedge) /* does it match with original edge? */ break; if (k == hg->vindex[hg->hvertex[j]+1]) { /* if no match was found then failure, else keep on */ ZOLTAN_PRINT_WARN(zz->Proc, yo, "Inconsistent hvertex/vedge"); err = ZOLTAN_WARN; break; } } End: ZOLTAN_TRACE_EXIT(zz, yo); return err; }
int Zoltan_Get_Coordinates( ZZ *zz, int num_obj, /* Input: number of objects */ ZOLTAN_ID_PTR global_ids, /* Input: global IDs of objects */ ZOLTAN_ID_PTR local_ids, /* Input: local IDs of objects; may be NULL. */ int *num_dim, /* Output: dimension of coordinates */ double **coords /* Output: array of coordinates; malloc'ed by fn if NULL upon input. */ ) { char *yo = "Zoltan_Get_Coordinates"; int i,j,rc; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; int alloced_coords = 0; ZOLTAN_ID_PTR lid; /* Temporary pointers to local IDs; used to pass NULL to query functions when NUM_LID_ENTRIES == 0. */ double dist[3]; double im[3][3]; double deg_ratio; double x; int order[3]; int reduce_dimensions, d, axis_aligned; int target_dim; int ierr = ZOLTAN_OK; char msg[256]; ZZ_Transform *tran; ZOLTAN_TRACE_ENTER(zz, yo); /* Error check -- make sure needed callback functions are registered. */ if (zz->Get_Num_Geom == NULL || (zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and " "either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN"); goto End; } /* Get problem dimension. */ *num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_NUM_GEOM_FN"); goto End; } if (*num_dim < 0 || *num_dim > 3) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid dimension returned from ZOLTAN_NUM_GEOM_FN"); goto End; } /* Get coordinates for object; allocate memory if not already provided. */ if (*num_dim > 0 && num_obj > 0) { if (*coords == NULL) { alloced_coords = 1; *coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double)); if (*coords == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error"); goto End; } } if (zz->Get_Geom_Multi != NULL) { zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID, num_obj, global_ids, local_ids, *num_dim, *coords, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_GEOM_MULTI_FN"); goto End; } } else { for (i = 0; i < num_obj; i++) { lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL); zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries, global_ids + i*num_gid_entries, lid, (*coords) + i*(*num_dim), &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_GET_GEOM_FN"); goto End; } } } } /* * For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the * center of mass and inertial matrix of the coordinates. * * For 3D problems: If the geometry is "flat", transform the points so the * two primary directions lie along the X and Y coordinate axes and project * to the Z=0 plane. If in addition the geometry is "skinny", project to * the X axis. (This creates a 2D or 1D problem respectively.) * * For 2D problems: If the geometry is essentially a line, transform it's * primary direction to the X axis and project to the X axis, yielding a * 1D problem. * * Return these points to the partitioning algorithm, in effect partitioning * in only the 2 (or 1) significant dimensions. */ if (((*num_dim == 3) || (*num_dim == 2)) && ((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){ Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i); Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS", (void *)&reduce_dimensions); Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)°_ratio); i = 0; reduce_dimensions = 0; deg_ratio = 10.0; Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); if (reduce_dimensions == 0){ goto End; } if (deg_ratio <= 1){ if (zz->Proc == 0){ ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0"); } deg_ratio = 10.0; } if (zz->LB.Method == RCB){ tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran); } else if (zz->LB.Method == RIB){ tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran); } else{ tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran); } d = *num_dim; if (tran->Target_Dim >= 0){ /* * On a previous load balancing call, we determined whether * or not the geometry was degenerate. If the geometry was * determined to be not degenerate, then we assume it is still * not degenerate, and we skip the degeneracy calculation. */ if (tran->Target_Dim > 0){ /* * The geometry *was* degenerate. We test the extent * of the geometry along the principal directions determined * last time to determine if it is still degenerate with that * orientation. If so, we transform the coordinates using the * same transformation we used last time. If not, we do the * entire degeneracy calculation again. */ if ((tran->Axis_Order[0] >= 0) && (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){ axis_aligned = 1; } else{ axis_aligned = 0; } projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist, d, axis_aligned, tran->Axis_Order); target_dim = get_target_dimension(dist, order, deg_ratio, d); if (target_dim > 0){ transform_coordinates(*coords, num_obj, d, tran); } else{ /* Set's Target_Dim to -1, flag to recompute degeneracy */ Zoltan_Initialize_Transformation(tran); } } } if (tran->Target_Dim < 0){ tran->Target_Dim = 0; /* * Get the center of mass and inertial matrix of coordinates. Ignore * vertex weights, we are only interested in geometry. Global operation. */ if (d == 2){ inertial_matrix2D(zz, *coords, num_obj, tran->CM, im); } else{ inertial_matrix3D(zz, *coords, num_obj, tran->CM, im); } /* * The inertial matrix is a 3x3 or 2x2 real symmetric matrix. Get its * three or two orthonormal eigenvectors. These usually indicate the * orientation of the geometry. */ rc = eigenvectors(im, tran->Evecs, d); if (rc){ if (zz->Proc == 0){ ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed"); } goto End; } /* * Here we check to see if the eigenvectors are very close * to the coordinate axes. If so, we can more quickly * determine whether the geometry is degenerate, and also more * quickly transform the geometry to the lower dimensional * space. */ axis_aligned = 0; for (i=0; i<d; i++){ tran->Axis_Order[i] = -1; } for (j=0; j<d; j++){ for (i=0; i<d; i++){ x = fabs(tran->Evecs[i][j]); if (NEAR_ONE(x)){ tran->Axis_Order[j] = i; /* e'vector j is very close to i axis */ break; } } if (tran->Axis_Order[j] < 0){ break; } } if ((tran->Axis_Order[0] >= 0) && (tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){ axis_aligned = 1; } /* * Calculate the extent of the geometry along the three lines defined * by the direction of the eigenvectors through the center of mass. */ projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist, d, axis_aligned, tran->Axis_Order); /* * Decide whether these distances indicate the geometry is * very flat in one or two directions. */ target_dim = get_target_dimension(dist, order, deg_ratio, d); if (target_dim > 0){ /* * Yes, geometry is degenerate */ if ((zz->Debug_Level > 0) && (zz->Proc == 0)){ if (d == 2){ sprintf(msg, "Geometry (~%f x %f), exceeds %f to 1.0 ratio", dist[order[0]], dist[order[1]], deg_ratio); } else{ sprintf(msg, "Geometry (~%f x %f x %f), exceeds %f to 1.0 ratio", dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio); } ZOLTAN_PRINT_INFO(zz->Proc, yo, msg); sprintf(msg, "We'll treat it as %d dimensional",target_dim); ZOLTAN_PRINT_INFO(zz->Proc, yo, msg); } if (axis_aligned){ /* ** Create new geometry, transforming the primary direction ** to the X-axis, and the secondary to the Y-axis. */ tran->Permutation[0] = tran->Axis_Order[order[0]]; if (target_dim == 2){ tran->Permutation[1] = tran->Axis_Order[order[1]]; } } else{ /* * Reorder the eigenvectors (they're the columns of evecs) from * longest projected distance to shorted projected distance. Compute * the transpose (the inverse) of the matrix. This will transform * the geometry to align along the X-Y plane, or along the X axis. */ for (i=0; i< target_dim; i++){ tran->Transformation[i][2] = 0.0; for (j=0; j<d; j++){ tran->Transformation[i][j] = tran->Evecs[j][order[i]]; } } for (i=target_dim; i< 3; i++){ for (j=0; j<3; j++){ tran->Transformation[i][j] = 0.0; } } } tran->Target_Dim = target_dim; transform_coordinates(*coords, num_obj, d, tran); } /* If geometry is very flat */ } /* If REDUCE_DIMENSIONS is true */ } /* If 2-D or 3-D rcb, rib or hsfc */ End: if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned."); if (alloced_coords) ZOLTAN_FREE(coords); } ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
/* Point drop for refinement after above partitioning */ int Zoltan_HSFC_Point_Assign ( ZZ *zz, double *x, int *proc, int *part) { double scaled[3]; double pt[3]; double fsfc; Partition *p; int i; int dim; HSFC_Data *d; int err; char *yo = "Zoltan_HSFC_Point_Assign"; ZOLTAN_TRACE_ENTER (zz, yo); d = (HSFC_Data *) zz->LB.Data_Structure; if (d == NULL) ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL, "No Decomposition Data available; use KEEP_CUTS parameter."); for (i=0; i<d->ndimension; i++){ pt[i] = x[i]; /* we don't want to change caller's "x" */ } if (d->tran.Target_Dim > 0){ /* degenerate geometry */ dim = d->tran.Target_Dim; Zoltan_Transform_Point(pt, d->tran.Transformation, d->tran.Permutation, d->ndimension, dim, pt); } else{ dim = d->ndimension; } /* Calculate scaled coordinates, calculate HSFC coordinate */ for (i = 0; i < dim; i++) { scaled[i] = (pt[i] - d->bbox_lo[i]) / d->bbox_extent[i]; if (scaled[i] < HSFC_EPSILON) scaled[i] = HSFC_EPSILON; if (scaled[i] > 1.0 - HSFC_EPSILON) scaled[i] = 1.0 - HSFC_EPSILON; } fsfc = d->fhsfc (zz, scaled); /* Note, this is a function call */ /* Find partition containing point and return its number */ p = (Partition *) bsearch (&fsfc, d->final_partition, zz->LB.Num_Global_Parts, sizeof (Partition), Zoltan_HSFC_compare); if (p == NULL) ZOLTAN_HSFC_ERROR (ZOLTAN_FATAL, "programming error, shouldn't happen"); if (part != NULL) { if (zz->LB.Remap) *part = zz->LB.Remap[p->index]; else *part = p->index; } if (proc != NULL) { if (zz->LB.Remap) *proc = Zoltan_LB_Part_To_Proc(zz, zz->LB.Remap[p->index], NULL); else *proc = Zoltan_LB_Part_To_Proc(zz, p->index, NULL); } err = ZOLTAN_OK; End: ZOLTAN_TRACE_EXIT (zz, yo); return err; }
int Zoltan_LocalHSFC_Order( ZZ *zz, /* Zoltan structure */ int num_obj, /* Number of (local) objects to order. */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm, ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */ ) { static char *yo = "Zoltan_LocalHSFC_Order"; int n, ierr=ZOLTAN_OK; double (*fhsfc)(ZZ*, double*); /* space filling curve function */ int wgt_dim=0; float *obj_wgts=0; int *parts=0; int numGeomDims=0; double *geomArray=0; /* Variables for bounding box */ double *minValInDim; double *maxValInDim; double *widthDim; double *hsfcKey=0; int *coordIndx=0; /* Counters */ int objNum; int dimNum; int offset=0; int myrank; MPI_Comm_rank(MPI_COMM_WORLD,&myrank); ZOLTAN_TRACE_ENTER(zz, yo); /******************************************************************/ /* If for some reason order_opt is NULL, allocate a new ZOOS here. */ /* This should really never happen. */ /******************************************************************/ if (!order_opt) { order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS)); strcpy(order_opt->method,"LOCAL_HSFC"); } /******************************************************************/ /* local HSFC only computes the rank vector */ order_opt->return_args = RETURN_RANK; /******************************************************************/ /* Check that num_obj equals the number of objects on this proc. */ /* This constraint may be removed in the future. */ /******************************************************************/ n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error."); return(ZOLTAN_FATAL); } if (n != num_obj) { /* Currently this is a fatal error. */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the number of objects."); return(ZOLTAN_FATAL); } /******************************************************************/ /******************************************************************/ /* Get lists of objects */ /******************************************************************/ ierr = Zoltan_Get_Obj_List(zz, &n, &gids, &lids, wgt_dim, &obj_wgts, &parts); if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Obj_List returned error."); return(ZOLTAN_FATAL); } /******************************************************************/ /******************************************************************/ /* Get geometry for objects*/ /******************************************************************/ ierr = Zoltan_Get_Coordinates(zz, n, gids, lids, &numGeomDims, &geomArray); if (ierr != 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Zoltan_Get_Coordinates returned error."); return(ZOLTAN_FATAL); } /******************************************************************/ /******************************************************************/ /* Place coords in bounding box */ /******************************************************************/ minValInDim = (double *) malloc(numGeomDims * sizeof (double)); maxValInDim = (double *) malloc(numGeomDims * sizeof (double)); widthDim = (double *) malloc(numGeomDims * sizeof (double)); for(dimNum=0; dimNum<numGeomDims; dimNum++) { minValInDim[dimNum] = HUGE_VAL; maxValInDim[dimNum] = -HUGE_VAL; } /*************************************************************/ /* Determine min, max, and width for each dimension */ /*************************************************************/ for (objNum=0; objNum<n; objNum++) { for(dimNum=0; dimNum<numGeomDims; dimNum++) { if (geomArray[objNum * numGeomDims + dimNum] < minValInDim[dimNum]) { minValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum]; } if (geomArray[objNum * numGeomDims + dimNum] > maxValInDim[dimNum]) { maxValInDim[dimNum] = geomArray[objNum * numGeomDims + dimNum]; } } } for(dimNum=0; dimNum<numGeomDims; dimNum++) { widthDim[dimNum] = maxValInDim[dimNum] - minValInDim[dimNum]; } /*************************************************************/ /*************************************************************/ /* Rescale values to fit in bounding box */ /*************************************************************/ for (objNum=0; objNum<n; objNum++) { for(dimNum=0; dimNum<numGeomDims; dimNum++) { geomArray[objNum * numGeomDims + dimNum] -= minValInDim[dimNum]; geomArray[objNum * numGeomDims + dimNum] /= widthDim[dimNum]; } } /*************************************************************/ free(minValInDim); minValInDim=0; free(maxValInDim); maxValInDim=0; free(widthDim); widthDim=0; /******************************************************************/ /******************************************************************/ /* Specify which HSFC function to use (based on dim) */ /******************************************************************/ if (numGeomDims==1) { fhsfc = Zoltan_HSFC_InvHilbert1d; } else if (numGeomDims==2) { fhsfc = Zoltan_HSFC_InvHilbert2d; } else if (numGeomDims==3) { fhsfc = Zoltan_HSFC_InvHilbert3d; } else /* this error should have been previously caught */ { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Geometry should be of dimension 1, 2, or 3."); return(ZOLTAN_FATAL); } /******************************************************************/ /******************************************************************/ /* Generate hsfc keys and indices to be sorted */ /******************************************************************/ hsfcKey = (double *) malloc(n * sizeof (double)); coordIndx = (int *) malloc(n *sizeof(int)); for (objNum=0; objNum<n; objNum++) { hsfcKey[objNum] = fhsfc(zz, &(geomArray[objNum * numGeomDims]) ); coordIndx[objNum] = objNum; } /******************************************************************/ /******************************************************************/ /* Sort indices based on keys */ /******************************************************************/ Zoltan_quicksort_pointer_dec_double (coordIndx, hsfcKey, 0, n-1); /******************************************************************/ /******************************************************************/ /* get ranks */ /******************************************************************/ /******************************************************/ /* Determine offsets */ /******************************************************/ MPI_Scan(&n, &offset, 1, MPI_INT, MPI_SUM, zz->Communicator); offset -= n; /* MPI_Scan is inclusive, so subtract off local size */ /******************************************************/ for(objNum=0; objNum<n; objNum++) { /*MMW temporary hack to make Cedric's interface give me want I need */ /*rank[coordIndx[objNum]] = objNum + offset; */ rank[objNum] = coordIndx[objNum] + offset; } /******************************************************************/ /* iperm is to be deprecated so not calculated*/ free(hsfcKey); free(coordIndx); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); }
int Zoltan_Preprocess_Graph( ZZ *zz, /* Zoltan structure */ ZOLTAN_ID_PTR *global_ids, ZOLTAN_ID_PTR *local_ids, ZOLTAN_Third_Graph *gr, /* Graph for third part libs */ ZOLTAN_Third_Geom *geo, ZOLTAN_Third_Part *prt, ZOLTAN_Third_Vsize *vsp ) { static char *yo = "Zoltan_Preprocess_Graph"; int ierr; float *float_vwgt, *float_ewgts; char msg[256]; char add_obj_weight[MAX_PARAM_STRING_LEN+1]; ZOLTAN_TRACE_ENTER(zz, yo); /* Initialize all local pointers to NULL. This is necessary * because we free all non-NULL pointers upon errors. */ gr->vtxdist = gr->xadj = gr->adjncy = NULL; gr->vwgt = gr->ewgts = NULL; float_vwgt = float_ewgts = NULL; if (gr->obj_wgt_dim >= 0) { /* Check weight dimensions */ if (zz->Obj_Weight_Dim<0){ sprintf(msg, "Object weight dimension is %d, " "but should be >= 0. Using Obj_Weight_Dim = 0.", zz->Obj_Weight_Dim); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); gr->obj_wgt_dim = 0; } else { gr->obj_wgt_dim = zz->Obj_Weight_Dim; } } else gr->obj_wgt_dim = 0; if (gr->edge_wgt_dim >= 0) { if (zz->Edge_Weight_Dim<0){ sprintf(msg, "Edge weight dimension is %d, " "but should be >= 0. Using Edge_Weight_Dim = 0.", zz->Edge_Weight_Dim); ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); gr->edge_wgt_dim = 0; } else if (zz->Edge_Weight_Dim>1){ ZOLTAN_PRINT_WARN(zz->Proc, yo, "This method does not support " "multidimensional edge weights. Using Edge_Weight_Dim = 1."); gr->edge_wgt_dim = 1; } else { gr->edge_wgt_dim = zz->Edge_Weight_Dim; } } else gr->edge_wgt_dim = 0; if (gr->graph_type >= 0) /* Default graph type is GLOBAL. */ gr->graph_type = GLOBAL_GRAPH; else gr->graph_type = - gr->graph_type; /* Get parameter options shared by ParMetis and Jostle */ gr->check_graph = 1; /* default */ gr->scatter = 1; /* default */ gr->final_output = 0; strcpy(add_obj_weight, "NONE"); /* default */ Zoltan_Bind_Param(Graph_params, "CHECK_GRAPH", (void *) &gr->check_graph); Zoltan_Bind_Param(Graph_params, "SCATTER_GRAPH", (void *) &gr->scatter); Zoltan_Bind_Param(Graph_params, "FINAL_OUTPUT", (void *) &gr->final_output); Zoltan_Bind_Param(Graph_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight); Zoltan_Assign_Param_Vals(zz->Params, Graph_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* If reorder is true, we already have the id lists. Ignore weights. */ if ((*global_ids == NULL) || (!gr->id_known)){ int * input_part; ierr = Zoltan_Get_Obj_List(zz, &gr->num_obj, global_ids, local_ids, gr->obj_wgt_dim, &float_vwgt, &input_part); if (prt) { prt->input_part = input_part; } else { /* Ordering, dont need part */ ZOLTAN_FREE(&input_part); } if (ierr){ /* Return error */ ZOLTAN_PARMETIS_ERROR(ierr, "Get_Obj_List returned error."); } } /* Build Graph for third party library data structures, or just get vtxdist. */ ierr = Zoltan_Build_Graph(zz, gr->graph_type, gr->check_graph, gr->num_obj, *global_ids, *local_ids, gr->obj_wgt_dim, gr->edge_wgt_dim, &gr->vtxdist, &gr->xadj, &gr->adjncy, &float_ewgts, &gr->adjproc); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){ ZOLTAN_PARMETIS_ERROR(ierr, "Zoltan_Build_Graph returned error."); } if (prt) { prt->part_sizes = prt->input_part_sizes; if (gr->num_obj >0) { prt->part = (indextype *)ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype)); if (!prt->part){ /* Not enough memory */ ZOLTAN_PARMETIS_ERROR(ZOLTAN_MEMERR, "Out of memory."); } memcpy (prt->part, prt->input_part, (gr->num_obj) * sizeof(indextype)); } else { prt->input_part = prt->part = NULL; } } /* Convert from float. */ /* Get vertex weights if needed */ if (gr->obj_wgt_dim){ ierr = Zoltan_Preprocess_Scale_Weights (gr, float_vwgt, &gr->vwgt, gr->num_obj, gr->obj_wgt_dim, 1, zz, "vertex", gr->vtxdist[zz->Proc]); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){ /* Return error code */ ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights."); } ZOLTAN_FREE(&float_vwgt); } if (strcasecmp(add_obj_weight, "NONE")){ if (Zoltan_Preprocess_Add_Weight(zz, gr, prt, add_obj_weight) != ZOLTAN_OK) ZOLTAN_PARMETIS_ERROR(ierr, "Error in adding vertex weights."); } /* Get edge weights if needed */ if (gr->get_data) gr->num_edges = gr->xadj[gr->num_obj]; else { gr->num_edges = 0; gr->edge_wgt_dim = 0; } if (gr->edge_wgt_dim){ ierr = Zoltan_Preprocess_Scale_Weights (gr, float_ewgts, &gr->ewgts, gr->num_edges, gr->edge_wgt_dim, 1, zz, "edge", 0); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){ /* Return error code */ ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights."); } if (!gr->final_output) ZOLTAN_FREE(&float_ewgts); else gr->float_ewgts = float_ewgts; } else ZOLTAN_FREE(&float_ewgts); if (geo){ ierr = Zoltan_Preprocess_Extract_Geom (zz, global_ids, local_ids, gr, geo); if (ierr) { ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL, "Error returned from Zoltan_Preprocess_Extract_Geom"); } } if (vsp) { ierr = Zoltan_Preprocess_Extract_Vsize (zz, global_ids, local_ids, gr, vsp); if (ierr) { ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL, "Error returned from Zoltan_Preprocess_Extract_Vsize"); } } /* Scatter graph? * If data distribution is highly imbalanced, it is better to * redistribute the graph data structure before calling ParMetis. * After partitioning, the results must be mapped back. */ if (gr->scatter < gr->scatter_min) gr->scatter = gr->scatter_min; if (gr->scatter>0) { ierr = Zoltan_Preprocess_Scatter_Graph (zz, gr, prt, geo, vsp); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL, "Error returned from Zoltan_Preprocess_Scatter_Graph"); } } /* Verify that graph is correct */ if (gr->get_data){ int flag; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) flag = 2; /* Maximum output level */ else flag = 1; /* Medium output level */ ierr = Zoltan_Verify_Graph(zz->Communicator, gr->vtxdist, gr->xadj, gr->adjncy, gr->vwgt, gr->ewgts, gr->obj_wgt_dim, gr->edge_wgt_dim, gr->graph_type, gr->check_graph, flag); } End: ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Migrate( ZZ *zz, /* Zoltan structure. */ int num_import, /* Number of non-local objects assigned to the processor in the new decomposition. */ ZOLTAN_ID_PTR import_global_ids, /* Array of global IDs for non-local objects assigned to this processor in the new decomposition; this field can be NULL if the application doesn't provide import IDs.*/ ZOLTAN_ID_PTR import_local_ids, /* Array of local IDs for non-local objects assigned to the processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_procs, /* Array of processor IDs of processors owning the non-local objects that are assigned to this processor in the new decomposition; this field can be NULL if the application does not provide import IDs. */ int *import_to_part, /* Array of partition numbers to which imported objects should be assigned. */ int num_export, /* Number of objs to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_global_ids, /* Array of global IDs of objects to be exported to other processors to establish the new decomposition. */ ZOLTAN_ID_PTR export_local_ids, /* Array of local IDs of objects to be exported to other processors to establish the new decomposition. */ int *export_procs, /* Array of processor IDs to which objects will be exported to establish the new decomposition. */ int *export_to_part /* Array of partition numbers to which exported objects should be assigned. */ ) { /* * Routine to help perform migration. If migration pre-processing routine * (ZOLTAN_PRE_MIGRATE_FN) is specified, this routine first calls that fn. * It then calls a function to obtain the size of the migrating objects * (ZOLTAN_OBJ_SIZE_FN). The routine next calls an application-specified * object packing routine (ZOLTAN_PACK_OBJ_FN) for each object * to be exported. It develops the needed communication map to move the * objects to other processors. It performs the communication according * to the map, and then calls an application-specified object unpacking * routine (ZOLTAN_UNPACK_OBJ_FN) for each object imported. */ char *yo = "Zoltan_Migrate"; int num_gid_entries, num_lid_entries; /* lengths of global & local ids */ int *sizes = NULL; /* sizes (in bytes) of the object data for export. */ int id_size; /* size (in bytes) of ZOLTAN_GID + padding for alignment */ int tag_size; /* size (in bytes) of ZOLTAN_GID + one int (for message size) */ char *export_buf = NULL; /* buffer for packing export data. */ char *import_buf = NULL; /* buffer for receiving imported data. */ char *tmp; /* temporary pointer into buffers. */ int i; /* loop counter. */ int tmp_size; /* size of a single object's data. */ int *idx = NULL; /* index used for multi-fn packs and unpacks. */ int idx_cnt = 0; /* index counter for idx array. */ ZOLTAN_ID_PTR tmp_id = NULL; /* pointer to storage for a global ID in comm buf */ ZOLTAN_ID_PTR lid; /* temporary pointer to a local ID; used to pass NULL to query functions when NUM_LID_ENTRIES=0. */ ZOLTAN_COMM_OBJ *imp_plan = NULL; /* Comm obj built from import lists. */ ZOLTAN_COMM_OBJ *exp_plan = NULL; /* Comm obj built from export lists. */ int msgtag, msgtag2; /* Tags for communication routines */ int total_send_size; /* Total size of outcoming message (in #items) */ int total_recv_size; /* Total size of incoming message (in #items) */ int aligned_int; /* size of an int padded for alignment */ int dest; /* temporary destination partition. */ int include_parts = 0; /* flag indicating whether partition info is provided */ int ierr = ZOLTAN_OK; int actual_num_exp = 0; int actual_exp_allocated = 0; ZOLTAN_ID_PTR actual_exp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_exp_lids = NULL; /* actually be packed. Objs that */ int *actual_exp_procs = NULL; /* are changing partition but not */ int *actual_exp_to_part = NULL; /* processor may not be included. */ int actual_num_imp = 0; int actual_imp_allocated = 0; ZOLTAN_ID_PTR actual_imp_gids = NULL; /* Arrays containing only objs to */ ZOLTAN_ID_PTR actual_imp_lids = NULL; /* actually be imported. Objs that */ int *actual_imp_procs = NULL; /* are changing partition but not */ int *actual_imp_to_part = NULL; /* processor may not be included. */ ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { goto End; } /* * Check that all procs use the same id types. */ ierr = check_input(zz, ((num_export >= 0 && export_to_part) || (num_import >= 0 && import_to_part)), &include_parts); if (ierr != ZOLTAN_OK) goto End; num_gid_entries = zz->Num_GID; num_lid_entries = zz->Num_LID; /* * Check that all necessary query functions are available. */ if (zz->Get_Obj_Size == NULL && zz->Get_Obj_Size_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_OBJ_SIZE_FN or ZOLTAN_OBJ_SIZE_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Pack_Obj == NULL && zz->Pack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_PACK_OBJ_FN or ZOLTAN_PACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Unpack_Obj == NULL && zz->Unpack_Obj_Multi == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register a " "ZOLTAN_UNPACK_OBJ_FN or ZOLTAN_UNPACK_OBJ_MULTI_FN function " "to use the migration-help tools."); ierr = ZOLTAN_FATAL; goto End; } if (num_export >= 0) { /* Build the actual export arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &actual_num_exp, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part, &actual_exp_allocated); if (ierr < 0) goto End; /* Compute communication map based on actual exports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&exp_plan, actual_num_exp, actual_exp_procs, zz->Communicator, msgtag, &actual_num_imp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } } else if (num_import >= 0) { /* Build the actual import arrays */ ierr = actual_arrays(zz, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, &actual_num_imp, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part, &actual_imp_allocated); if (ierr < 0) goto End; /* Compute communication map based on imports. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&imp_plan, actual_num_imp, actual_imp_procs, zz->Communicator, msgtag, &actual_num_exp); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Error returned from Zoltan_Comm_Create."); goto End; } /* Compute actual export lists for packing objects */ if (actual_num_exp > 0) { actual_exp_allocated = 1; actual_exp_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_exp); actual_exp_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, actual_num_exp); actual_exp_procs = (int *) ZOLTAN_MALLOC(sizeof(int) * actual_num_exp); if (include_parts) actual_exp_to_part = (int *) ZOLTAN_MALLOC(sizeof(int)*actual_num_exp); if (actual_exp_gids == NULL || (num_lid_entries && actual_exp_lids == NULL) || actual_exp_procs == NULL || (import_to_part != NULL && actual_exp_to_part == NULL)) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); ierr = ZOLTAN_MEMERR; goto End; } } msgtag2 = 32766; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_gids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) actual_exp_gids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_lids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) actual_exp_lids); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } Zoltan_Comm_Info(imp_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, actual_exp_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(imp_plan, msgtag2, (char *) actual_imp_to_part, (int) sizeof(int), (char *) actual_exp_to_part); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } } /* Create inverse plan (i.e., plan based on exports) so can set * variable sizes. * (Zoltan_Comm_Do_Reverse(imp_plan, ...) allows sending variable * but does not tell how large to allocate receive buffer. */ ierr = Zoltan_Comm_Invert_Plan(&imp_plan); exp_plan = imp_plan; imp_plan = NULL; } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Import or export lists needed."); ierr = ZOLTAN_FATAL; goto End; } if (zz->Migrate.Pre_Migrate_PP != NULL) { zz->Migrate.Pre_Migrate_PP(zz->Migrate.Pre_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Pre_Migrate != NULL) { zz->Migrate.Pre_Migrate(zz->Migrate.Pre_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PRE_MIGRATE_FN function."); goto End; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done pre-migration processing"); id_size = Zoltan_Align(num_gid_entries * sizeof(ZOLTAN_ID_TYPE)); /* Note that alignment is not strictly necessary when ZOLTAN_ID_TYPE is int or unsigned int. */ aligned_int = Zoltan_Align(sizeof(int)); tag_size = id_size + aligned_int; /* * For each object, allow space for its global ID and its data plus * one int (for the object data size). * Zoltan will pack the global IDs; the application must pack the data * through the pack routine. Zoltan needs the global IDs for unpacking, * as the order of the data received during communication is not * necessarily the same order as import_global_ids[]. * Zoltan also needs to communicate the sizes of the objects because * only the sender knows the size of each object. */ if (actual_num_exp > 0) { sizes = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!sizes) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_Size_Multi != NULL) { zz->Get_Obj_Size_Multi(zz->Get_Obj_Size_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, sizes, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE_MULTI function."); goto End; } } else { for (i = 0; i < actual_num_exp; i++){ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); sizes[i] = zz->Get_Obj_Size(zz->Get_Obj_Size_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_OBJ_SIZE function."); goto End; } } } total_send_size = 0; for (i = 0; i < actual_num_exp; i++) { sizes[i] = Zoltan_Align(sizes[i]); total_send_size += sizes[i] + tag_size; } export_buf = (char *) ZOLTAN_CALLOC(total_send_size, sizeof(char)); if (!export_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Pack_Obj_Multi != NULL) { /* Allocate an index array for ZOLTAN_PACK_OBJ_MULTI_FN. */ idx = (int *) ZOLTAN_MALLOC(actual_num_exp * sizeof(int)); if (!idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Pack the objects for export. */ idx_cnt = 0; tmp = export_buf; for (i = 0; i < actual_num_exp; i++) { /* Pack the object's global ID */ tmp_id = (ZOLTAN_ID_PTR) tmp; ZOLTAN_SET_GID(zz, tmp_id, &(actual_exp_gids[i*num_gid_entries])); tmp += id_size; /* Pack the object's size */ *((int *)tmp) = sizes[i]; tmp += aligned_int; /* If using ZOLTAN_PACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Pack_Obj_Multi != NULL) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){ printf("[%1d] DEBUG in %s: Packing objects with multi-pack\n", zz->Proc, yo); } zz->Pack_Obj_Multi(zz->Pack_Obj_Multi_Data, num_gid_entries, num_lid_entries, actual_num_exp, actual_exp_gids, actual_exp_lids, (actual_exp_to_part!=NULL ? actual_exp_to_part : actual_exp_procs), sizes, idx, export_buf, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ_MULTI function."); goto End; } } else { tmp = export_buf + tag_size; for (i = 0; i < actual_num_exp; i++) { if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){ printf("[%1d] DEBUG in %s: Packing object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, &(actual_exp_gids[i*num_gid_entries])); printf("size = %d bytes\n", sizes[i]); } /* Pack the object's data */ lid = (num_lid_entries ? &(actual_exp_lids[i*num_lid_entries]) : NULL); dest = (actual_exp_to_part != NULL ? actual_exp_to_part[i] : actual_exp_procs[i]); zz->Pack_Obj(zz->Pack_Obj_Data, num_gid_entries, num_lid_entries, &(actual_exp_gids[i*num_gid_entries]), lid, dest, sizes[i], tmp, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_PACK_OBJ function."); goto End; } tmp += sizes[i] + tag_size; } } ZOLTAN_FREE(&idx); tmp_id = NULL; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done packing objects"); /* Modify sizes[] to contain message sizes, not object sizes */ for (i=0; i<actual_num_exp; i++) { sizes[i] += tag_size; } msgtag--; ierr = Zoltan_Comm_Resize(exp_plan, sizes, msgtag, &total_recv_size); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Resize."); goto End; } if (actual_num_imp > 0) { import_buf = (char *) ZOLTAN_MALLOC(total_recv_size); if (!import_buf) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* * Send the export data using the communication plan. */ msgtag2 = 32765; ierr = Zoltan_Comm_Do(exp_plan, msgtag2, export_buf, 1, import_buf); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); goto End; } /* * Free whatever memory we can. */ Zoltan_Comm_Destroy(&exp_plan); ZOLTAN_FREE(&export_buf); ZOLTAN_FREE(&sizes); ZOLTAN_TRACE_DETAIL(zz, yo, "Done communication"); /* * Perform application-specified processing before unpacking the data. */ if (zz->Migrate.Mid_Migrate_PP != NULL) { zz->Migrate.Mid_Migrate_PP(zz->Migrate.Mid_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Mid_Migrate != NULL) { zz->Migrate.Mid_Migrate(zz->Migrate.Mid_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_MID_MIGRATE_FN function."); goto End; } } /* * Unpack the object data. */ if (actual_num_imp > 0) { if (zz->Unpack_Obj_Multi != NULL) { /* Allocate and fill input arrays for Unpack_Obj_Multi. */ sizes = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); tmp_id = (ZOLTAN_ID_PTR) ZOLTAN_MALLOC_GID_ARRAY(zz, actual_num_imp); idx = (int *) ZOLTAN_MALLOC(actual_num_imp * sizeof(int)); if (!sizes || !tmp_id || !idx) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } tmp = import_buf; idx_cnt = 0; for (i = 0; i < actual_num_imp; i++) { /* Unpack the object's global ID */ ZOLTAN_SET_GID(zz, &(tmp_id[i*num_gid_entries]), (ZOLTAN_ID_PTR) tmp); tmp += id_size; /* Unpack the object's size */ sizes[i] = *((int *)tmp); tmp += aligned_int; /* If using ZOLTAN_UNPACK_OBJ_MULTI_FN, build the index array. */ idx_cnt += tag_size; if (idx != NULL) { idx[i] = idx_cnt; } tmp += sizes[i]; idx_cnt += sizes[i]; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){ printf("[%1d] DEBUG in %s: Unpacking objects with multi-fn\n", zz->Proc,yo); } zz->Unpack_Obj_Multi(zz->Unpack_Obj_Multi_Data, num_gid_entries, actual_num_imp, tmp_id, sizes, idx, import_buf, &ierr); ZOLTAN_FREE(&import_buf); ZOLTAN_FREE(&sizes); ZOLTAN_FREE(&tmp_id); ZOLTAN_FREE(&idx); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_MULTI_FN."); goto End; } } else { tmp = import_buf; for (i = 0; i < actual_num_imp; i++) { tmp_size = *((int *)(tmp + id_size)); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){ printf("[%1d] DEBUG in %s: Unpacking object with gid ", zz->Proc, yo); ZOLTAN_PRINT_GID(zz, (ZOLTAN_ID_PTR)tmp); printf("size = %d bytes\n", tmp_size); } /* Unpack the object's data */ zz->Unpack_Obj(zz->Unpack_Obj_Data, num_gid_entries, (ZOLTAN_ID_PTR) tmp, tmp_size, tmp + tag_size, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_UNPACK_OBJ_FN."); goto End; } tmp += (tmp_size + tag_size); } ZOLTAN_FREE(&import_buf); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done unpacking objects"); if (zz->Migrate.Post_Migrate_PP != NULL) { zz->Migrate.Post_Migrate_PP(zz->Migrate.Post_Migrate_PP_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, import_to_part, num_export, export_global_ids, export_local_ids, export_procs, export_to_part, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_PP_FN function."); goto End; } } if (zz->Migrate.Post_Migrate != NULL) { zz->Migrate.Post_Migrate(zz->Migrate.Post_Migrate_Data, num_gid_entries, num_lid_entries, num_import, import_global_ids, import_local_ids, import_procs, num_export, export_global_ids, export_local_ids, export_procs, &ierr); if (ierr < 0) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from " "ZOLTAN_POST_MIGRATE_FN function."); goto End; } } End: if (actual_exp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_exp_gids, &actual_exp_lids, &actual_exp_procs, &actual_exp_to_part); } if (actual_imp_allocated) { Zoltan_Multifree(__FILE__, __LINE__, 4, &actual_imp_gids, &actual_imp_lids, &actual_imp_procs, &actual_imp_to_part); } if (ierr < 0) { if (exp_plan) Zoltan_Comm_Destroy(&exp_plan); Zoltan_Multifree(__FILE__, __LINE__, 5, &import_buf, &tmp_id, &sizes, &idx, &export_buf); } ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_LB_Balance( ZZ *zz, int *changes, int *num_gid_entries, int *num_lid_entries, int *num_import_objs, ZOLTAN_ID_PTR *import_global_ids, ZOLTAN_ID_PTR *import_local_ids, int **import_procs, int *num_export_objs, ZOLTAN_ID_PTR *export_global_ids, ZOLTAN_ID_PTR *export_local_ids, int **export_procs ) { /* * Wrapper around Zoltan_LB for backward compatibility with * previous Zoltan versions. * Appropriate only when (# requested partitions == # processors), uniformly * distributed. * Arguments correspond directly with arguments of Zoltan_LB. */ char *yo = "Zoltan_LB_Balance"; int ierr = ZOLTAN_OK; /* Error code */ int *import_to_part = NULL; /* Array used as dummy arg in partitioning. */ int *export_to_part = NULL; /* Array used as dummy arg in partitioning. */ ZOLTAN_TRACE_ENTER(zz, yo); /* Determine whether partition parameters were set. Report error if * values are unreasonable. */ if ((zz->LB.Num_Global_Parts_Param != -1 && zz->LB.Num_Global_Parts_Param != zz->Num_Proc) || (zz->LB.Num_Local_Parts_Param != -1 && zz->LB.Num_Local_Parts_Param != 1)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Non-uniform distribution of partitions over processors is specified; " "use Zoltan_LB_Partition."); ierr = ZOLTAN_FATAL; goto End; } ierr = Zoltan_LB(zz, 0, changes, num_gid_entries, num_lid_entries, num_import_objs, import_global_ids, import_local_ids, import_procs, &import_to_part, num_export_objs, export_global_ids, export_local_ids, export_procs, &export_to_part); End: /* Not returning import/export partition information; free it if allocated. */ if (import_to_part != NULL) Zoltan_Special_Free(zz, (void **) &import_to_part, ZOLTAN_SPECIAL_MALLOC_INT); if (export_to_part != NULL) Zoltan_Special_Free(zz, (void **) &export_to_part, ZOLTAN_SPECIAL_MALLOC_INT); ZOLTAN_TRACE_EXIT(zz, yo); return(ierr); }
int Zoltan_Random( ZZ *zz, /* The Zoltan structure. */ float *part_sizes, /* Input: Array of size zz->LB.Num_Global_Parts * zz->Obj_Weight_Dim containing the percentage of work to be assigned to each partition. */ int *num_import, /* Return -1. Random uses only export lists. */ ZOLTAN_ID_PTR *import_global_ids, /* Not used. */ ZOLTAN_ID_PTR *import_local_ids, /* Not used. */ int **import_procs, /* Not used. */ int **import_to_part, /* Not used. */ int *num_export, /* Output: Number of objects to export. */ ZOLTAN_ID_PTR *export_global_ids, /* Output: GIDs to export. */ ZOLTAN_ID_PTR *export_local_ids, /* Output: LIDs to export. */ int **export_procs, /* Output: Processsors to export to. */ int **export_to_part /* Output: Partitions to export to. */ ) { int ierr = ZOLTAN_OK; int i, count, num_obj; int max_export; double rand_frac = 1.0; /* Default is to move all objects. */ ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int *parts = NULL; float *dummy = NULL; static char *yo = "Zoltan_Random"; static int first_time = 1; ZOLTAN_TRACE_ENTER(zz, yo); /* Synchronize the random number generator. * This synchronization is needed only for sanity in our nightly testing. * If some other operation (eg., Zoltan_LB_Eval) changes the status of * the random number generator, the answers here will change. They won't * be wrong, but they will be different from our accepted answers. */ if (first_time) { Zoltan_Srand(zz->Seed, NULL); Zoltan_Rand(NULL); first_time=0; } /* No import lists computed. */ *num_import = -1; /* Get parameter values. */ Zoltan_Bind_Param(Random_params, "RANDOM_MOVE_FRACTION", (void *) &rand_frac); Zoltan_Assign_Param_Vals(zz->Params, Random_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); /* Get list of local objects. */ ierr = Zoltan_Get_Obj_List(zz, &num_obj, &global_ids, &local_ids, 0, &dummy, &parts); /* Bound number of objects to export. */ max_export = 1.5*rand_frac*num_obj; /* Allocate export lists. */ *export_global_ids = *export_local_ids = NULL; *export_procs = *export_to_part = NULL; if (max_export > 0) { if (!Zoltan_Special_Malloc(zz, (void **)export_global_ids, max_export, ZOLTAN_SPECIAL_MALLOC_GID) || !Zoltan_Special_Malloc(zz, (void **)export_local_ids, max_export, ZOLTAN_SPECIAL_MALLOC_LID) || !Zoltan_Special_Malloc(zz, (void **)export_procs, max_export, ZOLTAN_SPECIAL_MALLOC_INT) || !Zoltan_Special_Malloc(zz, (void **)export_to_part, max_export, ZOLTAN_SPECIAL_MALLOC_INT)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error."); ierr = ZOLTAN_MEMERR; goto End; } } /* Randomly assign ids to procs. */ count=0; for (i=0; i<num_obj; i++){ /* Randomly select some objects to move (export) */ if ((count<max_export) && (Zoltan_Rand(NULL)<rand_frac*ZOLTAN_RAND_MAX)){ /* export_global_ids[count] = global_ids[i]; */ ZOLTAN_SET_GID(zz, &((*export_global_ids)[count*zz->Num_GID]), &global_ids[i*zz->Num_GID]); if (local_ids) /* export_local_ids[count] = local_ids[i]; */ ZOLTAN_SET_LID(zz, &((*export_local_ids)[count*zz->Num_LID]), &local_ids[i*zz->Num_LID]); /* Randomly pick new partition number. */ (*export_to_part)[count] = Zoltan_Rand_InRange(NULL, zz->LB.Num_Global_Parts); /* Processor number is derived from partition number. */ (*export_procs)[count] = Zoltan_LB_Part_To_Proc(zz, (*export_to_part)[count], &global_ids[i*zz->Num_GID]); /* printf("Debug: Export gid %u to part %d and proc %d.\n", (*export_global_ids)[count], (*export_to_part)[count], (*export_procs)[count]); */ ++count; } } (*num_export) = count; End: /* Free local memory, but not export lists. */ ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_FREE(&parts); ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
/* Main partitioning function for hypergraph partitioning. */ int Zoltan_PHG_Partition ( ZZ *zz, /* Zoltan data structure */ HGraph *hg, /* Input hypergraph to be partitioned */ int p, /* Input: number partitions to be generated */ float *part_sizes, /* Input: array of length p containing percentages of work to be assigned to each partition */ Partition parts, /* Input: initial partition #s; aligned with vtx arrays. Output: computed partition #s */ PHGPartParams *hgp) /* Input: parameters for hgraph partitioning. */ { PHGComm *hgc = hg->comm; VCycle *vcycle=NULL, *del=NULL; int i, err = ZOLTAN_OK, middle; ZOLTAN_GNO_TYPE origVpincnt; /* for processor reduction test */ ZOLTAN_GNO_TYPE prevVcnt = 2*hg->dist_x[hgc->nProc_x]; /* initialized so that the */ ZOLTAN_GNO_TYPE prevVedgecnt = 2*hg->dist_y[hgc->nProc_y]; /* while loop will be entered before any coarsening */ ZOLTAN_GNO_TYPE tot_nPins, local_nPins; MPI_Datatype zoltan_gno_mpi_type; char *yo = "Zoltan_PHG_Partition"; int do_timing = (hgp->use_timers > 1); int fine_timing = (hgp->use_timers > 2); int vcycle_timing = (hgp->use_timers > 4 && hgp->ProRedL == 0); short refine = 0; struct phg_timer_indices *timer = Zoltan_PHG_LB_Data_timers(zz); int reset_geometric_matching = 0; char reset_geometric_string[4]; ZOLTAN_TRACE_ENTER(zz, yo); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); if (do_timing) { if (timer->vcycle < 0) timer->vcycle = Zoltan_Timer_Init(zz->ZTime, 0, "Vcycle"); if (timer->procred < 0) timer->procred = Zoltan_Timer_Init(zz->ZTime, 0, "Processor Reduction"); if (timer->match < 0) timer->match = Zoltan_Timer_Init(zz->ZTime, 1, "Matching"); if (timer->coarse < 0) timer->coarse = Zoltan_Timer_Init(zz->ZTime, 1, "Coarsening"); if (timer->coarsepart < 0) timer->coarsepart = Zoltan_Timer_Init(zz->ZTime, 1, "Coarse_Partition"); if (timer->refine < 0) timer->refine = Zoltan_Timer_Init(zz->ZTime, 1, "Refinement"); if (timer->project < 0) timer->project = Zoltan_Timer_Init(zz->ZTime, 1, "Project_Up"); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins; MPI_Allreduce(&local_nPins,&tot_nPins,1,zoltan_gno_mpi_type,MPI_SUM,hgc->Communicator); origVpincnt = tot_nPins; if (!(vcycle = newVCycle(zz, hg, parts, NULL, vcycle_timing))) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "VCycle is NULL."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } /* For geometric coarsening, hgp->matching pointer and string are reset * after geometric_levels of coarsening. Will need to reset them after * this vcycle is completed. Capture that fact now! */ if (!strcasecmp(hgp->redm_str, "rcb") || !strcasecmp(hgp->redm_str, "rib")) { reset_geometric_matching = 1; strcpy(reset_geometric_string, hgp->redm_str); } /****** Coarsening ******/ #define COARSEN_FRACTION_LIMIT 0.9 /* Stop if we don't make much progress */ while ((hg->redl>0) && (hg->dist_x[hgc->nProc_x] > (ZOLTAN_GNO_TYPE)hg->redl) && ((hg->dist_x[hgc->nProc_x] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVcnt + 0.5)) /* prevVcnt initialized to 2*hg->dist_x[hgc->nProc_x] */ || (hg->dist_y[hgc->nProc_y] < (ZOLTAN_GNO_TYPE) (COARSEN_FRACTION_LIMIT * prevVedgecnt + 0.5))) /* prevVedgecnt initialized to 2*hg->dist_y[hgc->nProc_y] */ && hg->dist_y[hgc->nProc_y] && hgp->matching) { ZOLTAN_GNO_TYPE *match = NULL; VCycle *coarser=NULL, *redistributed=NULL; prevVcnt = hg->dist_x[hgc->nProc_x]; prevVedgecnt = hg->dist_y[hgc->nProc_y]; #ifdef _DEBUG /* UVC: load balance stats */ Zoltan_PHG_LoadBalStat(zz, hg); #endif if (hgp->output_level >= PHG_DEBUG_LIST) { uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p); if (hgp->output_level > PHG_DEBUG_LIST) { err = Zoltan_HG_Info(zz, hg); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; } } if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL, "coarsening plot"); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->match, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_match < 0) { char str[80]; sprintf(str, "VC Matching %d", hg->info); vcycle->timer_match = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_match, hgc->Communicator); } /* Allocate and initialize Matching Array */ if (hg->nVtx && !(match = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC (hg->nVtx*sizeof(ZOLTAN_GNO_TYPE)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: Matching array"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; i++) match[i] = i; /* Calculate matching (packing or grouping) */ err = Zoltan_PHG_Matching (zz, hg, match, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) { ZOLTAN_FREE (&match); goto End; } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_match, hgc->Communicator); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->match, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->coarse, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_coarse < 0) { char str[80]; sprintf(str, "VC Coarsening %d", hg->info); vcycle->timer_coarse = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_coarse, hgc->Communicator); } if (!(coarser = newVCycle(zz, NULL, NULL, vcycle, vcycle_timing))) { ZOLTAN_FREE (&match); ZOLTAN_PRINT_ERROR (zz->Proc, yo, "coarser is NULL."); goto End; } /* Construct coarse hypergraph and LevelMap */ err = Zoltan_PHG_Coarsening (zz, hg, match, coarser->hg, vcycle->LevelMap, &vcycle->LevelCnt, &vcycle->LevelSndCnt, &vcycle->LevelData, &vcycle->comm_plan, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_coarse, hgc->Communicator); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarse, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } ZOLTAN_FREE (&match); if ((err=allocVCycle(coarser))!= ZOLTAN_OK) goto End; vcycle = coarser; hg = vcycle->hg; if (hgc->nProc > 1 && hgp->ProRedL > 0) { local_nPins = (ZOLTAN_GNO_TYPE)hg->nPins; MPI_Allreduce(&local_nPins, &tot_nPins, 1, zoltan_gno_mpi_type, MPI_SUM, hgc->Communicator); if (tot_nPins < (ZOLTAN_GNO_TYPE)(hgp->ProRedL * origVpincnt + 0.5)) { if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->procred, hgc->Communicator); } /* redistribute to half the processors */ origVpincnt = tot_nPins; /* update for processor reduction test */ if(hg->nVtx&&!(hg->vmap=(int*)ZOLTAN_MALLOC(hg->nVtx*sizeof(int)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: hg->vmap"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; i++) hg->vmap[i] = i; middle = (int)((float) (hgc->nProc-1) * hgp->ProRedL); if (hgp->nProc_x_req!=1&&hgp->nProc_y_req!=1) { /* Want 2D decomp */ if ((middle+1) > SMALL_PRIME && Zoltan_PHG_isPrime(middle+1)) --middle; /* if it was prime just use one less #procs (since it should be bigger than SMALL_PRIME it is safe to decrement) */ } if (!(hgc = (PHGComm*) ZOLTAN_MALLOC (sizeof(PHGComm)))) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory: PHGComm"); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if (!(redistributed=newVCycle(zz,NULL,NULL,vcycle,vcycle_timing))) { ZOLTAN_FREE (&hgc); ZOLTAN_PRINT_ERROR (zz->Proc, yo, "redistributed is NULL."); goto End; } Zoltan_PHG_Redistribute(zz,hgp,hg,0,middle,hgc, redistributed->hg, &vcycle->vlno,&vcycle->vdest); if (hgp->UseFixedVtx || hgp->UsePrefPart) redistributed->hg->bisec_split = hg->bisec_split; if ((err=allocVCycle(redistributed))!= ZOLTAN_OK) goto End; vcycle = redistributed; if (hgc->myProc < 0) /* I'm not in the redistributed part so I should go to uncoarsening refinement and wait */ { if (fine_timing) { if (timer->cpgather < 0) timer->cpgather = Zoltan_Timer_Init(zz->ZTime, 1, "CP Gather"); if (timer->cprefine < 0) timer->cprefine =Zoltan_Timer_Init(zz->ZTime, 0, "CP Refine"); if (timer->cpart < 0) timer->cpart = Zoltan_Timer_Init(zz->ZTime, 0, "CP Part"); } if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } goto Refine; } hg = vcycle->hg; hg->redl = hgp->redl; /* not set with hg creation */ if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->procred, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } } } } if (hgp->output_level >= PHG_DEBUG_LIST) { uprintf(hgc, "START %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d...\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p); if (hgp->output_level > PHG_DEBUG_LIST) { err = Zoltan_HG_Info(zz, hg); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; } } if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, NULL, "coarsening plot"); /* free array that may have been allocated in matching */ if (hgp->vtx_scal) { hgp->vtx_scal_size = 0; ZOLTAN_FREE(&(hgp->vtx_scal)); } if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->coarsepart, hgc->Communicator); } /****** Coarse Partitioning ******/ err = Zoltan_PHG_CoarsePartition (zz, hg, p, part_sizes, vcycle->Part, hgp); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) goto End; if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->coarsepart, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } Refine: del = vcycle; refine = 1; /****** Uncoarsening/Refinement ******/ while (vcycle) { VCycle *finer = vcycle->finer; hg = vcycle->hg; if (refine && hgc->myProc >= 0) { if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->refine, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_refine < 0) { char str[80]; sprintf(str, "VC Refinement %d", hg->info); vcycle->timer_refine = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_refine, hgc->Communicator); } err = Zoltan_PHG_Refinement (zz, hg, p, part_sizes, vcycle->Part, hgp); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->refine, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_refine, hgc->Communicator); if (hgp->output_level >= PHG_DEBUG_LIST) uprintf(hgc, "FINAL %3d |V|=%6d |E|=%6d #pins=%6d %d/%s/%s/%s p=%d bal=%.2f cutl=%.2f\n", hg->info, hg->nVtx, hg->nEdge, hg->nPins, hg->redl, hgp->redm_str, hgp->coarsepartition_str, hgp->refinement_str, p, Zoltan_PHG_Compute_Balance(zz, hg, part_sizes, 0, p, vcycle->Part), Zoltan_PHG_Compute_ConCut(hgc, hg, vcycle->Part, p, &err)); if (hgp->output_level >= PHG_DEBUG_PLOT) Zoltan_PHG_Plot(zz->Proc, hg->nVtx, p, hg->vindex, hg->vedge, vcycle->Part, "partitioned plot"); } if (finer) { int *rbuffer; /* Project coarse partition to fine partition */ if (finer->comm_plan) { refine = 1; if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->project, hgc->Communicator); } if (vcycle_timing) { if (vcycle->timer_project < 0) { char str[80]; sprintf(str, "VC Project Up %d", hg->info); vcycle->timer_project = Zoltan_Timer_Init(vcycle->timer, 0, str); } ZOLTAN_TIMER_START(vcycle->timer, vcycle->timer_project, hgc->Communicator); } /* easy to assign partitions to internal matches */ for (i = 0; i < finer->hg->nVtx; i++) if (finer->LevelMap[i] >= 0) /* if considers only the local vertices */ finer->Part[i] = vcycle->Part[finer->LevelMap[i]]; /* now that the course partition assignments have been propagated */ /* upward to the finer level for the local vertices, we need to */ /* fill the LevelData (matched pairs of a local vertex with a */ /* off processor vertex) with the partition assignment of the */ /* local vertex - can be done totally in the finer level! */ for (i = 0; i < finer->LevelCnt; i++) { ++i; /* skip over off processor lno */ finer->LevelData[i] = finer->Part[finer->LevelData[i]]; } /* allocate rec buffer to exchange LevelData information */ rbuffer = NULL; if (finer->LevelSndCnt > 0) { rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->LevelSndCnt * sizeof(int)); if (!rbuffer) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } /* get partition assignments from owners of externally matched vtxs */ Zoltan_Comm_Resize (finer->comm_plan, NULL, COMM_TAG, &i); Zoltan_Comm_Do_Reverse (finer->comm_plan, COMM_TAG+1, (char*) finer->LevelData, 2 * sizeof(int), NULL, (char*) rbuffer); /* process data to assign partitions to expernal matches */ for (i = 0; i < 2 * finer->LevelSndCnt;) { int lno, partition; lno = rbuffer[i++]; partition = rbuffer[i++]; finer->Part[lno] = partition; } ZOLTAN_FREE (&rbuffer); Zoltan_Comm_Destroy (&finer->comm_plan); if (do_timing) { ZOLTAN_TIMER_STOP(zz->ZTime, timer->project, hgc->Communicator); ZOLTAN_TIMER_START(zz->ZTime, timer->vcycle, hgc->Communicator); } if (vcycle_timing) ZOLTAN_TIMER_STOP(vcycle->timer, vcycle->timer_project, hgc->Communicator); } else { int *sendbuf = NULL, size; refine = 0; /* ints local and partition numbers */ if (finer->vlno) { sendbuf = (int*) ZOLTAN_MALLOC (2 * hg->nVtx * sizeof(int)); if (!sendbuf) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } for (i = 0; i < hg->nVtx; ++i) { sendbuf[2 * i] = finer->vlno[i]; /* assign local numbers */ sendbuf[2 * i + 1] = vcycle->Part[i];/* assign partition numbers */ } } ZOLTAN_FREE (&hgc); hgc = finer->hg->comm; /* updating hgc is required when the processors change */ /* Create comm plan to unredistributed processors */ err = Zoltan_Comm_Create(&finer->comm_plan, finer->vlno ? hg->nVtx : 0, finer->vdest, hgc->Communicator, COMM_TAG+2, &size); if (err != ZOLTAN_OK && err != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(hgc->myProc, yo, "Zoltan_Comm_Create failed."); goto End; } /* allocate rec buffer to exchange sendbuf information */ rbuffer = NULL; if (finer->hg->nVtx) { rbuffer = (int*) ZOLTAN_MALLOC (2 * finer->hg->nVtx * sizeof(int)); if (!rbuffer) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } /* Use plan to send partitions to the unredistributed processors */ Zoltan_Comm_Do(finer->comm_plan, COMM_TAG+3, (char *) sendbuf, 2*sizeof(int), (char *) rbuffer); MPI_Bcast(rbuffer, 2*finer->hg->nVtx, MPI_INT, 0, hgc->col_comm); /* process data to assign partitions to unredistributed processors */ for (i = 0; i < 2 * finer->hg->nVtx;) { int lno, partition; lno = rbuffer[i++]; partition = rbuffer[i++]; finer->Part[lno] = partition; } if (finer->vlno) ZOLTAN_FREE (&sendbuf); ZOLTAN_FREE (&rbuffer); Zoltan_Comm_Destroy (&finer->comm_plan); } } vcycle = finer; } /* while (vcycle) */ End: vcycle = del; while (vcycle) { if (vcycle_timing) { Zoltan_Timer_PrintAll(vcycle->timer, 0, hgc->Communicator, stdout); Zoltan_Timer_Destroy(&vcycle->timer); } if (vcycle->finer) { /* cleanup by level */ Zoltan_HG_HGraph_Free (vcycle->hg); if (vcycle->LevelData) Zoltan_Multifree (__FILE__, __LINE__, 4, &vcycle->Part, &vcycle->LevelMap, &vcycle->LevelData, &vcycle->hg); else if (vcycle->vlno) Zoltan_Multifree (__FILE__, __LINE__, 5, &vcycle->Part, &vcycle->vdest, &vcycle->vlno, &vcycle->LevelMap, &vcycle->hg); else Zoltan_Multifree (__FILE__, __LINE__, 3, &vcycle->Part, &vcycle->LevelMap, &vcycle->hg); } else /* cleanup top level */ Zoltan_Multifree (__FILE__, __LINE__, 2, &vcycle->LevelMap, &vcycle->LevelData); del = vcycle; vcycle = vcycle->finer; ZOLTAN_FREE(&del); } if (reset_geometric_matching) { strcpy(hgp->redm_str, reset_geometric_string); Zoltan_PHG_Set_Matching_Fn(hgp); } if (do_timing) ZOLTAN_TIMER_STOP(zz->ZTime, timer->vcycle, hgc->Communicator); ZOLTAN_TRACE_EXIT(zz, yo) ; return err; }
int Zoltan_Oct_migreg_migrate_orphans(ZZ *zz, pRegion RegionList, int nregions, int level, Map *array, int *c1, int *c2) { int i, j, k; /* index counters */ pRegion ptr; /* region in the mesh */ COORD origin; /* centroid coordinate information */ pRegion *regions = NULL; /* an array of regions */ int *npids = NULL; Region *regions2 = NULL; /* an array of regions */ int *npids2 = NULL; int nreg; /* number of regions */ COORD min, /* minimum bounds of an octant */ max; /* maximum bounds of an octant */ COORD cmin, /* minimum bounds of a child octant */ cmax; /* maximum bounds of a child octant */ COORD rmin, /* minimum bounds of a remote octant */ rmax; /* maximum bounds of a remote octant */ int new_num; int n; int dir = 0; pRList RootList; pOctant RootOct; OCT_Global_Info *OCT_info = (OCT_Global_Info *)(zz->LB.Data_Structure); char *yo = "Zoltan_Oct_migreg_migrate_orphans_static"; int ierr = ZOLTAN_OK; ZOLTAN_ID_PTR gids2, lids2; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; if(nregions > 0) { /* create the array of messages to be sent to other processors */ /* Array = (Message *) ZOLTAN_MALLOC(nregions * sizeof(Message)); */ if((regions = (pRegion *) ZOLTAN_MALLOC(nregions * sizeof(pRegion))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } if((npids = (int *) ZOLTAN_MALLOC(nregions * sizeof(int))) == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(®ions); return ZOLTAN_MEMERR; } } ptr = RegionList; n = nreg = 0; while((ptr != NULL) && (nregions > 0)) { if(ptr->attached == 1) { /* if region already attached to an octant, then skip to next region */ ptr = ptr->next; continue; } /* region not attached, have to find which processor to send to */ j=0; dir = 0; vector_set(min, OCT_info->OCT_gmin); vector_set(max, OCT_info->OCT_gmax); /* * for each level of refinement, find which child region belongs to. * translate which child to which entry in map array. */ for(i=0; i<level; i++) { Zoltan_Oct_bounds_to_origin(min, max, origin); if(OCT_info->OCT_dimension == 2) j = j * 4; else j = j * 8; k = Zoltan_Oct_child_which(OCT_info,origin, ptr->Coord); new_num = Zoltan_Oct_convert_idx_from_map(OCT_info, dir, k); dir = Zoltan_Oct_get_child_dir(OCT_info, dir, new_num); j += new_num; Zoltan_Oct_child_bounds(min, max, origin, k, cmin, cmax); vector_set(min, cmin); vector_set(max, cmax); } /* inform message which processor to send to */ npids[n] = array[j].npid; RootList = array[j].list; while((RootOct = RL_nextRootOctant(&RootList))) { Zoltan_Oct_bounds(RootOct,rmin,rmax); if (Zoltan_Oct_in_box_closure(OCT_info, ptr->Coord ,rmin, rmax)) { npids[n] = RootOct->npid; break; } } if((npids[n] != -1) && (npids[n] != zz->Proc)) { Zoltan_Oct_copy_info(zz, ptr, &(regions[n++])); } else { Zoltan_Oct_insert_orphan(zz, *ptr); } nreg++; /* increment region counter */ ptr = ptr->next; /* look at next region */ } /* * if regions looked at != number of regions in region list, * then there is an error */ if (nreg!=nregions) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "regions found != to expected number of regions"); return ZOLTAN_FATAL; } regions2 = (Region *) ZOLTAN_MALLOC(n * sizeof(Region)); gids2 = ZOLTAN_MALLOC_GID_ARRAY(zz, n); lids2 = ZOLTAN_MALLOC_LID_ARRAY(zz, n); npids2 = (int *) ZOLTAN_MALLOC(n * sizeof(int)); for(i=0; i<n; i++) { npids2[i] = npids[i]; vector_set(regions2[i].Coord, regions[i]->Coord); regions2[i].Weight = regions[i]->Weight; regions2[i].Global_ID = &(gids2[i*num_gid_entries]); regions2[i].Local_ID = (num_lid_entries ? &(lids2[i*num_lid_entries]) : NULL); ZOLTAN_SET_GID(zz, &(gids2[i*num_gid_entries]), regions[i]->Global_ID); ZOLTAN_SET_LID(zz, &(lids2[i*num_lid_entries]), regions[i]->Local_ID); regions2[i].Proc = regions[i]->Proc; regions2[i].attached = 0; } *c1 = n; /* migrate the orphan regions according to the message array */ Zoltan_Oct_migreg_migrate_regions(zz, regions2, gids2, lids2, npids2, n, c2); for (i=0; i < n; i++) { ZOLTAN_FREE(&(regions[i]->Global_ID)); ZOLTAN_FREE(&(regions[i]->Local_ID)); ZOLTAN_FREE(&(regions[i])); } ZOLTAN_FREE(®ions); ZOLTAN_FREE(&npids); ZOLTAN_FREE(®ions2); ZOLTAN_FREE(&gids2); ZOLTAN_FREE(&lids2); ZOLTAN_FREE(&npids2); return ierr; }
int Zoltan_Matrix_Build (ZZ* zz, Zoltan_matrix_options *opt, Zoltan_matrix* matrix, int request_GNOs, /* Input: Flag indicating calling code needs translation of extra GIDs to GNOs; partial 2D coloring needs this feature. */ int num_requested, /* Input: Local # of GIDs needing translation to GNOs. */ ZOLTAN_ID_PTR requested_GIDs, /* Input: Calling code requests the GNOs for these GIDs */ ZOLTAN_GNO_TYPE *requested_GNOs /* Output: Return GNOs of the requested GIDs. */ ) { static char *yo = "Zoltan_Matrix_Build"; int ierr = ZOLTAN_OK; int nX; ZOLTAN_GNO_TYPE tmp; ZOLTAN_GNO_TYPE *xGNO = NULL; ZOLTAN_ID_PTR xLID=NULL; ZOLTAN_ID_PTR xGID=NULL; ZOLTAN_ID_PTR yGID=NULL; ZOLTAN_ID_PTR pinID=NULL; float *xwgt = NULL; int * Input_Parts=NULL; struct Zoltan_DD_Struct *dd = NULL; int *proclist = NULL; int *xpid = NULL; int i; int gno_size_for_dd; MPI_Datatype zoltan_gno_mpi_type; int use_full_dd = (opt->speed == MATRIX_FULL_DD); int fast_build_base = opt->fast_build_base; matrix->opts.speed = opt->speed; matrix->opts.fast_build_base = opt->fast_build_base; ZOLTAN_TRACE_ENTER(zz, yo); if (num_requested && (!requested_GIDs || !requested_GNOs)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error in requested input; needed arrays are NULL.\n"); } /* ZOLTAN_GNO_TYPE is >= ZOLTAN_ID_TYPE */ gno_size_for_dd = sizeof(ZOLTAN_GNO_TYPE) / sizeof(ZOLTAN_ID_TYPE); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memset (matrix, 0, sizeof(Zoltan_matrix)); /* Set all fields to 0 */ memcpy (&matrix->opts, opt, sizeof(Zoltan_matrix_options)); /**************************************************/ /* Obtain vertex information from the application */ /**************************************************/ ierr = Zoltan_Get_Obj_List(zz, &nX, &xGID, &xLID, zz->Obj_Weight_Dim, &xwgt, &Input_Parts); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error getting object data"); goto End; } ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&xwgt); /*******************************************************************/ /* Assign vertex consecutive numbers (gnos) */ /*******************************************************************/ if (use_full_dd) { /* Zoltan computes a translation */ /* Have to use Data Directory if request_GNOs is true. */ if (nX) { xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (xGNO == NULL) MEMORY_ERROR; } ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, xGNO, nX, matrix->opts.randomize, &matrix->globalX); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error assigning global numbers to vertices"); goto End; } ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, gno_size_for_dd, 0, nX, 0); CHECK_IERR; /* Make our new numbering public */ Zoltan_DD_Update (dd, xGID, (ZOLTAN_ID_PTR) xGNO, NULL, NULL, nX); if (request_GNOs) { Zoltan_DD_Find(dd, requested_GIDs, (ZOLTAN_ID_PTR) requested_GNOs, NULL, NULL, num_requested, NULL); } } else { /* We don't want to use the DD */ /* * KDDKDD 2/10/11 This code cannot work when NUM_GID_ENTRIES>1. * KDDKDD 2/10/11 The assumption is that, if a user sets the * KDDKDD 2/10/11 appropriate parameter to enable this code, the user * KDDKDD 2/10/11 knows that his GIDs are compatible with integers. */ if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ xGNO = (ZOLTAN_GNO_TYPE*) ZOLTAN_MALLOC(nX*sizeof(ZOLTAN_GNO_TYPE)); if (nX && xGNO == NULL) MEMORY_ERROR; for (i=0; i < nX; i++) xGNO[i] = (ZOLTAN_GNO_TYPE)xGID[i] - fast_build_base; } else { xGNO = (ZOLTAN_GNO_TYPE *)xGID; if (fast_build_base) for (i = 0; i < nX; i++) xGNO[i] -= fast_build_base; } for (i = 0; i < num_requested; i++) requested_GNOs[i] = (ZOLTAN_GNO_TYPE)requested_GIDs[i] - fast_build_base; tmp = (ZOLTAN_GNO_TYPE)nX; MPI_Allreduce(&tmp, &matrix->globalX, 1, zoltan_gno_mpi_type, MPI_SUM, zz->Communicator); } /* I store : xGNO, xGID, xpid, */ ierr = Zoltan_DD_Create (&matrix->ddX, zz->Communicator, gno_size_for_dd, zz->Num_GID, sizeof(int), matrix->globalX/zz->Num_Proc, 0); CHECK_IERR; /* Hope a linear assignment will help a little */ if (matrix->globalX/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddX, matrix->globalX/zz->Num_Proc); /* Associate all the data with our xGNO */ xpid = (int*)ZOLTAN_MALLOC(nX*sizeof(int)); if (nX >0 && xpid == NULL) MEMORY_ERROR; for (i = 0 ; i < nX ; ++i) xpid[i] = zz->Proc; Zoltan_DD_Update (matrix->ddX, (ZOLTAN_ID_PTR)xGNO, xGID, (char *)xpid, NULL, nX); ZOLTAN_FREE(&xpid); if (matrix->opts.pinwgt) matrix->pinwgtdim = zz->Edge_Weight_Dim; else matrix->pinwgtdim = 0; ierr = matrix_get_edges(zz, matrix, &yGID, &pinID, nX, &xGID, &xLID, &xGNO, &xwgt, use_full_dd); CHECK_IERR; matrix->nY_ori = matrix->nY; if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)){ goto End; } if (matrix->opts.enforceSquare && matrix->redist) { /* Convert yGID to yGNO using the same translation as x */ /* Needed for graph : rowID = colID */ /* y and x may have different distributions */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nY * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, yGID, (ZOLTAN_ID_PTR)(matrix->yGNO), NULL, NULL, matrix->nY, NULL); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Hyperedge GIDs don't match.\n"); ierr = ZOLTAN_FATAL; ZOLTAN_FREE(&pinID); goto End; } } if (matrix->opts.local) { /* keep only local edges */ proclist = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); if (matrix->nPins && proclist == NULL) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } } else proclist = NULL; /* Convert pinID to pinGNO using the same translation as x */ if (use_full_dd) { matrix->pinGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_MALLOC(matrix->nPins* sizeof(ZOLTAN_GNO_TYPE)); if ((matrix->nPins > 0) && (matrix->pinGNO == NULL)) { ZOLTAN_FREE(&pinID); MEMORY_ERROR; } ierr = Zoltan_DD_Find (dd, pinID, (ZOLTAN_ID_PTR)(matrix->pinGNO), NULL, NULL, matrix->nPins, proclist); if (ierr != ZOLTAN_OK) { ZOLTAN_PRINT_ERROR(zz->Proc,yo,"Undefined GID found.\n"); ierr = ZOLTAN_FATAL; goto End; } ZOLTAN_FREE(&pinID); Zoltan_DD_Destroy(&dd); dd = NULL; } else { if (sizeof(ZOLTAN_GNO_TYPE) != sizeof(ZOLTAN_ID_TYPE)){ matrix->pinGNO = (ZOLTAN_GNO_TYPE *)ZOLTAN_MALLOC(matrix->nPins * sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nPins && !matrix->pinGNO){ ZOLTAN_FREE(&pinID); MEMORY_ERROR; } for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] = (ZOLTAN_GNO_TYPE)pinID[i] - fast_build_base; ZOLTAN_FREE(&pinID); } else{ matrix->pinGNO = (ZOLTAN_GNO_TYPE *) pinID; if (fast_build_base) for (i=0; i < matrix->nPins; i++) matrix->pinGNO[i] -= fast_build_base; pinID = NULL; } } /* if (matrix->opts.local) { /\* keep only local edges *\/ */ /* int *nnz_list; /\* nnz offset to delete *\/ */ /* int nnz; /\* number of nnz to delete *\/ */ /* int i; */ /* nnz_list = (int*) ZOLTAN_MALLOC(matrix->nPins*sizeof(int)); */ /* if (matrix->nPins && nnz_list == NULL) MEMORY_ERROR; */ /* for (i = 0, nnz=0 ; i < matrix->nPins ; ++i) { */ /* if (proclist[i] == zz->Proc) continue; */ /* nnz_list[nnz++] = i; */ /* } */ /* ZOLTAN_FREE(&proclist); */ /* Zoltan_Matrix_Delete_nnz(zz, matrix, nnz, nnz_list); */ /* } */ if (!matrix->opts.enforceSquare) { /* Hyperedges name translation is different from the one of vertices */ matrix->yGNO = (ZOLTAN_GNO_TYPE*)ZOLTAN_CALLOC(matrix->nY, sizeof(ZOLTAN_GNO_TYPE)); if (matrix->nY && matrix->yGNO == NULL) MEMORY_ERROR; /* int nGlobalEdges = 0; */ ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, matrix->yGNO, matrix->nY, matrix->opts.randomize, &matrix->globalY); CHECK_IERR; /* /\**************************************************************************************** */ /* * If it is desired to remove dense edges, divide the list of edges into */ /* * two lists. The ZHG structure will contain the removed edges (if final_output is true), */ /* * and the kept edges will be returned. */ /* ****************************************************************************************\/ */ /* totalNumEdges = zhg->globalHedges; */ /* ierr = remove_dense_edges_matrix(zz, zhg, edgeSizeThreshold, final_output, */ /* &nLocalEdges, &nGlobalEdges, &nPins, */ /* &edgeGNO, &edgeSize, &edgeWeight, &pinGNO, &pinProcs); */ /* if (nGlobalEdges < totalNumEdges){ */ /* /\* re-assign edge global numbers if any edges were removed *\/ */ /* ierr = Zoltan_PHG_GIDs_to_global_numbers(zz, edgeGNO, nLocalEdges, */ /* randomizeInitDist, &totalNumEdges); */ /* if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error reassigning global numbers to edges"); */ /* goto End; */ /* } */ /* } */ /* We have to define ddY : yGNO, yGID, ywgt */ ierr = Zoltan_DD_Create (&matrix->ddY, zz->Communicator, gno_size_for_dd, zz->Num_GID, 0, matrix->globalY/zz->Num_Proc, 0); /* Hope a linear assignment will help a little */ if (matrix->globalY/zz->Num_Proc) Zoltan_DD_Set_Neighbor_Hash_Fn1(matrix->ddY, matrix->globalY/zz->Num_Proc); /* Associate all the data with our yGNO */ Zoltan_DD_Update (matrix->ddY, (ZOLTAN_ID_PTR)matrix->yGNO, yGID, NULL, NULL, matrix->nY); } End: ZOLTAN_FREE(&xpid); ZOLTAN_FREE(&xLID); ZOLTAN_FREE(&xGNO); ZOLTAN_FREE(&xGID); ZOLTAN_FREE(&xwgt); ZOLTAN_FREE(&Input_Parts); ZOLTAN_FREE(&proclist); if (dd != NULL) Zoltan_DD_Destroy(&dd); /* Already stored in the DD */ ZOLTAN_FREE(&yGID); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
static int Zoltan_Oct_migreg_migrate_regions(ZZ *zz, Region *regions, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int *npids, int nregions, int *c2) { char *yo = "Zoltan_Oct_migreg_migrate_regions"; int i; /* index counter */ int ierr = ZOLTAN_OK; int n_import; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned by communication routines */ Region *import_objs = NULL; /* Array of import objects used to request the objs from other processors. */ ZOLTAN_ID_PTR import_gids = NULL; /* Array of global IDs of import_objs. */ ZOLTAN_ID_PTR import_lids = NULL; /* Array of local IDs of import_objs. */ int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; ierr = Zoltan_Comm_Create(&comm_plan, nregions, npids, zz->Communicator, MIGMIGREGCommCreate, &n_import); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Create"); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } *c2 = n_import; if (n_import > 0) { import_objs = (Region *) ZOLTAN_MALLOC(n_import * sizeof(Region)); import_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, n_import); import_lids = ZOLTAN_MALLOC_LID_ARRAY(zz, n_import); if (!import_objs || !import_gids || (num_lid_entries && !import_lids)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_MEMERR; } } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo, (char *) regions, sizeof(Region), (char *) import_objs); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-1, (char *) gids, sizeof(ZOLTAN_ID_TYPE)*num_gid_entries, (char *) import_gids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } if (num_lid_entries > 0) { ierr = Zoltan_Comm_Do(comm_plan, MIGMIGREGCommDo-2, (char *) lids, sizeof(ZOLTAN_ID_TYPE)*num_lid_entries, (char *) import_lids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Comm_Do."); ZOLTAN_TRACE_EXIT(zz, yo); ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); return (ierr); } } for (i=0; i<n_import; i++) { import_objs[i].Global_ID = &(import_gids[i*num_gid_entries]); import_objs[i].Local_ID = (num_lid_entries ? &(import_lids[i*num_lid_entries]) : NULL); Zoltan_Oct_insert_orphan(zz, import_objs[i]); } ZOLTAN_FREE(&import_objs); ZOLTAN_FREE(&import_gids); ZOLTAN_FREE(&import_lids); ierr = Zoltan_Comm_Destroy(&comm_plan); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } return ierr; }
static int _Zoltan_Get_Obj_List( ZZ *zz, int *num_obj, ZOLTAN_ID_PTR *global_ids, ZOLTAN_ID_PTR *local_ids, int wdim, float **objwgts, int **parts, int special_malloc ) { char *yo = "_Zoltan_Get_Obj_List"; int i, n; int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; int alloced_gids = 0, alloced_lids = 0; int gid_off, lid_off; ZOLTAN_ID_PTR lid, next_lid; /* Temporary pointers to local IDs; used to pass NULL to query functions when NUM_LID_ENTRIES == 0. */ float *next_objwgt; /* Temporarry pointer to an object weight; used to pass NULL to query functions when wdim=0 */ int ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); *num_obj = 0; *objwgts = NULL; if (zz->Get_Num_Obj != NULL) { *num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); goto End; } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); ierr = ZOLTAN_FATAL; goto End; } if (*num_obj > 0) { /* * Test global_ids and local_ids for NULL. * Should be NULL for doing partitioning. * Should not be NULL if doing ordering. */ if (special_malloc){ if (*global_ids == NULL) { Zoltan_Special_Malloc(zz, (void **)global_ids, *num_obj, ZOLTAN_SPECIAL_MALLOC_GID); alloced_gids = 1; } if (*local_ids == NULL) { Zoltan_Special_Malloc(zz, (void **)local_ids, *num_obj, ZOLTAN_SPECIAL_MALLOC_LID); alloced_lids = 1; } Zoltan_Special_Malloc(zz, (void **)parts, *num_obj, ZOLTAN_SPECIAL_MALLOC_INT); } else{ if (*global_ids == NULL) { *global_ids = ZOLTAN_MALLOC_GID_ARRAY(zz, *num_obj); alloced_gids = 1; } if (*local_ids == NULL) { *local_ids = ZOLTAN_MALLOC_LID_ARRAY(zz, *num_obj); alloced_lids = 1; } *parts = (int *) ZOLTAN_MALLOC(*num_obj * sizeof(int)); } if (wdim > 0) *objwgts = (float*) ZOLTAN_MALLOC (sizeof(float) * *num_obj * wdim); if ((*global_ids == NULL) || (num_lid_entries > 0 && *local_ids == NULL) || (*parts == NULL) || (wdim > 0 && *objwgts == NULL)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory Error."); ierr = ZOLTAN_MEMERR; goto End; } if (zz->Get_Obj_List != NULL){ /* Get object list directly */ zz->Get_Obj_List(zz->Get_Obj_List_Data, num_gid_entries, num_lid_entries, *global_ids, *local_ids, wdim, *objwgts, &ierr); } else if ((zz->Get_First_Obj != NULL) && (zz->Get_Next_Obj != NULL)){ /* Use iterator functions to loop through object list */ if (zz->Get_First_Obj(zz->Get_First_Obj_Data, num_gid_entries, num_lid_entries, *global_ids, *local_ids, wdim, *objwgts, &ierr)){ n = *num_obj; i = 0; while (!ierr && (i<n-1)){ gid_off = i * num_gid_entries; lid_off = i * num_lid_entries; lid = (num_lid_entries ? &((*local_ids)[lid_off]) : NULL); next_lid = (num_lid_entries ? &((*local_ids)[lid_off+num_lid_entries]) : NULL); next_objwgt = (wdim ? (*objwgts) + (i+1)*wdim : NULL); zz->Get_Next_Obj(zz->Get_Next_Obj_Data, num_gid_entries, num_lid_entries, &((*global_ids)[gid_off]), lid, &((*global_ids)[gid_off+num_gid_entries]), next_lid, wdim, next_objwgt, &ierr); i++; } } } else { /* No way to get objects */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_OBJ_LIST_FN or " "ZOLTAN_FIRST_OBJ_FN/ZOLTAN_NEXT_OBJ_FN."); ierr = ZOLTAN_FATAL; goto End; } /* Get partition information for objects. */ /* Call user-callback if provided; otherwise, all parts == zz->Proc */ if (zz->Get_Part == NULL && zz->Get_Part_Multi == NULL) { for (i = 0; i < *num_obj; i++) (*parts)[i] = zz->Proc; } else if (zz->Get_Part_Multi) { zz->Get_Part_Multi(zz->Get_Part_Multi_Data, num_gid_entries, num_lid_entries, *num_obj, *global_ids, *local_ids, *parts, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_PART_MULTI_FN"); goto End; } } else { for (i = 0; i < *num_obj; i++) { lid = (num_lid_entries ? &((*local_ids)[i*num_lid_entries]) : NULL); (*parts)[i] = zz->Get_Part(zz->Get_Part_Data, num_gid_entries, num_lid_entries, &((*global_ids)[i*num_gid_entries]), lid, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from ZOLTAN_PART_FN"); goto End; } } } } End: if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no lists returned."); if (special_malloc){ if (alloced_gids) Zoltan_Special_Free(zz, (void **)global_ids, ZOLTAN_SPECIAL_MALLOC_GID); if (alloced_lids) Zoltan_Special_Free(zz, (void **)local_ids, ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz, (void **)parts, ZOLTAN_SPECIAL_MALLOC_INT); } else{ if (alloced_gids) ZOLTAN_FREE(global_ids); if (alloced_lids) ZOLTAN_FREE(local_ids); ZOLTAN_FREE(parts); } ZOLTAN_FREE(objwgts); } ZOLTAN_TRACE_EXIT(zz, yo); return ierr; }
int Zoltan_Invert_Lists( ZZ *zz, /* Zoltan structure. */ int num_in, /* Number of objects in the input lists. */ ZOLTAN_ID_PTR in_global_ids, /* Array of input global IDs. */ ZOLTAN_ID_PTR in_local_ids, /* Array of input local IDs. */ int *in_procs, /* Array of processor IDs of processors owning the input objects. */ int *in_to_part, /* Optional: Array of partition numbers to which input objects should be assigned. */ int *num_out, /* Returned value: Number of output objs. */ ZOLTAN_ID_PTR *out_global_ids,/* Returned value: Array of global IDs of output objects. */ ZOLTAN_ID_PTR *out_local_ids, /* Returned value: Array of local IDs of output objects. */ int **out_procs, /* Returned value: Array of processor IDs to which output objects are assigned. */ int **out_to_part /* Optional: Returned value: Array of partition numbers to which output objects should be assigned. */ ) { /* * Routine to compute the inverse map. Can be used in two ways: * 1. Given, for each processor, a list of objects to be received by the * processor, compute the list of objects that the processor needs to send * to other processors to satisfy their needs. * 2. Given, for each processor, a list of objects to be sent to other * processors, compute the list of objects that the processor needs to receive * to satisfy its needs. */ char *yo = "Zoltan_Invert_Lists"; char msg[256]; ZOLTAN_COMM_OBJ *comm_plan; /* Object returned communication routines */ int msgtag, msgtag2; /* Message tags for communication routines */ int num_gid_entries = zz->Num_GID; int num_lid_entries = zz->Num_LID; int include_parts; /* Flag indicating whether to compute inverse list for partitions. */ int ierr, ret_ierr = ZOLTAN_OK; ZOLTAN_TRACE_ENTER(zz, yo); /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Check that all procs use the same id types. */ ierr = check_invert_input(zz, num_in, in_procs, in_to_part, &num_gid_entries, &num_lid_entries, &include_parts); if (ierr != ZOLTAN_OK) { ZOLTAN_TRACE_EXIT(zz, yo); return ierr; } /* Initialize returned arrays. */ *out_global_ids = NULL; *out_local_ids = NULL; *out_procs = NULL; if (include_parts) *out_to_part = NULL; /* * Compute communication map and num_out, the number of objs this * processor has to out to establish the new decomposition. */ msgtag = 32767; ierr = Zoltan_Comm_Create(&comm_plan, num_in, in_procs, zz->Communicator, msgtag, num_out); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Create.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; goto End; } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm create"); /* * Allocate space for the object tags that need to be outed. Communicate * to get the list of objects to be outed. */ if (*num_out > 0) { if (!Zoltan_Special_Malloc(zz,(void **)out_global_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_GID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_local_ids,*num_out, ZOLTAN_SPECIAL_MALLOC_LID)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (!Zoltan_Special_Malloc(zz,(void **)out_procs,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } if (include_parts) { if (!Zoltan_Special_Malloc(zz,(void **)out_to_part,*num_out, ZOLTAN_SPECIAL_MALLOC_INT)) { ret_ierr = ZOLTAN_MEMERR; goto End; } } } /* * Use the communication plan to send global IDs, local IDs, and processor * numbers. Do in separate communications to avoid a memory copy and to * simplify implementation when a data type is added to the comm. package * (to support heterogeneous computing). */ msgtag2 = 32766; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_global_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*(num_gid_entries)), (char *) *out_global_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } if (num_lid_entries) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_local_ids, (int) (sizeof(ZOLTAN_ID_TYPE)*num_lid_entries), (char *) *out_local_ids); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } Zoltan_Comm_Info(comm_plan, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, *out_procs, NULL); if (include_parts) { msgtag2--; ierr = Zoltan_Comm_Do(comm_plan, msgtag2, (char *) in_to_part, (int) sizeof(int), (char *) *out_to_part); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { sprintf(msg, "Error %s returned from Zoltan_Comm_Do.", (ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL")); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); ret_ierr = ierr; } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done comm_do"); End: Zoltan_Comm_Destroy(&comm_plan); if (ret_ierr == ZOLTAN_MEMERR) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Special_Free(zz,(void**)out_global_ids,ZOLTAN_SPECIAL_MALLOC_GID); Zoltan_Special_Free(zz,(void**)out_local_ids,ZOLTAN_SPECIAL_MALLOC_LID); Zoltan_Special_Free(zz,(void**)out_procs,ZOLTAN_SPECIAL_MALLOC_INT); if (include_parts) Zoltan_Special_Free(zz,(void**)out_to_part,ZOLTAN_SPECIAL_MALLOC_INT); } ZOLTAN_TRACE_EXIT(zz, yo); return (ret_ierr); }
/* * void Zoltan_Oct_gen_tree_from_input_data() * * This function will create a root node of on each processor which will * then be used to create an octree with regions associated with it. The * tree will then be balanced and the output used to balance "mesh regions" * on several processors. */ static void Zoltan_Oct_gen_tree_from_input_data(ZZ *zz, int oct_wgtflag, int *c1, int *c2, int *c3, float *c0, int createpartree) { char *yo = "Zoltan_Oct_gen_tree_from_input_data"; pRList RootList; /* list of all local roots */ pOctant RootOct; /* root octree octant */ COORD min, /* min coord bounds of objects */ max; /* max coord bounds of objects */ int num_extra; /* number of orphaned objects */ int num_objs; /* total number of local objects */ pRegion ptr, /* pointer to iterate trough region list */ ptr1; /* pointer to iterate trough region list */ pOctant root; /* root of the partition tree */ int i; /* index counter */ int count, /* count for leaf nodes in partition tree */ proc, /* proc leaf node of parition tree belongs to */ extra, /* extra leaf node flag, if not evenly divisible */ remainder; /* remainder of node, or processors to fill */ pOctant cursor, /* cursor to iterate through octant list */ cursor2, /* another cursor to iterate through octant list */ parent; /* parent of an octant */ int level, /* number of levels of refinement */ n, /* index counter */ part; /* partition counter */ Map *array; /* map of which processors own which octants */ int hold; /* used for calculating partition divisions */ int ierr = 0; #ifdef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN double bounds[6] = {DBL_MAX,DBL_MAX,DBL_MAX,-DBL_MAX,-DBL_MAX,-DBL_MAX}; COORD global_min, global_max; #endif /* KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN */ int nroots = 0; /*test*/ /* COORD gmin,gmax; */ OCT_Global_Info *OCT_info = (OCT_Global_Info *) (zz->LB.Data_Structure); ZOLTAN_TRACE_ENTER(zz, yo); /* * If there are no objects on this processor, do not create a root octant. * The partitioner will probably assign objects to this processor */ if(zz->Get_Num_Obj == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register ZOLTAN_NUM_OBJ_FN function"); abort(); } *c3 = num_objs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr) { fprintf(stderr, "OCT [%d] %s: Error returned from user defined " "Get_Num_Obj function.\n", zz->Proc, yo); exit (-1); } ptr1 = NULL; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_get_bounds"); /* Need A Function To Get The Bounds Of The Local Objects */ Zoltan_Oct_get_bounds(zz, &ptr1, &num_objs, min, max, oct_wgtflag, c0); #ifndef KDDKDD_NEW_BOUNDS_GEOM_QUERY_FN /* For now, don't want to add the new query function to Zoltan. */ /* Zoltan_Oct_get_bounds appears to compute the global min and max from */ /* the object input. */ vector_set(OCT_info->OCT_gmin, min); vector_set(OCT_info->OCT_gmax, max); #else /*test*/ /*getMaxBounds(&gmin, &gmax);*/ if(zz->Get_Bounds_Geom == NULL) { fprintf(stderr, "OCT %s\n\t%s\n", "Error in octree load balance:", "Must register Get_Bounds_Geom function"); abort(); } zz->Get_Bounds_Geom(zz->Get_Bounds_Geom_Data, bounds, &ierr); MPI_Allreduce(&(bounds[0]), &(global_min[0]), 3, MPI_DOUBLE, MPI_MIN, zz->Communicator); MPI_Allreduce(&(bounds[3]), &(global_max[0]), 3, MPI_DOUBLE, MPI_MAX, zz->Communicator); vector_set(OCT_info->OCT_gmin, global_min); vector_set(OCT_info->OCT_gmax, global_max); #endif /* * the following code segment was added to create a pseudo global octree * needed for the partitioner. The basic idea is to regroup all the * regions into something close to an octree partitioning and build the * tree from that. * NOTE: This way of doing things is very costly, especially when calling * this for the first time on a mesh not partitioned in an octree style * partitioning. */ level = 0; /* initialize level count */ /* * if more than 1 processor, need to find what level of refinement needed * to initially partition bounding box among the processors */ if(zz->Num_Proc > 1) { n = zz->Num_Proc; if(OCT_info->OCT_dimension == 2) hold = 4; else hold = 8; remainder = hold; for(; remainder > 0; level++) { int pr = (int)POW(hold, level); remainder = n - pr; } level--; } ZOLTAN_TRACE_DETAIL(zz, yo, "Before createpartree"); if(createpartree) { /* create the global root octant */ root = Zoltan_Oct_POct_new(OCT_info); Zoltan_Oct_setbounds(root, OCT_info->OCT_gmin, OCT_info->OCT_gmax); /* Zoltan_Oct_setOrientation(root, 0); */ /* subdivide to as many levels as calculated */ for(i=0; i<level; i++) { cursor = root; while(cursor != NULL) { if(Zoltan_Oct_isTerminal(cursor)) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); Zoltan_Oct_terminal_refine(zz, cursor, 0); cursor = cursor2; } else cursor = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); } } #if 0 if(zz->Proc == 0) for(i=0; i<8; i++) if(Zoltan_Oct_child(root, i) == NULL) fprintf(stderr,"NULL child pointer\n"); else fprintf(stderr, "child %d exists\n", i); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Before create map array"); /* this part creates the map array */ if(OCT_info->OCT_dimension == 2) { hold = (int)POW(4, level); /* ignoring the z+ octants */ if(hold == 0) hold = 1; } else hold = (int)POW(8, level); part = hold / zz->Num_Proc; /* how many octants per partition */ remainder = hold % zz->Num_Proc; /* extra octants, not evenly divisible */ extra = zz->Num_Proc - remainder;/* where to start adding extra octants */ array = (Map *) ZOLTAN_MALLOC(hold * sizeof(Map)); /* alloc map array */ if(array == NULL) { fprintf(stderr, "OCT ERROR on proc %d, could not allocate array map\n", zz->Proc); abort(); } /* initialize variables */ proc = 0; count = 0; i = 0; cursor = root; while(cursor != NULL) { cursor2 = Zoltan_Oct_POct_nextDfs(OCT_info, cursor); if((Zoltan_Oct_isTerminal(cursor)) && (i < hold)) { if(proc == extra) { part++; extra = -1; } if(count != part) { array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); count++; } else { count = 1; proc++; array[i].npid = proc; array[i].list = RL_initRootList(); Zoltan_Oct_bounds(cursor, min, max); vector_set(array[i].min, min); vector_set(array[i].max, max); } if(proc == zz->Proc) { array[i].npid = -1; /* KDDKDD Added RL_freeList below. The * KDDKDD implementation from RPI leaked memory because the * KDDKDD test cases for setting array[i].list were not mutually * KDDKDD exclusive. Freeing the list produces the result we got * KDDKDD before, without the memory leak. */ /* LGG -- it seems to me that this array[i].list assignment is * not really necessary. It looks as though it has already been * assigned with the same information from the prev if-else * commented out RL_freeList(), and RL_initRootList() */ /*RL_freeList(&(array[i].list));*/ /* KDDKDD End addition */ /*array[i].list = RL_initRootList();*/ parent = Zoltan_Oct_parent(cursor); if(parent != NULL) Zoltan_Oct_setchild(parent, cursor->which, NULL); /* octant into local root list */ Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, -1); Zoltan_Oct_setMapIdx(cursor, i); nroots++; /* Zoltan_Oct_POct_setparent(OCT_info, cursor, NULL, zz->Proc); octant into local root list */ } i++; } cursor = cursor2; } RootList = Zoltan_Oct_POct_localroots(OCT_info); RootOct = RL_nextRootOctant(&RootList); if(RootOct != root) { /* KDDKDDFREE changed root to &root to allow root to be reset to NULL */ Zoltan_Oct_POct_delTree(OCT_info,&root); } OCT_info->map = array; OCT_info->mapsize = hold; } /* * attach the regions to the root... Zoltan_Oct_fix will create the octree * starting with the root and subdividing as needed */ num_extra = Zoltan_Oct_fix(zz, ptr1, num_objs); ZOLTAN_TRACE_DETAIL(zz, yo, "Calling Zoltan_Oct_migreg_migrate_orphans"); Zoltan_Oct_migreg_migrate_orphans(zz, ptr1, num_extra, level, OCT_info->map, c1, c2); /* ZOLTAN_FREE(&array); */ while(ptr1 != NULL) { ptr = ptr1->next; ZOLTAN_FREE(&(ptr1->Global_ID)); ZOLTAN_FREE(&(ptr1->Local_ID)); ZOLTAN_FREE(&ptr1); ptr1 = ptr; } ZOLTAN_TRACE_EXIT(zz, yo); }
/* if !copy, inmat is not usable after this call */ int Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */ Zoltan_matrix_2d *outmat, int copy) { static char *yo = "Zoltan_Matrix2d_Distribute"; int ierr = ZOLTAN_OK; int nProc_x, nProc_y; int myProc_x, myProc_y; int i, j, cnt; int *proclist = NULL; Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL; ZOLTAN_GNO_TYPE *perm_y = NULL; float *wgtarray = NULL; float *tmpwgtarray = NULL; int msg_tag = 1021982; ZOLTAN_COMM_OBJ *plan; MPI_Comm communicator = MPI_COMM_NULL; int nProc; ZOLTAN_GNO_TYPE *yGNO = NULL; ZOLTAN_GNO_TYPE *pinGNO = NULL; ZOLTAN_GNO_TYPE tmp_gno; void *partdata = NULL; MPI_Datatype zoltan_gno_mpi_type; ZOLTAN_TRACE_ENTER(zz, yo); zoltan_gno_mpi_type = Zoltan_mpi_gno_type(); memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix)); if(copy) { /* TODO: We need to copy the arrays also */ Zoltan_Matrix_Reset (&outmat->mtx); /* Copy also directories */ outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX); if (inmat.ddY == inmat.ddX) outmat->mtx.ddY = outmat->mtx.ddX; else outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY); } communicator = outmat->comm->Communicator; nProc = outmat->comm->nProc; nProc_x = outmat->comm->nProc_x; nProc_y = outmat->comm->nProc_y; myProc_x = outmat->comm->myProc_x; myProc_y = outmat->comm->myProc_y; KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates"); ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx); /* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */ if (inmat.opts.speed != MATRIX_NO_REDIST) { if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) { /* I need to know the original distribution */ if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */ outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear; } else { int *cmember = NULL; cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int)); if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR; Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL, outmat->mtx.nY, NULL); KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register"); partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO, cmember, zz->Num_Proc, zz->Num_Proc); ZOLTAN_FREE(&cmember); Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata); } } /* * Build comm plan for sending non-zeros to their target processors in * 2D data distribution. */ /* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */ proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int)); sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc)); if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR; wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR; yGNO = outmat->mtx.yGNO; pinGNO = outmat->mtx.pinGNO; KDDKDDKDD(zz->Proc, " CommPlan Hash"); cnt = 0; for (i = 0; i < outmat->mtx.nY; i++) { ZOLTAN_GNO_TYPE edge_gno=-1; /* processor row for the edge */ edge_gno = yGNO[i]; for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) { ZOLTAN_GNO_TYPE vtx_gno=-1; /* processor column for the vertex */ vtx_gno = pinGNO[j]; proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = vtx_gno; memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim, outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) { proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData, &sendbuf[cnt].part_y); if (proclist[cnt] < 0) /* Discard this nnz */ continue; sendbuf[cnt].GNO[0] = edge_gno; sendbuf[cnt].GNO[1] = -1; memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float)); cnt++; } } if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) Zoltan_Distribute_Partition_Free(&outmat->hashDistData); if (outmat->mtx.yend != outmat->mtx.ystart + 1) ZOLTAN_FREE(&outmat->mtx.yend); outmat->mtx.yend = NULL; ZOLTAN_FREE(&outmat->mtx.ystart); ZOLTAN_FREE(&outmat->mtx.yGNO); ZOLTAN_FREE(&outmat->mtx.pinGNO); ZOLTAN_FREE(&outmat->mtx.pinwgt); ZOLTAN_FREE(&outmat->mtx.yGID); /* * Send pins to their target processors. * They become non-zeros in the 2D data distribution. */ KDDKDDKDD(zz->Proc, " CommPlan Create"); msg_tag--; ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins); ZOLTAN_FREE(&proclist); nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc)); if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros); ZOLTAN_FREE(&sendbuf); if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */ tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float)); if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR; msg_tag--; Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float), (char *) tmpwgtarray); ZOLTAN_FREE(&wgtarray); } Zoltan_Comm_Destroy(&plan); /* Unpack the non-zeros received. */ KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs"); /* TODO: do take care about singletons */ Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray, &outmat->mtx); } /* Now we just have to change numbering */ outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE)); outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE)); if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR; /* FIXME: Work only in 1D */ tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY; MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator); for (i = 1 ; i <= nProc_y ; i ++) { outmat->dist_y[i] += outmat->dist_y[i-1]; } outmat->dist_x[1] = outmat->mtx.globalX; perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE)); if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR; for (i = 0 ; i < outmat->mtx.nY ; ++i){ perm_y[i] = i + outmat->dist_y[myProc_y]; } KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute"); Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y); KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done"); End: ZOLTAN_FREE(&perm_y); ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&sendbuf); ZOLTAN_FREE(&nonzeros); ZOLTAN_FREE(&tmpwgtarray); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_ParMetis( ZZ *zz, /* Zoltan structure */ float *part_sizes, /* Input: Array of size zz->Num_Global_Parts containing the percentage of work to be assigned to each partition. */ int *num_imp, /* number of objects to be imported */ ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */ ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */ int **imp_procs, /* list of processors to import from */ int **imp_to_part, /* list of partitions to which imported objects are assigned. */ int *num_exp, /* number of objects to be exported */ ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */ ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */ int **exp_procs, /* list of processors to export to */ int **exp_to_part /* list of partitions to which exported objects are assigned. */ ) { char *yo = "Zoltan_ParMetis"; int ierr; ZOLTAN_Third_Graph gr; ZOLTAN_Third_Geom *geo = NULL; ZOLTAN_Third_Vsize vsp; ZOLTAN_Third_Part prt; ZOLTAN_Output_Part part; ZOLTAN_ID_PTR global_ids = NULL; ZOLTAN_ID_PTR local_ids = NULL; int use_timers = 0; int timer_p = -1; int get_times = 0; double times[5]; double pmv3_itr = 0.0; realtype itr = 0.0; indextype options[MAX_PARMETIS_OPTIONS]; char alg[MAX_PARAM_STRING_LEN]; #ifdef ZOLTAN_PARMETIS MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */ /* change our zz struct. */ #endif indextype i; realtype *imb_tols; indextype ncon; indextype edgecut; indextype wgtflag; indextype numflag = 0; indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */ ZOLTAN_TRACE_ENTER(zz, yo); Zoltan_Third_Init(&gr, &prt, &vsp, &part, imp_gids, imp_lids, imp_procs, imp_to_part, exp_gids, exp_lids, exp_procs, exp_to_part); if (sizeof(realtype) != sizeof(float)) { int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1); prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype)); for (i = 0; i < tmp; i++) prt.input_part_sizes[i] = (realtype) part_sizes[i]; /* KDD 2/2014: removed re-scaling part sizes so they sum to one. * part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes. * plus, the code here was wrong for multiple object weights. * similar scaling code did not exist in the Scotch interface. */ prt.part_sizes = prt.input_part_sizes; } else prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes; ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } gr.graph_type = 0; #ifdef ZOLTAN_PARMETIS SET_GLOBAL_GRAPH(&gr.graph_type); /* Select type of graph, negative because we impose them */ /* TODO: add a parameter to select the type, shared with Scotch */ /* if (strcmp (graph_type, "GLOBAL") != 0) { */ /* gr.graph_type = - LOCAL_GRAPH; */ /* if (zz->Num_Proc > 1) { */ /* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */ /* gr.graph_type = - GLOBAL_GRAPH; */ /* retval = ZOLTAN_WARN; */ /* } */ /* } */ #else /* graph is local */ SET_LOCAL_GRAPH(&gr.graph_type); #endif /* ZOLTAN_PARMETIS */ /* Some algorithms use geometry data */ if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */ geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom)); memset (geo, 0, sizeof(ZOLTAN_Third_Geom)); /* ParMETIS will crash if geometric method and some procs have no nodes. */ /* Avoid fatal crash by setting scatter to level 2 or higher. */ gr.scatter_min = 2; if (geo == NULL) { ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory."); return (ZOLTAN_MEMERR); } if (strcmp(alg, "PARTGEOM") == 0) { gr.get_data = 0; } } timer_p = Zoltan_Preprocess_Timer(zz, &use_timers); /* Start timer */ get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME); if (get_times){ MPI_Barrier(zz->Communicator); times[0] = Zoltan_Time(zz->Timer); } vsp.vsize_malloc = 0; #ifdef PARMETIS31_ALWAYS_FREES_VSIZE if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) { /* ParMETIS will free this memory; use malloc to allocate so ZOLTAN_MALLOC counters don't show an error. */ vsp.vsize_malloc = 1 ; } #endif /* PARMETIS31_ALWAYS_FREES_VSIZE */ ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp); if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) { Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL); return (ierr); } /* Get object sizes if requested */ if (options[PMV3_OPT_USE_OBJ_SIZE] && (zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) && (!strcmp(alg, "ADAPTIVEREPART") || gr.final_output)) gr.showMoveVol = 1; /* Get a time here */ if (get_times) times[1] = Zoltan_Time(zz->Timer); /* Get ready to call ParMETIS */ edgecut = -1; wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0); numflag = 0; ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1); if (!prt.part_sizes){ ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL."); } if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) { for (i=0; i<num_part; i++){ indextype j; printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i); for (j=0; j<ncon; j++) printf("%f ", prt.part_sizes[i*ncon+j]); printf("\n"); } } /* if (strcmp(alg, "ADAPTIVEREPART") == 0) */ for (i = 0; i < num_part*ncon; i++) if (prt.part_sizes[i] == 0) ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! " "ParMETIS 3.x will likely fail. Please use a " "different method, or remove the zero-sized " "parts from the problem."); /* Set Imbalance Tolerance for each weight component. */ imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype)); if (!imb_tols){ /* Not enough memory */ ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory."); } for (i=0; i<ncon; i++) imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]); /* Now we can call ParMetis */ /* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */ #ifdef ZOLTAN_PARMETIS if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */ /* First check for ParMetis 3 routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartKway"); ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOMKWAY") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeomKway"); ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts, &wgtflag, &numflag, &ndims, geo->xyz, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "PARTGEOM") == 0){ indextype ndims = geo->ndims; ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_PartGeom"); ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "ADAPTIVEREPART") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_AdaptiveRepart"); ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, &itr, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else if (strcmp(alg, "REFINEKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library " "ParMETIS_V3_RefineKway"); ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &ncon, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part, &comm); ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown ParMetis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_METIS /* TODO: I don't know how to set balance ! */ if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Check for Metis routines */ if (strcmp(alg, "PARTKWAY") == 0){ ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library "); /* Use default options for METIS */ #if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5 options[0] = 0; METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts, &wgtflag, &numflag, &num_part, prt.part_sizes, options, &edgecut, prt.part); #else METIS_SetDefaultOptions(options); METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy, gr.vwgt, vsp.vsize, gr.ewgts, &num_part, prt.part_sizes, imb_tols, options, &edgecut, prt.part); #endif ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library"); } else { /* Sanity check: This should never happen! */ char msg[256]; sprintf(msg, "Unknown Metis algorithm %s.", alg); ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg); } } #endif /* ZOLTAN_METIS */ /* Get a time here */ if (get_times) times[2] = Zoltan_Time(zz->Timer); if (gr.final_output) { /* Do final output now because after the data will not be coherent: unscatter only unscatter part data, not graph */ ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr); } /* Ignore the timings of Final Ouput */ if (get_times) times[3] = Zoltan_Time(zz->Timer); ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part); Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part, num_exp, exp_gids, exp_lids, exp_procs, exp_to_part); /* Get a time here */ if (get_times) times[4] = Zoltan_Time(zz->Timer); if (get_times) Zoltan_Third_DisplayTime(zz, times); if (use_timers && timer_p >= 0) ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator); Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL); if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols); if (geo != NULL) ZOLTAN_FREE(&geo); ZOLTAN_FREE(&global_ids); ZOLTAN_FREE(&local_ids); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); }
int Zoltan_Order( ZZ *zz, /* Zoltan structure */ int num_gid_entries, /* # of entries for a global id */ int num_obj, /* Number of objects to order */ ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */ /* The application must allocate enough space */ int *rank, /* rank[i] is the rank of gids[i] */ int *iperm /* iperm[rank[i]]=i, only for sequential ordering */ ) { /* * Main user-call for ordering. * Input: * zz, a Zoltan structure with appropriate function pointers set. * gids, a list of global ids or enough space to store such a list * lids, a list of local ids or enough space to store such a list * Output: * num_gid_entries * num_lid_entries * gids, a list of global ids (filled in if empty on entry) * lids, a list of local ids (filled in if empty on entry) * rank, rank[i] is the global rank of gids[i] * Return values: * Zoltan error code. */ char *yo = "Zoltan_Order"; int ierr; double start_time, end_time; double order_time[2] = {0.0,0.0}; char msg[256]; int comm[2],gcomm[2]; ZOLTAN_ORDER_FN *Order_fn; struct Zoltan_Order_Options opt; int * vtxdist = NULL; ZOLTAN_ID_PTR local_gids=NULL, lids=NULL; int local_num_obj; int *local_rank = NULL, *local_iperm=NULL; struct Zoltan_DD_Struct *dd = NULL; ZOLTAN_TRACE_ENTER(zz, yo); if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) Zoltan_Print_Key_Params(zz); start_time = Zoltan_Time(zz->Timer); /* * Compute Max number of array entries per ID over all processors. * This is a sanity-maintaining step; we don't want different * processors to have different values for these numbers. */ comm[0] = zz->Num_GID; comm[1] = zz->Num_LID; MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator); zz->Num_GID = gcomm[0]; if (num_gid_entries != zz->Num_GID) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value"); return (ZOLTAN_FATAL); } zz->Order.nbr_objects = num_obj; zz->Order.rank = rank; zz->Order.iperm = iperm; zz->Order.gids = gids; zz->Order.lids = lids; zz->Order.start = NULL; zz->Order.ancestor = NULL; zz->Order.leaves = NULL; zz->Order.nbr_leaves = 0; zz->Order.nbr_blocks = 0; /* * Return if this processor is not in the Zoltan structure's * communicator. */ if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) { ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_OK); } /* * Get ordering options from parameter list. */ /* Set default parameter values */ strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN); #ifdef HAVE_MPI strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN); #else strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN); #endif /* HAVE_MPI */ opt.use_order_info = 0; opt.start_index = 0; opt.reorder = 0; Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method); Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type); Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index); Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder); Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info); Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); zz->Order.start_index = opt.start_index; /* * Check that the user has allocated space for the return args. */ if (!(gids && rank)) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine."); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } /* * Find the selected method. */ if (!strcmp(opt.method, "NONE")) { if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS) ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_WARN); } #ifdef ZOLTAN_PARMETIS else if (!strcmp(opt.method, "NODEND")) { Order_fn = Zoltan_ParMetis_Order; } else if (!strcmp(opt.method, "METIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "LOCAL"); } else if (!strcmp(opt.method, "PARMETIS")) { Order_fn = Zoltan_ParMetis_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); strcpy(opt.order_type, "GLOBAL"); } #endif /* ZOLTAN_PARMETIS */ #ifdef ZOLTAN_SCOTCH else if (!strcmp(opt.method, "SCOTCH")) { Order_fn = Zoltan_Scotch_Order; /* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */ strcpy(opt.method, "NODEND"); /* strcpy(opt.order_type, "GLOBAL"); */ } #endif /* ZOLTAN_SCOTCH */ else { fprintf(stderr, "%s\n", opt.method); ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method"); ZOLTAN_TRACE_EXIT(zz, yo); return (ZOLTAN_FATAL); } if (!strcmp(opt.order_type, "GLOBAL")) strcpy (opt.order_type, "DIST"); if (!strcmp(opt.order_type, "LOCAL")) strcpy (opt.order_type, "SERIAL"); strcpy(zz->Order.order_type, opt.order_type); /* * Construct the heterogenous machine description. */ ierr = Zoltan_Build_Machine_Desc(zz); if (ierr == ZOLTAN_FATAL) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description"); /* * Call the actual ordering function. * Compute gid according to the local graph. */ if (zz->Get_Num_Obj != NULL) { local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr); if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj."); return (ierr); } } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN."); return (ZOLTAN_FATAL); } local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj); local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int)); lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj); ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt); ZOLTAN_FREE(&lids); if (ierr) { sprintf(msg, "Ordering routine returned error code %d.", ierr); if (ierr == ZOLTAN_WARN) { ZOLTAN_PRINT_WARN(zz->Proc, yo, msg); } else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); Zoltan_Multifree(__FILE__, __LINE__, 3, &local_gids, &local_rank, &local_iperm); ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } } ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering"); /* Compute inverse permutation if necessary */ if ((!(opt.return_args & RETURN_RANK) && (rank != NULL)) || (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) { ierr = Zoltan_Get_Distribution(zz, &vtxdist); if (ierr) { /* Error */ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n"); return (ierr); } if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) { /* Compute rank from iperm */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index); } else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) { /* Compute iperm from rank */ ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation"); Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index); } ZOLTAN_FREE(&vtxdist); } ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation"); /* TODO: Use directly the "graph" structure to avoid to duplicate things. */ /* I store : GNO, rank, iperm */ ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0); /* Hope a linear assignment will help a little */ Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj); /* Associate all the data with our xGNO */ Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj); ZOLTAN_FREE(&local_gids); ZOLTAN_FREE(&local_rank); ZOLTAN_FREE(&local_iperm); Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL, num_obj, NULL); Zoltan_DD_Destroy(&dd); ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results"); end_time = Zoltan_Time(zz->Timer); order_time[0] = end_time - start_time; if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) { int i; Zoltan_Print_Sync_Start(zz->Communicator, TRUE); printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc); for (i = 0; i < num_obj; i++) { printf("GID = "); ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)])); printf(", rank = %3d\n", rank[i]); } printf("\n"); Zoltan_Print_Sync_End(zz->Communicator, TRUE); } /* Print timing info */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) { if (zz->Proc == zz->Debug_Proc) { printf("ZOLTAN Times: \n"); } Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0], "ZOLTAN Balance: "); } ZOLTAN_TRACE_EXIT(zz, yo); if (ierr) return (ierr); else return (ZOLTAN_OK); }
int Zoltan_Scatter_Graph( indextype **vtxdist, indextype **xadj, indextype **adjncy, weighttype **vwgt, indextype **vsize, weighttype **adjwgt, realtype **xyz, int ndims, /* # dimensions of xyz geometry data */ int vwgt_dim, ZZ *zz, ZOLTAN_COMM_OBJ **plan ) { static char *yo = "Zoltan_Scatter_Graph"; char msg[256]; indextype *old_vtxdist, *old_adjncy; indextype *old_xadj; indextype *old_vsize; weighttype *old_vwgt, *old_adjwgt; realtype *old_xyz; int *ptr, *proclist = NULL, *proclist2 = NULL; int i, j, num_obj, old_num_obj, num_edges, nrecv; int use_graph; /* do we use graph data, or only the geometry? */ int use_vsize; /* do we use the vsize array? */ int ewgt_dim= zz->Edge_Weight_Dim; ZOLTAN_COMM_OBJ *plan2; ZOLTAN_TRACE_ENTER(zz, yo); /* Save pointers to "old" data distribution */ old_adjncy = NULL; old_xadj = NULL; old_vwgt = old_adjwgt = NULL; old_vsize = NULL; old_xyz = NULL; old_vtxdist = *vtxdist; if (xadj) old_xadj = *xadj; if (adjncy) old_adjncy = *adjncy; if (vwgt) old_vwgt = *vwgt; if (vsize) old_vsize = *vsize; if (adjwgt) old_adjwgt = *adjwgt; if (xyz) old_xyz = *xyz; old_num_obj = (int)(old_vtxdist[zz->Proc+1] - old_vtxdist[zz->Proc]); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Old number of objects = %d\n", zz->Proc, old_num_obj); /* Compute new distribution, *vtxdist */ (*vtxdist) = (indextype *)ZOLTAN_MALLOC((zz->Num_Proc+1)* sizeof(indextype)); for (i=0; i<=zz->Num_Proc; i++){ (*vtxdist)[i] = (i*old_vtxdist[zz->Num_Proc])/zz->Num_Proc; } /* Check if any proc has graph data */ i = (old_xadj != NULL); MPI_Allreduce(&i, &use_graph, 1, MPI_INT, MPI_LOR, zz->Communicator); j = (old_vsize != NULL); MPI_Allreduce(&j, &use_vsize, 1, MPI_INT, MPI_LOR, zz->Communicator); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: use_graph = %1d, use_vsize = %1d\n", zz->Proc, use_graph, use_vsize); /* Reset all data pointers to NULL for now */ *xadj = NULL; *adjncy = NULL; *vwgt = *adjwgt = NULL; *xyz = NULL; if (use_vsize) *vsize = NULL; /* Convert the xdj array so that it contains the degree of each vertex */ if (use_graph){ for (i=0; i<old_num_obj; i++){ old_xadj[i] = old_xadj[i+1] - old_xadj[i]; } } /* Allocate new space for vertex data */ num_obj = (int)((*vtxdist)[zz->Proc+1] - (*vtxdist)[zz->Proc]); if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: New number of objects = %d\n", zz->Proc, num_obj); if (use_graph) *xadj = (indextype *) ZOLTAN_MALLOC((num_obj+1)*sizeof(indextype)); if (vwgt_dim) *vwgt = (weighttype *) ZOLTAN_MALLOC(vwgt_dim*num_obj*sizeof(weighttype)); if (use_vsize) *vsize = (indextype *) ZOLTAN_MALLOC(num_obj*sizeof(indextype)); if (ndims) *xyz = (realtype *) ZOLTAN_MALLOC(ndims*num_obj*sizeof(realtype)); if (old_num_obj > 0) { /* Set up the communication plan for the vertex data */ proclist = (int *) ZOLTAN_MALLOC(old_num_obj * sizeof(int)); /* Let j be the new owner of vertex old_vtxdist[zz->Proc]+i */ j = 0; while (old_vtxdist[zz->Proc] >= (*vtxdist)[j+1]) j++; for (i=0; i<old_num_obj; i++){ if (old_vtxdist[zz->Proc]+i >= (*vtxdist)[j+1]) j++; proclist[i] = j; } } Zoltan_Comm_Create(plan, old_num_obj, proclist, zz->Communicator, TAG1, &nrecv); if (nrecv != num_obj){ sprintf(msg,"Proc %d received %d object but expected %d.", zz->Proc, nrecv, num_obj); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); /* Free data */ ZOLTAN_FREE(&proclist); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_FATAL; } /* Do the communication. To save memory, we do not pack all the data into * a buffer, but send directly from the old arrays to the new arrays. * We use the vertex communication plan for all the vertex-based arrays * and the edge communication plan for all the edge-based arrays. */ if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Starting vertex-based communication.\n", zz->Proc); if (use_graph){ Zoltan_Comm_Do( *plan, TAG2, (char *) old_xadj, sizeof(indextype), (char *) *xadj); } if (vwgt_dim){ Zoltan_Comm_Do( *plan, TAG3, (char *) old_vwgt, vwgt_dim*sizeof(weighttype), (char *) *vwgt); } if (use_vsize){ Zoltan_Comm_Do( *plan, TAG4, (char *) old_vsize, sizeof(indextype), (char *) *vsize); } if (ndims){ Zoltan_Comm_Do( *plan, TAG5, (char *) old_xyz, ndims*sizeof(realtype), (char *) *xyz); } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Finished vertex-based communication.\n", zz->Proc); if (use_graph){ /* Rebuild xadj from degrees */ for (i=1; i<num_obj; i++) (*xadj)[i] += (*xadj)[i-1]; for (i=num_obj; i>0; i--) (*xadj)[i] = (*xadj)[i-1]; (*xadj)[0] = 0; /* Allocate space for new edge data structures */ num_edges = (*xadj)[num_obj]; *adjncy = (indextype *) ZOLTAN_MALLOC(num_edges*sizeof(indextype)); if (ewgt_dim) *adjwgt = (weighttype *) ZOLTAN_MALLOC(ewgt_dim*num_edges*sizeof(weighttype)); /* Set up the communication plan for the edge data. */ ptr = proclist2 = (int *) ZOLTAN_MALLOC(old_xadj[old_num_obj] * sizeof(int)); for (i=0; i<old_num_obj; i++) for (j=0; j<old_xadj[i]; j++) *ptr++ = proclist[i]; if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) { printf("[%1d] Debug: Allocated proclist of length " TPL_IDX_SPEC " for edges.\n", zz->Proc, old_xadj[old_num_obj]); } Zoltan_Comm_Create(&plan2, (int)old_xadj[old_num_obj], proclist2, zz->Communicator, TAG1, &nrecv); if (nrecv != num_edges){ sprintf(msg,"Proc %d received %d edges but expected %d.", zz->Proc, nrecv, num_edges); ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg); /* Free data */ ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&proclist2); ZOLTAN_FREE(&old_vtxdist); ZOLTAN_FREE(&old_xadj); ZOLTAN_FREE(&old_adjncy); ZOLTAN_FREE(&old_vwgt); ZOLTAN_FREE(&old_vsize); ZOLTAN_FREE(&old_adjwgt); ZOLTAN_FREE(&old_xyz); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_FATAL; } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Starting edge-based communication.\n", zz->Proc); /* Do the communication. */ Zoltan_Comm_Do( plan2, TAG2, (char *) old_adjncy, sizeof(indextype), (char *) *adjncy); if (ewgt_dim){ Zoltan_Comm_Do( plan2, TAG3, (char *) old_adjwgt, ewgt_dim*sizeof(weighttype), (char *) *adjwgt); } if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) printf("[%1d] Debug: Finished edge-based communication.\n", zz->Proc); /* Free the comm. plan for edge data */ Zoltan_Comm_Destroy(&plan2); } /* end of use_graph */ /* Free data structures */ ZOLTAN_FREE(&proclist); ZOLTAN_FREE(&proclist2); ZOLTAN_FREE(&old_vtxdist); ZOLTAN_FREE(&old_xadj); ZOLTAN_FREE(&old_adjncy); ZOLTAN_FREE(&old_vwgt); ZOLTAN_FREE(&old_vsize); ZOLTAN_FREE(&old_adjwgt); ZOLTAN_FREE(&old_xyz); ZOLTAN_TRACE_EXIT(zz, yo); return ZOLTAN_OK; }
/* * void Zoltan_Oct_migrate_objects() * * sets up the export_tags, and import_tags for the application that called * this load balancing routine */ int Zoltan_Oct_migrate_objects(ZZ *zz, pOctant *octs, int *newpids, int nocts, int *nsenregs, pRegion *import_regions, int *nrecregs, float *c2, float *c3, int *counter3, int *counter4) { int i; /* index counter */ int *tag_pids; /* array of which processors to send information */ int np_regs; /* number of regions previously imported */ Region *p_reg; /* previously imported regions */ ZOLTAN_ID_PTR p_gids; /* global IDs of previously imported regions */ ZOLTAN_ID_PTR p_lids; /* local IDs of previously imported regions */ ZOLTAN_ID_PTR exported_gids, exported_lids; Region *exported_regions; int max_objs; int ierr = ZOLTAN_OK; char *yo = "Zoltan_Oct_migrate_objects"; p_reg = *import_regions = exported_regions = NULL; exported_gids = p_gids = NULL; exported_lids = p_lids = NULL; /* tag all the regions to be exported */ ierr = tag_regions(zz, octs, newpids, nocts, &exported_regions, &exported_gids, &exported_lids, nsenregs, &tag_pids, &p_reg, &p_gids, &p_lids, &np_regs, c2, &max_objs); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } /* get all the region tags that are being imported */ ierr = malloc_new_objects(zz, *nsenregs, exported_regions, exported_gids, exported_lids, tag_pids, nrecregs, import_regions, p_reg, p_gids, p_lids, np_regs, c3); if(ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) { ZOLTAN_TRACE_EXIT(zz, yo); return (ierr); } if(np_regs > 0){ ZOLTAN_FREE(&p_reg); ZOLTAN_FREE(&p_gids); ZOLTAN_FREE(&p_lids); } ZOLTAN_FREE(&exported_regions); ZOLTAN_FREE(&exported_gids); ZOLTAN_FREE(&exported_lids); if(max_objs > (*counter3)) (*counter3) = max_objs; i = (max_objs - (*nsenregs) + (*nrecregs) - np_regs); if(i > (*counter3)) (*counter3) = i; (*counter4) = (*nrecregs) - np_regs; /* fprintf(stderr,"(%d) nrectags = %d\n", zz->Proc, (*nrecregs)); * for(i=0; i<(*nrecregs); i++) * fprintf(stderr,"%d\n", (*import_regions)[i].Proc); */ ZOLTAN_FREE(&tag_pids); return ierr; }