static int
Zoltan_Preprocess_Add_Weight (ZZ *zz,
ZOLTAN_Third_Graph * gr,
ZOLTAN_Third_Part * prt,
char * add_obj_weight)
{
static char * yo = "Zoltan_Preprocess_Add_Weight";
/* Add a vertex weight? */
int add_type = 0;
weighttype *vwgt_new;
int ierr = ZOLTAN_OK;
int i,j;
vwgt_new = (weighttype *)ZOLTAN_MALLOC((gr->obj_wgt_dim+1)*gr->num_obj
* sizeof(weighttype));
if ((!strcasecmp(add_obj_weight, "UNIT")) || (!strcasecmp(add_obj_weight, "VERTICES"))){
add_type = 1;
}
else if ((!strcasecmp(add_obj_weight, "VERTEX DEGREE")) ||
(!strcasecmp(add_obj_weight, "PINS")) ||
(!strcasecmp(add_obj_weight, "NONZEROS"))){
add_type = 2;
}
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid parameter value for ADD_OBJ_WEIGHT!\n");
ierr = ZOLTAN_WARN;
add_type = 0;
}
if (add_type){
if (prt != NULL) {
/* update part_sizes array */
ierr = Zoltan_LB_Add_Part_Sizes_Weight(zz,
(gr->obj_wgt_dim ? gr->obj_wgt_dim : 1),
gr->obj_wgt_dim+1,
prt->input_part_sizes, &prt->part_sizes);
}
/* Add implicit weights in new array */
for (i=0; i<gr->num_obj; i++){
/* First copy old weights */
for (j=0; j<gr->obj_wgt_dim; j++){
vwgt_new[i*(gr->obj_wgt_dim+1)+j] = gr->vwgt[i*gr->obj_wgt_dim+j];
}
if (add_type==1)
/* unit weight */
vwgt_new[i*(gr->obj_wgt_dim+1)+gr->obj_wgt_dim] = 1;
else if (add_type==2)
/* weight is vertex degree (EBEB should we add +1?) */
/* vwgt_new[i*(gr->obj_wgt_dim+1)+gr->obj_wgt_dim] = 10*(gr->xadj[i+1] -gr->xadj[i]) + 1; */
vwgt_new[i*(gr->obj_wgt_dim+1)+gr->obj_wgt_dim] = gr->xadj[i+1] -gr->xadj[i];
}
/* Use new vwgt array */
if (gr->vwgt) ZOLTAN_FREE(&gr->vwgt);
gr->vwgt = vwgt_new;
gr->obj_wgt_dim += 1;
}
return (ierr);
}
static int
Zoltan_Postprocess_Order (ZZ *zz,
ZOLTAN_Third_Graph *gr,
ZOLTAN_Output_Order *ord)
{
int ierr = ZOLTAN_OK;
int i;
/* Ordering */
/* ParMetis produces the rank vector in Zoltan lingo */
if (ord->rank != NULL) {
/* Check if start_index != 0 */
if (ord->order_opt && ord->order_opt->start_index) {
int start_index;
start_index = ord->order_opt->start_index;
for (i=0; i<gr->num_obj; i++){
ord->rank[i] += start_index;
}
}
}
else {
ZOLTAN_PRINT_WARN(zz->Proc, __func__, "rank is NULL, no data returned");
ierr = ZOLTAN_WARN;
}
/* If we did local ordering via METIS, then we also have the inv. perm. */
if ((gr->graph_type == LOCAL_GRAPH) && (ord->iperm != NULL)){
int start_index;
start_index = ord->order_opt->start_index;
for (i=0; i<gr->num_obj; i++){
ord->iperm[i] += start_index;
}
/* EBEB: Return parameter that says we have computed both return args? */
}
/* Fill in the Zoltan Order Struct */
/* EBEB: For now, discard separator info */
if (0){ /* DEBUG: Print separator sizes to file */
FILE *fp;
fp = fopen("separators.txt", "w");
fprintf(fp, "%i\n", ord->num_part);
for (i=0; i<ord->num_part; ++i){
fprintf(fp, TPL_IDX_SPEC " ", ord->sep_sizes[i]);
}
fprintf(fp, "\n");
for (i=ord->num_part; i<2*ord->num_part-1; ++i){
fprintf(fp, TPL_IDX_SPEC " ", ord->sep_sizes[i]);
}
fprintf(fp, "\n");
fclose(fp);
}
return (ierr);
}
void Zoltan_HG_HGraph_Print(
ZZ *zz, /* the Zoltan data structure */
ZHG *zoltan_hg,
HGraph *hg,
Partition parts,
FILE *fp
)
{
/* Printing routine. Can be used to print a Zoltan_HGraph or just an HGraph.
* Set zoltan_hg to NULL if want to print only an HGraph.
* Lots of output; synchronized across processors, so is a bottleneck.
*/
int i;
int p;
int num_gid = zz->Num_GID;
int num_lid = zz->Num_LID;
char *yo = "Zoltan_HG_HGraph_Print";
if (zoltan_hg != NULL && hg != &zoltan_hg->HG) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Input hg != Zoltan HG");
return;
}
#if 0
Zoltan_Print_Sync_Start (zz->Communicator, 1);
#else
for (p=0; p < zz->Num_Proc; p++){
if (p == zz->Proc){
#endif
/* Print Vertex Info */
fprintf (fp, "%s Proc %d\n", yo, zz->Proc);
fprintf (fp, "Vertices (GID, LID, index)\n");
for (i = 0; i < zoltan_hg->nObj; i++) {
fprintf(fp, "(");
ZOLTAN_PRINT_GID(zz, &zoltan_hg->objGID[i * num_gid]);
fprintf(fp, ", ");
ZOLTAN_PRINT_LID(zz, &zoltan_hg->objLID[i * num_lid]);
fprintf(fp, ", %d)\n", i);
}
Zoltan_HG_Print(zz, hg, parts, fp, "Build");
fflush(fp);
#if 0
Zoltan_Print_Sync_End(zz->Communicator, 1);
#else
}
MPI_Barrier(zz->Communicator);
MPI_Barrier(zz->Communicator);
MPI_Barrier(zz->Communicator);
}
MPI_Barrier(zz->Communicator);
MPI_Barrier(zz->Communicator);
MPI_Barrier(zz->Communicator);
#endif
}
int Zoltan_Set_Timer_Param(
const char *name, /* input: name of variable */
const char *val, /* input: value of variable */
int *timer) /* output: timer type */
{
PARAM_UTYPE result; /* value returned from Check_Param */
int index; /* index returned from Check_Param */
int status;
PARAM_VARS Timer_params[] = {
{ "TIMER", NULL, "STRING", 0 },
{ NULL, NULL, NULL, 0 }
};
char *yo = "Zoltan_Set_Timer_Param";
(*timer) = ZOLTAN_TIME_WALL; /* default timer value */
status = Zoltan_Check_Param(name, val, Timer_params, &result, &index);
if (status == 0 && index == 0) {
if (!strcmp(result.sval, "WALL"))
(*timer) = ZOLTAN_TIME_WALL;
else if (strcmp(result.sval, "CPU")==0) {
(*timer) = ZOLTAN_TIME_CPU;
}
else if (strcmp(result.sval, "USER")==0){
#ifndef NO_TIMES
(*timer) = ZOLTAN_TIME_USER;
#else
ZOLTAN_PRINT_WARN(-1, yo, "User time not available;"
" CPU clock time will be used instead.");
(*timer) = ZOLTAN_TIME_CPU;
#endif
}
else{
char msg[256];
sprintf(msg, "Unknown timer option %s.", result.sval);
ZOLTAN_PRINT_WARN(-1, yo, msg);
status = 2; /* Illegal parameter */
}
}
return(status);
}
int Zoltan_Bind_Param_Vec(
PARAM_VARS *params, /* parameter structure */
char *name, /* parameter name */
void *var, /* pointer to variable to be associated
with the parameter name */
int dim /* dimension of parameter vector */
)
{
/*
* Function to bind a parameter name to a variable.
* On output:
* ZOLTAN_OK indicates success.
* ZOLTAN_WARN indicates that parameter name was not found (misspelled?).
* No binding took place. A warning message is printed in this case.
* ZOLTAN_FATAL signals something more serious.
*/
char *yo = "Zoltan_Bind_Param";
char msg[256];
char *name2; /* clean version of name */
int flag; /* return value from function */
PARAM_VARS *ptr; /* pointer to a parameter */
/* First convert to upper case & remove leading white space. */
flag = Zoltan_Clean_String(name, &name2);
if (flag)
return (flag);
/* Search through parameter array to find name2 */
for (ptr = params; ptr->name != NULL; ptr++) {
if (!strcmp(name2, ptr->name)) { /* string match */
ptr->ptr = var;
ptr->length = dim;
ZOLTAN_FREE(&name2);
return (ZOLTAN_OK);
}
}
/* If we reach this point, the parameter name must be invalid */
sprintf(msg, "Parameter name %s not found; it will"
"not be bound to any variable.", name2);
ZOLTAN_PRINT_WARN(-1, yo, msg);
ZOLTAN_FREE(&name2);
return (ZOLTAN_WARN);
}
int Zoltan_Drum_Stop_Monitors(ZZ *zz) {
int ierr;
char *yo = "Zoltan_Drum_Stop_Monitors";
FILE *fp;
if ((zz->Drum.use_drum == 0) || (zz->Drum.start_monitors == 0))
return ZOLTAN_OK;
if (!zz->Drum.dmm) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "DRUM not initialized");
return ZOLTAN_FATAL;
}
ierr = DRUM_stopMonitoring(zz->Drum.dmm);
if (ierr == DRUM_FATAL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to stop DRUM monitors");
return ZOLTAN_FATAL;
}
ierr = DRUM_computePowers(zz->Drum.dmm);
if (ierr == DRUM_FATAL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to compute DRUM powers");
return ZOLTAN_FATAL;
}
/* print the "power file" if it was requested */
if (zz->Proc == 0 && strcmp(zz->Drum.power_filename,"")) {
fp = fopen(zz->Drum.power_filename, "a");
if (fp) {
DRUM_printMachineModel(zz->Drum.dmm, fp);
fclose(fp);
}
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Could not open power file");
return ZOLTAN_WARN;
}
}
else {
if (zz->Proc == 0) {
printf("Skipping power file output\n"); fflush(stdout);
}
}
return ZOLTAN_OK;
}
int Zoltan_Build_Machine_Desc(
ZZ *zz /* The Zoltan structure. */
)
{
char *yo = "Zoltan_Build_Machine_Desc";
int ierr = ZOLTAN_OK;
int use_mach_desc;
char filename[256];
Zoltan_Bind_Param(Mach_params, "USE_MACHINE_DESC", (void *) &use_mach_desc);
Zoltan_Bind_Param(Mach_params, "MACHINE_DESC_FILE", (void *) filename);
use_mach_desc = 0;
strcpy(filename, MACHINE_DESC_FILE_DEFAULT);
Zoltan_Assign_Param_Vals(zz->Params, Mach_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
if (use_mach_desc > 0) {
/* If zz->Machine_Desc already exists, don't rebuild it
* unless USE_MACHINE_DESC has been set to 2.
*/
if ((zz->Machine_Desc == NULL) || (use_mach_desc==2)){
/* Read machine description from file.
* Use Zoltan_Get_Processor_Name to extract the sub-machine
* on which this Zoltan structure is running.
* Broadcast the machine structure to all procs.
*/
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Sorry, heterogeneous load-balancing "
"is still under development!");
ierr = ZOLTAN_WARN;
}
}
else {
zz->Machine_Desc = NULL;
}
return ierr;
}
int Zoltan_PHG_isPrime(int n)
{
/* Naive program to test for primality. */
/* Returns accurate results for n <= maxValid. */
static const int maxValid = 250000;
static const int primes[] = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37,
41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97,
101, 103, 107, 109, 113, 127, 131, 137, 139,
149, 151, 157, 163, 167, 173, 179, 181,
191, 193, 197, 199, 211, 223, 227, 229,
233, 239, 241, 251, 257, 263, 269, 271, 277,
281, 283, 293, 307, 311, 313, 317, 331, 337,
347, 349, 353, 359, 367, 373, 379, 383, 389,
397, 401, 409, 419, 421, 431, 433, 439,
443, 449, 457, 461, 463, 467, 479, 487, 491, 499};
static const int numprimes = sizeof(primes) / sizeof(int);
int i;
int rootn;
int isprime = 1;
if (n == 1) return 0;
rootn = sqrt((double)n)+1;
for (i = 0; (i < numprimes) && (primes[i] < rootn); i++)
if (!(n%primes[i])) {
isprime = 0;
break;
}
if (isprime && n>maxValid) {
char str[128];
sprintf(str, "Warning: isPrime function may not be accurate for n(%i)>%d\n",
n, maxValid);
ZOLTAN_PRINT_WARN(-1, "Zoltan_PHG_isPrime", str);
}
return isprime;
}
static int
give_proc (indextype vertex, const indextype *vtxdist, int numProc, int *myproc)
{
int currentproc;
if ((((*myproc) >= 0) && (*myproc) < numProc) &&
(vertex >= vtxdist[*myproc]) && (vertex < vtxdist[*myproc+1])) {
return (*myproc);
}
currentproc = vertex / (vtxdist[1]-vtxdist[0]); /* Assume that vertices are balanced */
if (currentproc >= numProc)
currentproc = numProc - 1;
if ((vertex < vtxdist[0])||( vertex >= vtxdist[numProc])) {
ZOLTAN_PRINT_WARN ((*myproc), "Symmetrize Graph problem (1)", "Unknown vertex");
return (-1);
}
while (1) {
if (vertex >= vtxdist[currentproc + 1]) {
currentproc ++;
continue;
}
if (vertex < vtxdist[currentproc]) {
currentproc --;
continue;
}
break;
}
*myproc =currentproc;
return (currentproc);
}
int Zoltan_PHG_Vertex_Visit_Order(
ZZ *zz,
HGraph *hg,
PHGPartParams *hgp,
int *order)
{
int i, j, edge;
int *ldegree=NULL, *gdegree=NULL; /* local/global degree */
int *lpins=NULL, *gpins=NULL; /* local/global sum of pins */
char *yo= "Zoltan_PHG_Vertex_Visit_Order";
/* Start with linear order. */
for (i=0; i<hg->nVtx; i++)
order[i] = i;
/* Permute order array according to chosen strategy. */
switch (hgp->visit_order){
case 0:
/* random node visit order (recommended) */
/* Synchronize so each proc in column visits in same order */
Zoltan_Srand_Sync(Zoltan_Rand(NULL), &(hg->comm->RNGState_col),
hg->comm->col_comm);
Zoltan_Rand_Perm_Int (order, hg->nVtx, &(hg->comm->RNGState_col));
break;
case 1:
/* linear (natural) vertex visit order */
break;
case 2:
{
/* increasing vertex weight */
float *tmpvwgt;
if (hg->VtxWeightDim == 1)
tmpvwgt = hg->vwgt;
else {
/* Sort based on first component of multidimensional weight */
tmpvwgt = (float *) ZOLTAN_MALLOC(hg->nVtx * sizeof(float));
for (i = 0; i < hg->nVtx; i++)
tmpvwgt[i] = hg->vwgt[i*hg->VtxWeightDim];
}
Zoltan_quicksort_pointer_inc_float (order, tmpvwgt, 0, hg->nVtx-1);
if (tmpvwgt != hg->vwgt) ZOLTAN_FREE(&tmpvwgt);
break;
}
case 3:
/* increasing vertex degree */
/* intentionally fall through into next case */
case 4:
/* increasing vertex degree, weighted by # pins */
/* allocate 4 arrays of size hg->nVtx with a single malloc */
if (!(ldegree = (int *) ZOLTAN_MALLOC (4*sizeof(int) * hg->nVtx))){
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Out of memory");
ZOLTAN_FREE (&ldegree);
return ZOLTAN_MEMERR;
}
/* first local data, then global data */
lpins = ldegree + hg->nVtx;
gdegree = lpins + hg->nVtx;
gpins = gdegree + hg->nVtx;
/* loop over vertices */
for (i=0; i<hg->nVtx; i++){
ldegree[i] = hg->vindex[i+1] - hg->vindex[i]; /* local degree */
lpins[i] = 0;
/* loop over edges, sum up #pins */
for (j= hg->vindex[i]; j < hg->vindex[i+1]; j++) {
edge = hg->vedge[j];
lpins[i] += hg->hindex[edge+1] - hg->hindex[edge];
}
}
/* sum up local degrees in each column to get global degrees */
/* also sum up #pins in same communication */
MPI_Allreduce(ldegree, gdegree, 2*hg->nVtx, MPI_INT, MPI_SUM,
hg->comm->col_comm);
/* sort by global values. same on every processor. */
if (hgp->visit_order == 3)
Zoltan_quicksort_pointer_inc_int_int (order, gdegree, gpins,
0, hg->nVtx-1);
else /* hgp->visit_order == 4 */
Zoltan_quicksort_pointer_inc_int_int (order, gpins, gdegree,
0, hg->nVtx-1);
ZOLTAN_FREE (&ldegree);
break;
/* add more cases here */
}
return ZOLTAN_OK;
}
int Zoltan_HG_Check (
ZZ *zz,
HGraph *hg
)
{
int i;
int iedge; /* iedge denotes an hyperedge */
int j; /* j is the index of a vertex */
int k; /* k is an index of hyperedge */
int *check;
char str[256];
int err = ZOLTAN_OK;
char *yo = "Zoltan_HG_Check";
ZOLTAN_TRACE_ENTER(zz, yo);
if ((hg->nEdge && !hg->hindex) ||
(hg->nVtx && !hg->vindex) ||
(hg->nPins && (!hg->vedge || !hg->hvertex))) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "NULL arrays found");
err = ZOLTAN_WARN;
goto End;
}
if (hg->nPins) {
check = (int*) ZOLTAN_MALLOC(((hg->nEdge > hg->nVtx) ? hg->nEdge : hg->nVtx)
* sizeof(int));
if (check == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unable to allocate memory.");
err = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < hg->nEdge; i++)
check[i] = -1;
for (i = 0; i < hg->nVtx; i++)
for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
if (check[hg->vedge[j]] < i)
check[hg->vedge[j]] = i;
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple hedge for same vertex.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->nVtx; i++)
check[i] = -1;
for (i = 0; i < hg->nEdge; i++)
for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
if (check[hg->hvertex[j]] < i)
check[hg->hvertex[j]] = i;
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found multiple vertex for same hedge.");
err = ZOLTAN_WARN;
}
ZOLTAN_FREE (&check);
}
for (i = 0; i < hg->VtxWeightDim * hg->nVtx; i++)
if (hg->vwgt[i] < 0.0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative vertex weight.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->EdgeWeightDim * hg->nEdge; i++)
if (hg->ewgt[i] < 0.0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found negative edge weight.");
err = ZOLTAN_WARN;
}
if (hg->comm && hg->comm->nProc_x == 1) {
/* In 2D distribution, check makes sense only if proc has entire hedge */
for (i = 0; i < hg->nEdge; i++)
if ((hg->hindex[i+1] == hg->hindex[i])) {
sprintf (str, "Found hyperedge with no vertices: "
"edge %d has %d vtxs\n", i, (hg->hindex[i+1] - hg->hindex[i]));
ZOLTAN_PRINT_WARN(zz->Proc, yo, str);
err = ZOLTAN_WARN;
}
}
for (i = 0; i < hg->nEdge; i++)
for (j = hg->hindex[i]; j < hg->hindex[i+1]; j++)
if (hg->hvertex[j] < 0 || hg->hvertex[j] >= hg->nVtx) {
ZOLTAN_PRINT_WARN(zz->Proc, yo,"Found vertex out of range in hvertex.");
err = ZOLTAN_WARN;
}
for (i = 0; i < hg->nVtx; i++)
for (j = hg->vindex[i]; j < hg->vindex[i+1]; j++)
if (hg->vedge[j] < 0 || hg->vedge[j] >= hg->nEdge) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Found edge out of range in vedge.");
err = ZOLTAN_WARN;
}
/* starting from (hindex,hvertex), for each edge determine each associated
* vertex. Then for each vertex lookup associated edges using (vindex, vedge)
*/
for (iedge = 0; iedge < hg->nEdge; iedge++)
for (j = hg->hindex[iedge]; j < hg->hindex[iedge+1]; j++) {
/* for each hyperedge get index to vertices */
for (k=hg->vindex[hg->hvertex[j]]; k<hg->vindex[hg->hvertex[j]+1]; k++)
/* for each vertex of current hyperedge get index to hyperedges */
if (hg->vedge[k] == iedge) /* does it match with original edge? */
break;
if (k == hg->vindex[hg->hvertex[j]+1]) {
/* if no match was found then failure, else keep on */
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Inconsistent hvertex/vedge");
err = ZOLTAN_WARN;
break;
}
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return err;
}
int Zoltan_Set_Key_Param(
ZZ *zz, /* Zoltan structure */
const char *name, /* name of variable */
const char *val, /* value of variable */
int idx /* index of vector param, -1 if scalar */
)
{
char *yo = "Zoltan_Set_Key_Param";
char msg[256];
int status; /* return code */
PARAM_UTYPE result; /* value returned from Check_Param */
int index; /* index returned from Check_Param */
int tmp;
int export, import;
status = Zoltan_Check_Param(name, val, Key_params, &result, &index);
if (status == 0) {
switch (index) {
case 0: /* Imbalance_Tol */
if (result.def)
result.fval = ZOLTAN_LB_IMBALANCE_TOL_DEF;
if (result.fval < 1.0) {
sprintf(msg, "Invalid Imbalance_Tol value (%g) "
"being set to %g.", result.fval, ZOLTAN_LB_IMBALANCE_TOL_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.fval = ZOLTAN_LB_IMBALANCE_TOL_DEF;
}
if (idx > zz->Obj_Weight_Dim){
sprintf(msg, "Imbalance_Tol index %d > Obj_Weight_Dim = %d\n",
idx, zz->Obj_Weight_Dim);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
else if (idx < -1){
sprintf(msg, "Invalid Imbalance_Tol index %d\n", idx);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
else if (idx == -1){
/* Set all entries to the same value. */
for (idx=0; idx<zz->LB.Imb_Tol_Len; idx++)
zz->LB.Imbalance_Tol[idx] = result.fval;
}
else
zz->LB.Imbalance_Tol[idx] = result.fval;
status = 3; /* Don't add to Params field of ZZ */
break;
case 1: /* Help_Migrate */
if (result.def)
result.ival = ZOLTAN_AUTO_MIGRATE_DEF;
zz->Migrate.Auto_Migrate = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 2: /* Object weight dim. */
if (result.def)
result.ival = ZOLTAN_OBJ_WEIGHT_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Obj_Weight_Dim value (%d) "
"being set to %d.", result.ival, ZOLTAN_OBJ_WEIGHT_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_OBJ_WEIGHT_DEF;
}
zz->Obj_Weight_Dim = result.ival;
if (zz->Obj_Weight_Dim > zz->LB.Imb_Tol_Len){
/* Resize and reallocate Imb_Tol. */
zz->LB.Imb_Tol_Len += 10;
zz->LB.Imbalance_Tol = (float *) ZOLTAN_REALLOC(zz->LB.Imbalance_Tol, zz->LB.Imb_Tol_Len * sizeof(float));
}
status = 3; /* Don't add to Params field of ZZ */
break;
case 3: /* Edge weight dim. */
case 13:
if (result.def)
result.ival = ZOLTAN_EDGE_WEIGHT_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Edge_Weight_Dim value (%d) "
"being set to %d.", result.ival, ZOLTAN_EDGE_WEIGHT_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_EDGE_WEIGHT_DEF;
}
zz->Edge_Weight_Dim = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 4: /* Debug level */
if (result.def)
result.ival = ZOLTAN_DEBUG_LEVEL_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Debug_Level value (%d) "
"being set to %d.", result.ival, ZOLTAN_DEBUG_LEVEL_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_DEBUG_LEVEL_DEF;
}
zz->Debug_Level = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 5: /* Debug processor */
if (result.def)
result.ival = ZOLTAN_DEBUG_PROC_DEF;
if (result.ival < 0 || result.ival > zz->Num_Proc) {
sprintf(msg, "Invalid Debug_Processor value (%d) "
"being set to %d.", result.ival, ZOLTAN_DEBUG_PROC_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_DEBUG_PROC_DEF;
}
zz->Debug_Proc = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 6: /* Deterministic flag */
if (result.def)
result.ival = ZOLTAN_DETERMINISTIC_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Deterministic value (%d) "
"being set to %d.", result.ival, ZOLTAN_DETERMINISTIC_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_DETERMINISTIC_DEF;
}
zz->Deterministic = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 7: /* Timer */
status = Zoltan_Set_Timer_Param(name, val, &tmp);
zz->Timer = tmp;
Zoltan_Timer_ChangeFlag(zz->ZTime, zz->Timer);
if (status==0) status = 3; /* Don't add to Params field of ZZ */
break;
case 8: /* Num_GID_Entries */
if (result.def)
result.ival = ZOLTAN_NUM_ID_ENTRIES_DEF;
if (result.ival < 1) {
sprintf(msg, "Invalid Num_GID_Entries value (%d); "
"being set to %d.", result.ival, ZOLTAN_NUM_ID_ENTRIES_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_NUM_ID_ENTRIES_DEF;
}
zz->Num_GID = result.ival;
status = 3;
break;
case 9: /* Num_LID_Entries */
if (result.def)
result.ival = ZOLTAN_NUM_ID_ENTRIES_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Num_LID_Entries value (%d); "
"being set to %d.", result.ival, ZOLTAN_NUM_ID_ENTRIES_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_NUM_ID_ENTRIES_DEF;
}
zz->Num_LID = result.ival;
status = 3;
break;
case 10: /* LB.Return_Lists */
export = (strstr(result.sval, "EXPORT") != NULL);
import = (strstr(result.sval, "IMPORT") != NULL);
if ((export && import) || (strcmp(result.sval, "ALL") == 0)) {
tmp = ZOLTAN_LB_ALL_LISTS; /* export AND import lists */
status = 3;
}
else if (import){
tmp = ZOLTAN_LB_IMPORT_LISTS; /* import lists */
status = 3;
}
else if (export){
tmp = ZOLTAN_LB_EXPORT_LISTS; /* export lists */
status = 3;
}
else if (strstr(result.sval, "PART")!=NULL) {
/* list of every object's part assignment */
tmp = ZOLTAN_LB_COMPLETE_EXPORT_LISTS;
status = 3;
}
else if (strcmp(result.sval, "NONE")==0) {
tmp = ZOLTAN_LB_NO_LISTS; /* no lists */
status = 3;
}
else if (strcmp(result.sval, "CANDIDATE_LISTS")==0) {
tmp = ZOLTAN_LB_CANDIDATE_LISTS; /* candidates needed in matching */
status = 3;
}
else{
tmp = ZOLTAN_LB_RETURN_LISTS_DEF;
sprintf(msg, "Unknown return_lists option %s.", result.sval);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
status = 2; /* Illegal parameter */
}
zz->LB.Return_Lists = tmp;
break;
case 11: /* LB_Method */
if (result.def)
strcpy(result.sval, "RCB");
status = Zoltan_LB_Set_LB_Method(zz,result.sval);
if (status == ZOLTAN_OK)
status = 3;
break;
case 12: /* Tflops Special flag */
if (result.def)
result.ival = ZOLTAN_TFLOPS_SPECIAL_DEF;
if (result.ival < 0) {
sprintf(msg, "Invalid Tflops Special value (%d) "
"being set to %d.", result.ival, ZOLTAN_TFLOPS_SPECIAL_DEF);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = ZOLTAN_TFLOPS_SPECIAL_DEF;
}
zz->Tflops_Special = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 14: /* Num_Global_Parts */
case 15:
if (result.def)
result.ival = zz->Num_Proc;
if (result.ival < 1) {
sprintf(msg, "Invalid Num_Global_Parts value (%d); "
"being set to %d.", result.ival,zz->Num_Proc);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = zz->Num_Proc;
}
zz->LB.Num_Global_Parts_Param = result.ival;
status = 3;
break;
case 16: /* Num_Local_Parts */
case 17:
if (result.def)
result.ival = -1;
if (result.ival < -1) {
sprintf(msg, "Invalid Num_Local_Parts value (%d); "
"being set to %d.", result.ival,-1);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
result.ival = -1;
}
zz->LB.Num_Local_Parts_Param = result.ival;
status = 3;
break;
case 18: /* Migrate_Only_Proc_Changes */
if (result.def)
result.ival = ZOLTAN_MIGRATE_ONLY_PROC_CHANGES_DEF;
zz->Migrate.Only_Proc_Changes = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 19: /* LB.Remap */
if (result.def)
result.ival = 0;
zz->LB.Remap_Flag = result.ival;
status = 3; /* Don't add to Params field of ZZ */
break;
case 20: /* Seed */
if (result.def)
result.ival = Zoltan_Seed();
zz->Seed = result.ival;
Zoltan_Srand(result.ival, NULL);
status = 3;
break;
case 21: /* LB_APPROACH */
if (result.def)
strcpy(result.sval, ZOLTAN_LB_APPROACH_DEF);
strcpy(zz->LB.Approach, result.sval);
status = 3;
break;
} /* end switch (index) */
static int Zoltan_Parmetis_Parse(
ZZ* zz,
indextype *options,
char* alg,
realtype* itr,
double *pmv3_itr,
ZOLTAN_Output_Order *ord
)
{
static char * yo = "Zoltan_Parmetis_Parse";
int i;
int output_level, seed, coarse_alg, fold, use_obj_size;
/* Always use ParMetis option array because Zoltan by default
produces no output (silent mode). ParMetis requires options[0]=1
when options array is to be used. */
options[0] = 1;
for (i = 1; i < MAX_PARMETIS_OPTIONS; i++)
options[i] = 0;
/* Set the default option values. */
output_level = 0;
coarse_alg = 2;
use_obj_size = 1;
fold = 0;
seed = GLOBAL_SEED;
if(ord == NULL) {
/* Map LB_APPROACH to suitable PARMETIS_METHOD */
if (!strcasecmp(zz->LB.Approach, "partition")){
strcpy(alg, "PARTKWAY");
}
else if (!strcasecmp(zz->LB.Approach, "repartition")){
strcpy(alg, "ADAPTIVEREPART");
*pmv3_itr = 100.; /* Ratio of inter-proc comm. time to data redist. time;
100 gives similar partition quality to GDiffusion */
}
else if (!strcasecmp(zz->LB.Approach, "refine")){
strcpy(alg, "REFINEKWAY");
}
else { /* If no LB_APPROACH is set, use repartition */
strcpy(alg, "ADAPTIVEREPART");
*pmv3_itr = 100.; /* Ratio of inter-proc comm. time to data redist. time;
100 gives similar partition quality to GDiffusion */
}
}
else {
strcpy(alg, "NODEND");
}
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_METHOD",
(void *) alg);
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_OUTPUT_LEVEL",
(void *) &output_level);
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_SEED",
(void *) &seed);
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_ITR",
(void *) pmv3_itr);
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_COARSE_ALG",
(void *) &coarse_alg);
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_FOLD",
(void *) &fold);
Zoltan_Assign_Param_Vals(zz->Params, Parmetis_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
/* Copy option values to ParMetis options array */
/* In this version of Zoltan, processors and partitions are coupled. */
/* This will likely change in future releases, and then the options */
/* value should change to DISCOUPLED. */
options[PMV3_OPTION_PSR] = COUPLED;
if (pmv3method(alg)){
/* ParMetis 3.0 options */
options[PMV3_OPTION_DBGLVL] = output_level;
options[PMV3_OPTION_SEED] = seed;
options[PMV3_OPT_USE_OBJ_SIZE] = use_obj_size;
if (ord == NULL)
*itr = (realtype)*pmv3_itr;
}
/* If ordering, use ordering method instead of load-balancing method */
if (ord && ord->order_opt && ord->order_opt->method){
strcpy(alg, ord->order_opt->method);
}
if ((zz->Num_Proc == 1) &&
(!strcmp(alg, "ADAPTIVEREPART") ||
!strcmp(alg, "REPARTLDIFFUSION") ||
!strcmp(alg, "REPARTGDIFFUSION") ||
!strcmp(alg, "REPARTREMAP") ||
!strcmp(alg, "REPARTMLREMAP"))) {
/* These ParMETIS methods fail on one processor; an MPI command assumes
at least two processors. */
char str[256];
sprintf(str, "ParMETIS method %s fails on one processor due to a bug"
" in ParMETIS v3.x; resetting method to PartKway.", alg);
ZOLTAN_PRINT_WARN(zz->Proc, yo, str);
strcpy(alg, "PARTKWAY");
return (ZOLTAN_WARN);
}
return(ZOLTAN_OK);
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int num_gid_entries, /* # of entries for a global id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm /* iperm[rank[i]]=i, only for sequential ordering */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
int * vtxdist = NULL;
ZOLTAN_ID_PTR local_gids=NULL, lids=NULL;
int local_num_obj;
int *local_rank = NULL, *local_iperm=NULL;
struct Zoltan_DD_Struct *dd = NULL;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = gcomm[0];
if (num_gid_entries != zz->Num_GID) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "num_gid_entries doesn't have the good value");
return (ZOLTAN_FATAL);
}
zz->Order.nbr_objects = num_obj;
zz->Order.rank = rank;
zz->Order.iperm = iperm;
zz->Order.gids = gids;
zz->Order.lids = lids;
zz->Order.start = NULL;
zz->Order.ancestor = NULL;
zz->Order.leaves = NULL;
zz->Order.nbr_leaves = 0;
zz->Order.nbr_blocks = 0;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
#ifdef HAVE_MPI
strncpy(opt.order_type, "DIST", MAX_PARAM_STRING_LEN);
#else
strncpy(opt.order_type, "SERIAL", MAX_PARAM_STRING_LEN);
#endif /* HAVE_MPI */
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
zz->Order.start_index = opt.start_index;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && rank)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
#ifdef ZOLTAN_PARMETIS
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_SCOTCH
else if (!strcmp(opt.method, "SCOTCH")) {
Order_fn = Zoltan_Scotch_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
/* strcpy(opt.order_type, "GLOBAL"); */
}
#endif /* ZOLTAN_SCOTCH */
else {
fprintf(stderr, "%s\n", opt.method);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
if (!strcmp(opt.order_type, "GLOBAL"))
strcpy (opt.order_type, "DIST");
if (!strcmp(opt.order_type, "LOCAL"))
strcpy (opt.order_type, "SERIAL");
strcpy(zz->Order.order_type, opt.order_type);
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
* Compute gid according to the local graph.
*/
if (zz->Get_Num_Obj != NULL) {
local_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Get_Num_Obj.");
return (ierr);
}
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_OBJ_FN.");
return (ZOLTAN_FATAL);
}
local_gids = ZOLTAN_MALLOC_GID_ARRAY(zz, local_num_obj);
local_rank = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
local_iperm = (int*) ZOLTAN_MALLOC(local_num_obj*sizeof(int));
lids = ZOLTAN_MALLOC_LID_ARRAY(zz, local_num_obj);
ierr = (*Order_fn)(zz, local_num_obj, local_gids, lids, local_rank, local_iperm, &opt);
ZOLTAN_FREE(&lids);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
Zoltan_Multifree(__FILE__, __LINE__, 3,
&local_gids, &local_rank, &local_iperm);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
if ((!(opt.return_args & RETURN_RANK) && (rank != NULL))
|| (!(opt.return_args & RETURN_IPERM) && (iperm != NULL))) {
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr) {
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK) && (rank != NULL)) {
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_iperm, local_rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM) && (iperm != NULL)) {
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, local_rank, local_iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Invert Permutation");
/* TODO: Use directly the "graph" structure to avoid to duplicate things. */
/* I store : GNO, rank, iperm */
ierr = Zoltan_DD_Create (&dd, zz->Communicator, zz->Num_GID, (local_rank==NULL)?0:1, (local_iperm==NULL)?0:1, local_num_obj, 0);
/* Hope a linear assignment will help a little */
Zoltan_DD_Set_Neighbor_Hash_Fn1(dd, local_num_obj);
/* Associate all the data with our xGNO */
Zoltan_DD_Update (dd, local_gids, (ZOLTAN_ID_PTR)local_rank, (ZOLTAN_ID_PTR) local_iperm, NULL, local_num_obj);
ZOLTAN_FREE(&local_gids);
ZOLTAN_FREE(&local_rank);
ZOLTAN_FREE(&local_iperm);
Zoltan_DD_Find (dd, gids, (ZOLTAN_ID_PTR)rank, (ZOLTAN_ID_PTR)iperm, NULL,
num_obj, NULL);
Zoltan_DD_Destroy(&dd);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done Registering results");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < num_obj; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Check_Param(
const char *name, /* name of parameter being reset */
const char *val, /* new value for parameter */
PARAM_VARS * params, /* structure describing parameters */
PARAM_UTYPE *result, /* pointer to return value */
int *matched_index) /* where in struct the match occurs */
{
char *yo = "Zoltan_Check_Param";
char msg[256];
int i; /* loop counter */
int status; /* return code: */
/* 0 => name found and value OK */
/* 1 => name not found */
/* 2 => name found, but value bad */
status = 1;
i = 0;
while (params->name != NULL) {
if (!strcmp(params->name, name)) {
status = 0;
break;
}
i++;
params++;
}
if (status == 0) { /* name found */
*matched_index = i;
if (!strcmp(val, "DEFAULT")){
result->def = 1;
}
else {
/* Figure out what type it is and read value. */
result->def = 0;
if (!strcmp(params->type, "INT") || !strcmp(params->type, "INTEGER")) {
/* First special case if True or False */
if (*val == 'T')
(*result).ival = 1;
else if (*val == 'F')
(*result).ival = 0;
else {
/* Check that there's a digit here */
for (i = strlen(val); i >= 0; i--)
if (isdigit((int)(val[i])))
break;
if (i < 0)
status = 2;
else {
(*result).ival = atoi(val);
}
}
}
else if ((!strcmp(params->type, "FLOAT")) ||
(!strcmp(params->type, "REAL")) ||
(!strcmp(params->type, "DOUBLE"))) {
/* Check that there's a digit here */
for (i = strlen(val); i >= 0; i--)
if (isdigit((int)(val[i])))
break;
if (i < 0)
status = 2;
else {
(*result).fval = atof(val);
(*result).dval = atof(val);
}
}
else if (!strcmp(params->type, "LONG")) {
/* First special case if True or False */
if (*val == 'T')
(*result).lval = 1;
else if (*val == 'F')
(*result).lval = 0;
else {
/* Check that there's a digit here */
for (i = strlen(val); i >= 0; i--)
if (isdigit((int)(val[i])))
break;
if (i < 0)
status = 2;
else {
(*result).lval = atol(val);
}
}
}
else if (!strcmp(params->type, "STRING")) {
strncpy((*result).sval, val, MAX_PARAM_STRING_LEN);
}
else if (!strcmp(params->type, "CHAR")) {
(*result).cval = *val;
}
else {
sprintf(msg, "Bad type for parameter `%s'",
params->name);
ZOLTAN_PRINT_WARN(-1, yo, msg);
status = 2;
}
}
}
else { /* name not matched */
*matched_index = -1;
}
return (status);
}
static int rib_fn(
ZZ *zz, /* The Zoltan structure with info for
the RIB balancer. */
int *num_import, /* Number of non-local objects assigned to
this processor in the new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
num_import returns the number of input
objects as given by ZOLTAN_NUM_OBJ_FN. */
ZOLTAN_ID_PTR *import_global_ids, /* Returned value: array of global IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains GIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
ZOLTAN_ID_PTR *import_local_ids, /* Returned value: array of local IDs for
non-local objects in this processor's new
decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
this array contains LIDs for all input
objs as given by ZOLTAN_OBJ_LIST_FN.*/
int **import_procs, /* Returned value: array of processor IDs for
processors owning the non-local objects in
this processor's new decomposition.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int **import_to_part, /* Returned value: array of parts to
which objects are imported.
When LB.Return_Lists==CANDIDATE_LISTS,
the returned array is NULL. */
int *num_export, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; number of
input objs as given by ZOLTAN_NUM_OBJ_FN */
ZOLTAN_ID_PTR *export_global_ids, /* Returned value only when
LB.Return_Lists==CANDIDATE_LISTS; for each
input obj (from ZOLTAN_OBJ_LIST_FN),
return a candidate obj from the part to which
the obj is assigned; used in PHG matching */
double overalloc, /* amount to overallocate by when realloc
of dot array must be done.
1.0 = no extra; 1.5 = 50% extra; etc. */
int wgtflag, /* No. of weights per dot supplied by user. */
int check_geom, /* Check input & output for consistency? */
int stats, /* Print timing & count summary? */
int gen_tree, /* (0) do not (1) do generate full treept */
int average_cuts, /* (0) don't (1) compute the cut to be the
average of the closest dots. */
float *part_sizes /* Input: Array of size
zz->Num_Global_Parts * max(zz->Obj_Weight_Dim, 1)
containing the percentage of work to be
assigned to each part. */
)
{
char yo[] = "rib_fn";
int proc,nprocs; /* my proc id, total # of procs */
struct Dot_Struct *dotpt; /* temporary pointer to local dot arrays */
int pdotnum; /* # of dots - decomposition changes it */
int *dotmark = NULL; /* which side of median for each dot */
int dotnum; /* number of dots */
int dotmax = 0; /* max # of dots arrays can hold */
int dottop; /* dots >= this index are new */
int proclower; /* 1st proc in lower set */
int procmid; /* 1st proc in upper set */
int partlower; /* 1st part in lower set */
int partmid; /* 1st part in upper set */
int set; /* which set processor is in = 0/1 */
int old_set; /* set processor was in last cut = 0/1 */
int root; /* part that stores last cut */
int num_procs; /* number of procs in current set */
int num_parts; /* number of parts in current set */
int ierr = ZOLTAN_OK; /* error flag. */
double *value = NULL; /* temp array for median_find */
double *wgts = NULL; /* temp array for serial_rib */
double valuehalf; /* median cut position */
double cm[3]; /* Center of mass of objects */
double evec[3]; /* Eigenvector defining direction */
int first_guess = 0; /* flag if first guess for median search */
int allocflag; /* have to re-allocate space */
double time1=0,time2=0; /* timers */
double time3=0,time4=0; /* timers */
double timestart=0,timestop=0; /* timers */
double timers[4]={0.,0.,0.,0.};
/* diagnostic timers
0 = start-up time before recursion
1 = time before median iterations
2 = time in median iterations
3 = communication time */
ZOLTAN_GNO_TYPE counters[7]; /* diagnostic counts
0 = unused
1 = # of dots sent
2 = # of dots received
3 = most dots this proc ever owns
4 = most dot memory this proc ever allocs
5 = # of times a previous cut is re-used
6 = # of reallocs of dot array */
int i, j; /* local variables */
int use_ids; /* When true, global and local IDs will be
stored along with dots in the RCB_STRUCT.
When false, storage, manipulation, and
communication of IDs is avoided.
Set by call to Zoltan_RB_Use_IDs(). */
RIB_STRUCT *rib = NULL; /* Pointer to data structures for RIB */
struct rib_tree *treept = NULL; /* tree of cuts - single cut on exit*/
double start_time, end_time;
double lb_time[2]={0,0};
int tfs[2], tmp_tfs[2]; /* added for Tflops_Special; max number
of procs and parts over all processors
in each iteration (while loop) of
parallel partitioning. */
int old_nprocs; /* added for Tflops_Special */
int old_nparts; /* added for Tflops_Special */
double valuelo; /* smallest value of value[i] */
double valuehi; /* largest value of value[i] */
double weight[RB_MAX_WGTS]; /* weight for current set */
double weightlo[RB_MAX_WGTS]; /* weight of lower side of cut */
double weighthi[RB_MAX_WGTS]; /* weight of upper side of cut */
double fractionlo[RB_MAX_WGTS]; /* desired wt in lower half */
int *dotlist = NULL; /* list of dots for find_median.
allocated above find_median for
better efficiency (don't necessarily
have to realloc for each find_median).*/
int rectilinear_blocks = 0; /* parameter for find_median (not used by rib) */
int fp=0; /* first part assigned to this proc. */
int np=0; /* number of parts assigned to this proc. */
int wgtdim; /* max(wgtflag,1) */
int *dindx = NULL, *tmpdindx = NULL;
/* MPI data types and user functions */
MPI_Comm local_comm, tmp_comm;
int free_comm = FALSE; /* Flag indicating whether MPI_Comm_free
should be called on local_comm at end. */
ZOLTAN_TRACE_ENTER(zz, yo);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestart = time1 = Zoltan_Time(zz->Timer);
}
/* setup for parallel */
proc = zz->Proc;
nprocs = zz->Num_Proc;
num_parts = zz->LB.Num_Global_Parts;
/*
* Determine whether to store, manipulate, and communicate global and
* local IDs.
*/
use_ids = Zoltan_RB_Use_IDs(zz);
/*
* Build the RIB Data structure and
* set pointers to information in it.
*/
start_time = Zoltan_Time(zz->Timer);
ierr = Zoltan_RIB_Build_Structure(zz, &pdotnum, &dotmax, wgtflag, overalloc,
use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RIB_Build_Structure.");
goto End;
}
rib = (RIB_STRUCT *) (zz->LB.Data_Structure);
treept = rib->Tree_Ptr;
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
start_time = end_time;
/* local copies of calling parameters */
dottop = dotnum = pdotnum;
/* initialize timers and counters */
counters[0] = 0;
counters[1] = 0;
counters[2] = 0;
counters[3] = dotnum;
counters[4] = dotmax;
counters[5] = 0;
counters[6] = 0;
/* Ensure there are dots */
MPI_Allreduce(&dotnum, &i, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (i == 0){
if (proc == 0){
ZOLTAN_PRINT_WARN(proc, yo, "RIB partitioning called with no objects");
}
timestart = timestop = 0;
goto EndReporting;
}
/* If using RIB for matching, need to generate candidate lists.
* Candidate lists include input GIDs, LIDs as provided by the application.
* We need to capture that input here before we move any dots!
* We return it in the import lists.
* Candidates will be computed after partitioning and returned in the
* export lists.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
ierr = Zoltan_RB_Candidates_Copy_Input(zz, dotnum,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
/* create mark and list arrays for dots */
allocflag = 0;
if (dotmax > 0) {
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
else {
dotmark = NULL;
value = NULL;
dotlist = NULL;
}
/* set dot weights = 1.0 if user didn't and determine total weight */
dotpt = &rib->Dots;
if (dotpt->nWeights == 0) {
weightlo[0] = (double) dotnum;
dotpt->uniformWeight = 1.0;
wgtdim = 1;
}
else {
double *wgt;
for (j=0; j<dotpt->nWeights; j++){
weightlo[j] = 0.0;
wgt = dotpt->Weight + j;
for (i=0; i < dotnum; i++){
weightlo[j] += *wgt;
wgt += dotpt->nWeights;
}
}
wgtdim = dotpt->nWeights;
}
MPI_Allreduce(weightlo, weight, wgtdim, MPI_DOUBLE, MPI_SUM, zz->Communicator);
if (check_geom) {
ierr = Zoltan_RB_check_geom_input(zz, dotpt, dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_input");
goto End;
}
}
/* create local communicator for use in recursion */
if (zz->Tflops_Special)
local_comm = zz->Communicator;
else {
MPI_Comm_dup(zz->Communicator,&local_comm);
free_comm = TRUE;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time2 = Zoltan_Time(zz->Timer);
timers[0] = time2 - time1;
}
/* recursively halve until just one part or proc in set */
old_nprocs = num_procs = nprocs;
old_nparts = num_parts;
partlower = 0;
root = 0;
old_set = 1;
ierr = Zoltan_LB_Proc_To_Part(zz, proc, &np, &fp);
for (i = fp; i < (fp + np); i++) {
treept[i].parent = 0;
treept[i].left_leaf = 0;
}
if (zz->Tflops_Special) {
proclower = 0;
tfs[0] = nprocs;
tfs[1] = num_parts;
}
while ((num_parts > 1 && num_procs > 1) ||
(zz->Tflops_Special && tfs[0] > 1 && tfs[1] > 1)) {
ierr = Zoltan_Divide_Machine(zz, zz->Obj_Weight_Dim, part_sizes,
proc, local_comm, &set,
&proclower, &procmid, &num_procs,
&partlower, &partmid, &num_parts,
fractionlo);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error in Zoltan_Divide_Machine.");
goto End;
}
/* tfs[0] is max number of processors in all sets over all processors -
* tfs[1] is max number of parts in all sets over all processors -
* force all processors to go through all levels of parallel rib */
if (zz->Tflops_Special) {
tmp_tfs[0] = num_procs;
tmp_tfs[1] = num_parts;
MPI_Allreduce(tmp_tfs, tfs, 2, MPI_INT, MPI_MAX, local_comm);
}
/* create mark array and active list for dots */
if (allocflag) {
allocflag = 0;
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ierr = ZOLTAN_MEMERR;
goto End;
}
}
dotpt = &rib->Dots;
if (old_nparts > 1 && old_nprocs > 1) { /* test added for Tflops_Special;
compute values only if looping
to decompose, not if looping to
keep Tflops_Special happy. */
ierr = compute_rib_direction(zz, zz->Tflops_Special, rib->Num_Geom,
&valuelo, &valuehi, dotpt, NULL, dotnum,
wgtflag, cm, evec, value,
local_comm, proc, old_nprocs, proclower);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from compute_rib_direction");
goto End;
}
}
else { /* For Tflops_Special: initialize value when looping only
for Tflops_Special */
for (i = 0; i < dotmax; i++)
value[i] = 0.0;
valuelo = valuehi = 0.0;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time2 = Zoltan_Time(zz->Timer);
if (!Zoltan_RB_find_median(
zz->Tflops_Special, value, dotpt->Weight, dotpt->uniformWeight, dotmark, dotnum, proc,
fractionlo, local_comm, &valuehalf, first_guess,
nprocs, old_nprocs, proclower, old_nparts,
wgtflag, valuelo, valuehi, weight[0], weightlo,
weighthi, dotlist, rectilinear_blocks, average_cuts)) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_find_median.");
ierr = ZOLTAN_FATAL;
goto End;
}
if (set) /* set weight for current part */
for (j=0; j<wgtdim; j++) weight[j] = weighthi[j];
else
for (j=0; j<wgtdim; j++) weight[j] = weightlo[j];
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
time3 = Zoltan_Time(zz->Timer);
/* store cut info in tree only if proc "owns" partmid */
/* test of partmid > 0 prevents treept[0] being set when this cut is
only removing low-numbered processors (proclower to procmid-1) that
have no parts in them from the processors remaining to
be partitioned. */
if (partmid > 0 && partmid == fp) {
treept[partmid].cm[0] = cm[0];
treept[partmid].cm[1] = cm[1];
treept[partmid].cm[2] = cm[2];
treept[partmid].ev[0] = evec[0];
treept[partmid].ev[1] = evec[1];
treept[partmid].ev[2] = evec[2];
treept[partmid].cut = valuehalf;
treept[partmid].parent = old_set ? -(root+1) : root+1;
/* The following two will get overwritten when the information
is assembled if this is not a terminal cut */
treept[partmid].left_leaf = -partlower;
treept[partmid].right_leaf = -partmid;
}
if (old_nprocs > 1 && partmid > 0 && partmid != partlower + old_nparts) {
/* old_nprocs > 1 test: Don't reset these values if proc is in loop only
* because of other procs for Tflops_Special.
* partmid > 0 test: Don't reset these values if low-numbered processors
* (proclower to procmid-1) have zero parts and this cut is removing
* them from the processors remaining to be partitioned.
* partmid != partlower + old_nparts test: Don't reset these values if
* cut is removing high-numbered processors with zero parts from
* the processors remaining to be partitioned.
*/
old_set = set;
root = partmid;
}
ierr = Zoltan_RB_Send_Outgoing(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark,
&dottop, &dotnum, &dotmax,
set, &allocflag, overalloc,
stats, counters, use_ids,
local_comm, proclower,
old_nprocs, partlower, partmid);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Send_Outgoing.");
goto End;
}
/* create new communicators */
if (zz->Tflops_Special) {
if (set) {
proclower = procmid;
partlower = partmid;
}
old_nprocs = num_procs;
old_nparts = num_parts;
}
else {
if (set) partlower = partmid;
MPI_Comm_split(local_comm,set,proc,&tmp_comm);
MPI_Comm_free(&local_comm);
local_comm = tmp_comm;
old_nprocs = num_procs;
old_nparts = num_parts;
}
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
time4 = Zoltan_Time(zz->Timer);
timers[1] += time2 - time1;
timers[2] += time3 - time2;
timers[3] += time4 - time3;
}
}
/* have recursed all the way to a single processor sub-domain */
/* Send dots to correct processors for their parts. This is needed
most notably when a processor has zero parts on it, but still has
some dots after the parallel partitioning. */
ierr = Zoltan_RB_Send_To_Part(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotmark, &dottop,
&dotnum, &dotmax, &allocflag, overalloc,
stats, counters, use_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from Zoltan_RB_Send_To_Part");
goto End;
}
/* All dots are now on the processors they will end up on; now generate
* more parts if needed. */
if (num_parts > 1) {
if (dotpt->nWeights)
wgts = (double *) ZOLTAN_MALLOC(dotpt->nWeights * dotnum * sizeof(double));
dindx = (int *) ZOLTAN_MALLOC(dotnum * 2 * sizeof(int));
tmpdindx = dindx + dotnum;
if (allocflag) {
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!(dotmark = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))
|| !(value = (double *) ZOLTAN_MALLOC(dotmax*sizeof(double)))
|| !(dotlist = (int *) ZOLTAN_MALLOC(dotmax*sizeof(int)))) {
ZOLTAN_PRINT_ERROR(proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
}
for (i = 0; i < dotnum; i++)
dindx[i] = i;
ierr = serial_rib(zz, &rib->Dots, dotmark, dotlist, old_set, root,
rib->Num_Geom, weight[0], dotnum, num_parts,
&(dindx[0]), &(tmpdindx[0]), partlower,
proc, wgtflag, stats, gen_tree,
rectilinear_blocks, average_cuts,
treept, value, wgts, part_sizes);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo, "Error returned from serial_rib");
goto End;
}
ZOLTAN_FREE(&wgts);
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME)) {
MPI_Barrier(zz->Communicator);
timestop = time1 = Zoltan_Time(zz->Timer);
}
/* error checking and statistics */
if (check_geom) {
ierr = Zoltan_RB_check_geom_output(zz, &rib->Dots, part_sizes, np, fp,
dotnum, pdotnum, NULL);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_check_geom_output");
goto End;
}
}
EndReporting:
/* update calling routine parameters */
start_time = Zoltan_Time(zz->Timer);
pdotnum = dotnum;
/* Perform remapping (if requested) */
if (zz->LB.Remap_Flag) {
ierr = Zoltan_RB_Remap(zz, &(rib->Global_IDs), &(rib->Local_IDs),
&(rib->Dots), &dotnum, &dotmax,
&allocflag, overalloc, stats, counters, use_ids);
/* Note: dottop is no longer valid after remapping. Remapping might
destroy the nice local-followed-by-non-local ordering of the
dots array. Do not use dottop after remapping. */
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_RB_Remap.");
goto End;
}
}
/* build return arguments */
if (zz->LB.Return_Lists != ZOLTAN_LB_NO_LISTS &&
zz->LB.Return_Lists != ZOLTAN_LB_CANDIDATE_LISTS) {
/* zz->LB.Return_Lists is true ==> use_ids is true */
ierr = Zoltan_RB_Return_Arguments(zz, rib->Global_IDs, rib->Local_IDs,
&rib->Dots, num_import,
import_global_ids, import_local_ids,
import_procs, import_to_part,
dotnum);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc, yo,
"Error returned from Zoltan_RB_Return_Arguments.");
goto End;
}
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
/* Select a candidate for each part and return it in the export_GIDs. */
ierr = Zoltan_RB_Candidates_Output(zz, dotnum, dindx,
rib->Global_IDs, rib->Local_IDs,
&rib->Dots,
*num_import, *import_global_ids,
num_export, export_global_ids);
if (ierr < 0) {
ZOLTAN_PRINT_ERROR(proc,yo,
"Error returned from Zoltan_RB_Return_Candidates.");
goto End;
}
}
ZOLTAN_FREE(&dindx);
if (gen_tree) {
int *displ, *recvcount;
int sendcount;
struct rib_tree *treetmp = NULL; /* temporary tree of cuts; used to keep
valgrind from reporting overlapped
memory in MPI_Allgatherv */
treetmp = (struct rib_tree *)
ZOLTAN_MALLOC(zz->LB.Num_Global_Parts* sizeof(struct rib_tree));
displ = (int *) ZOLTAN_MALLOC(2 * zz->Num_Proc * sizeof(int));
if (!displ || !treetmp) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
ierr = ZOLTAN_MEMERR;
goto End;
}
recvcount = displ + zz->Num_Proc;
ierr = Zoltan_RB_Tree_Gatherv(zz, sizeof(struct rib_tree), &sendcount,
recvcount, displ);
/*
* Create copy of treept so that MPI_Allgatherv doesn't use same
* memory for sending and receiving; removes valgrind warning.
*/
for (i = 0; i < zz->LB.Num_Global_Parts; i++)
treetmp[i] = treept[i];
MPI_Allgatherv(&treetmp[fp], sendcount, MPI_BYTE, treept, recvcount, displ,
MPI_BYTE, zz->Communicator);
for (i = 1; i < zz->LB.Num_Global_Parts; i++){
if (treept[i].parent > 0)
treept[treept[i].parent - 1].left_leaf = i;
else if (treept[i].parent < 0)
treept[-treept[i].parent - 1].right_leaf = i;
}
ZOLTAN_FREE(&displ);
ZOLTAN_FREE(&treetmp);
}
else {
treept[0].right_leaf = -1;
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
print_rib_tree(zz, np, fp, &(treept[fp]));
end_time = Zoltan_Time(zz->Timer);
lb_time[0] += (end_time - start_time);
if (stats || (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME))
Zoltan_RB_stats(zz, timestop-timestart, &rib->Dots, dotnum,
part_sizes, timers, counters, stats, NULL, NULL, FALSE);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME) {
if (zz->Proc == zz->Debug_Proc)
printf("ZOLTAN RIB Times: \n");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Build: ");
Zoltan_Print_Stats(zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN RIB: ");
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
/* zz->Debug_Level >= ZOLTAN_DEBUG_ALL ==> use_ids is true */
Zoltan_RB_Print_All(zz, rib->Global_IDs, &rib->Dots,
dotnum, *num_import,
*import_global_ids, *import_procs);
}
End:
/* Free memory allocated by the algorithm. */
if (free_comm) MPI_Comm_free(&local_comm);
ZOLTAN_FREE(&dotmark);
ZOLTAN_FREE(&value);
ZOLTAN_FREE(&dotlist);
if (!gen_tree && /* don't need parts */
rib && (rib->Tran.Target_Dim < 0)) { /* don't need transformation */
/* Free all memory used. */
Zoltan_RIB_Free_Structure(zz);
}
else if (rib != NULL) {
/* Free only Dots and IDs; keep other structures. */
ZOLTAN_FREE(&(rib->Global_IDs));
ZOLTAN_FREE(&(rib->Local_IDs));
Zoltan_Free_And_Reset_Dot_Structure(&(rib->Dots));
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
int Zoltan_ParMetis(
ZZ *zz, /* Zoltan structure */
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
containing the percentage of work to be
assigned to each partition. */
int *num_imp, /* number of objects to be imported */
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
int **imp_procs, /* list of processors to import from */
int **imp_to_part, /* list of partitions to which imported objects are
assigned. */
int *num_exp, /* number of objects to be exported */
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
int **exp_procs, /* list of processors to export to */
int **exp_to_part /* list of partitions to which exported objects are
assigned. */
)
{
char *yo = "Zoltan_ParMetis";
int ierr;
ZOLTAN_Third_Graph gr;
ZOLTAN_Third_Geom *geo = NULL;
ZOLTAN_Third_Vsize vsp;
ZOLTAN_Third_Part prt;
ZOLTAN_Output_Part part;
ZOLTAN_ID_PTR global_ids = NULL;
ZOLTAN_ID_PTR local_ids = NULL;
int use_timers = 0;
int timer_p = -1;
int get_times = 0;
double times[5];
double pmv3_itr = 0.0;
float itr = 0.0;
int options[MAX_OPTIONS];
char alg[MAX_PARAM_STRING_LEN+1];
int i;
float *imb_tols;
int ncon;
int edgecut;
int wgtflag;
int numflag = 0;
int num_part = zz->LB.Num_Global_Parts;/* passed to Jostle/ParMETIS. Don't */
MPI_Comm comm = zz->Communicator;/* want to risk letting external packages */
/* change our zz struct. */
#ifndef ZOLTAN_PARMETIS
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"ParMetis requested but not compiled into library.");
return ZOLTAN_FATAL;
#endif /* ZOLTAN_PARMETIS */
ZOLTAN_TRACE_ENTER(zz, yo);
#ifdef ZOLTAN_PARMETIS
/* Check for outdated/unsupported ParMetis versions. */
#if (PARMETIS_MAJOR_VERSION == 3) && (PARMETIS_MINOR_VERSION == 0)
if (zz->Proc == 0)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "ParMetis 3.0 is no longer supported by Zoltan. Please upgrade to ParMetis 3.1 (or later).");
ierr = ZOLTAN_WARN;
#endif
#endif /* ZOLTAN_PARMETIS */
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
imp_gids, imp_lids, imp_procs, imp_to_part,
exp_gids, exp_lids, exp_procs, exp_to_part);
prt.input_part_sizes = prt.part_sizes = part_sizes;
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
gr.graph_type = 0;
#ifdef ZOLTAN_PARMETIS
SET_GLOBAL_GRAPH(&gr.graph_type);
/* Select type of graph, negative because we impose them */
/* TODO: add a parameter to select the type, shared with Scotch */
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
/* gr.graph_type = - LOCAL_GRAPH; */
/* if (zz->Num_Proc > 1) { */
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
/* gr.graph_type = - GLOBAL_GRAPH; */
/* retval = ZOLTAN_WARN; */
/* } */
/* } */
#else /* graph is local */
SET_LOCAL_GRAPH(&gr.graph_type);
#endif /* ZOLTAN_PARMETIS */
/* Some algorithms use geometry data */
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
/* ParMETIS will crash if geometric method and some procs have no nodes. */
/* Avoid fatal crash by setting scatter to level 2 or higher. */
gr.scatter_min = 2;
if (geo == NULL) {
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
return (ZOLTAN_MEMERR);
}
if (strcmp(alg, "PARTGEOM") == 0) {
gr.get_data = 0;
}
}
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
/* Start timer */
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
if (get_times){
MPI_Barrier(zz->Communicator);
times[0] = Zoltan_Time(zz->Timer);
}
vsp.vsize_malloc = 0;
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
/* ParMETIS will free this memory; use malloc to allocate so
ZOLTAN_MALLOC counters don't show an error. */
vsp.vsize_malloc = 1 ;
}
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr, geo, &prt, &vsp);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
ierr = Zoltan_Parmetis_Check_Error(zz, alg, &gr, &prt);
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
return (ierr);
}
/* Get object sizes if requested */
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
gr.showMoveVol = 1;
/* Get a time here */
if (get_times) times[1] = Zoltan_Time(zz->Timer);
/* Get ready to call ParMETIS or Jostle */
edgecut = -1;
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
numflag = 0;
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
if (!prt.part_sizes){
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
}
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
for (i=0; i<num_part; i++){
int j;
printf("Debug: Size(s) for part %1d = ", i);
for (j=0; j<ncon; j++)
printf("%f ", prt.part_sizes[i*ncon+j]);
printf("\n");
}
}
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
for (i = 0; i < num_part*ncon; i++)
if (prt.part_sizes[i] == 0)
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
"ParMETIS 3.x will likely fail. Please use a "
"different method, or remove the zero-sized "
"parts from the problem.");
/* Set Imbalance Tolerance for each weight component. */
imb_tols = (float *) ZOLTAN_MALLOC(ncon * sizeof(float));
if (!imb_tols){
/* Not enough memory */
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
for (i=0; i<ncon; i++)
imb_tols[i] = zz->LB.Imbalance_Tol[i];
/* Now we can call ParMetis */
#ifdef ZOLTAN_PARMETIS
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
/* First check for ParMetis 3 routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS 3 library ParMETIS_V3_PartKway");
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS 3 library ParMETIS_V3_PartGeomKway");
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
&wgtflag, &numflag, &geo->ndims, geo->xyz, &ncon,
&num_part, prt.part_sizes,
imb_tols, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "PARTGEOM") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS 3 library ParMETIS_V3_PartGeom");
ParMETIS_V3_PartGeom(gr.vtxdist, &geo->ndims, geo->xyz, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS 3 library ParMETIS_V3_AdaptiveRepart");
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
&num_part, prt.part_sizes, imb_tols,
&itr, options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else if (strcmp(alg, "REFINEKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS 3 library ParMETIS_V3_RefineKway");
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
&wgtflag, &numflag, &ncon, &num_part,
prt.part_sizes, imb_tols,
options, &edgecut, prt.part, &comm);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_PARMETIS */
#ifdef ZOLTAN_METIS
/* TODO: I don't know how to set balance ! */
if (IS_LOCAL_GRAPH(gr.graph_type)) {
/* Check for Metis routines */
if (strcmp(alg, "PARTKWAY") == 0){
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS 4 library METIS_WPartGraphKway");
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
gr.vwgt, gr.ewgts, &wgtflag,
&numflag, &num_part, prt.part_sizes,
options, &edgecut, prt.part);
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
}
else {
/* Sanity check: This should never happen! */
char msg[256];
sprintf(msg, "Unknown Metis algorithm %s.", alg);
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
}
}
#endif /* ZOLTAN_METIS */
/* Get a time here */
if (get_times) times[2] = Zoltan_Time(zz->Timer);
if (gr.final_output) { /* Do final output now because after the data will not be coherent:
unscatter only unscatter part data, not graph */
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp,
use_timers, itr);
}
/* Ignore the timings of Final Ouput */
if (get_times) times[3] = Zoltan_Time(zz->Timer);
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr, geo, &prt, &vsp, NULL, &part);
Zoltan_Third_Export_User(&part, num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
/* Get a time here */
if (get_times) times[4] = Zoltan_Time(zz->Timer);
if (get_times) Zoltan_Third_DisplayTime(zz, times);
if (use_timers && timer_p >= 0)
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
if (geo != NULL) ZOLTAN_FREE(&geo);
/* KDD ALREADY FREED BY Zoltan_Third_Exit ZOLTAN_FREE(&global_ids); */
ZOLTAN_FREE(&local_ids);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
int Zoltan_Preprocess_Graph(
ZZ *zz, /* Zoltan structure */
ZOLTAN_ID_PTR *global_ids,
ZOLTAN_ID_PTR *local_ids,
ZOLTAN_Third_Graph *gr, /* Graph for third part libs */
ZOLTAN_Third_Geom *geo,
ZOLTAN_Third_Part *prt,
ZOLTAN_Third_Vsize *vsp
)
{
static char *yo = "Zoltan_Preprocess_Graph";
int ierr;
float *float_vwgt, *float_ewgts;
char msg[256];
char add_obj_weight[MAX_PARAM_STRING_LEN+1];
ZOLTAN_TRACE_ENTER(zz, yo);
/* Initialize all local pointers to NULL. This is necessary
* because we free all non-NULL pointers upon errors.
*/
gr->vtxdist = gr->xadj = gr->adjncy = NULL;
gr->vwgt = gr->ewgts = NULL;
float_vwgt = float_ewgts = NULL;
if (gr->obj_wgt_dim >= 0) {
/* Check weight dimensions */
if (zz->Obj_Weight_Dim<0){
sprintf(msg, "Object weight dimension is %d, "
"but should be >= 0. Using Obj_Weight_Dim = 0.",
zz->Obj_Weight_Dim);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
gr->obj_wgt_dim = 0;
}
else {
gr->obj_wgt_dim = zz->Obj_Weight_Dim;
}
}
else
gr->obj_wgt_dim = 0;
if (gr->edge_wgt_dim >= 0) {
if (zz->Edge_Weight_Dim<0){
sprintf(msg, "Edge weight dimension is %d, "
"but should be >= 0. Using Edge_Weight_Dim = 0.",
zz->Edge_Weight_Dim);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
gr->edge_wgt_dim = 0;
}
else if (zz->Edge_Weight_Dim>1){
ZOLTAN_PRINT_WARN(zz->Proc, yo, "This method does not support "
"multidimensional edge weights. Using Edge_Weight_Dim = 1.");
gr->edge_wgt_dim = 1;
}
else {
gr->edge_wgt_dim = zz->Edge_Weight_Dim;
}
}
else
gr->edge_wgt_dim = 0;
if (gr->graph_type >= 0)
/* Default graph type is GLOBAL. */
gr->graph_type = GLOBAL_GRAPH;
else
gr->graph_type = - gr->graph_type;
/* Get parameter options shared by ParMetis and Jostle */
gr->check_graph = 1; /* default */
gr->scatter = 1; /* default */
gr->final_output = 0;
strcpy(add_obj_weight, "NONE"); /* default */
Zoltan_Bind_Param(Graph_params, "CHECK_GRAPH", (void *) &gr->check_graph);
Zoltan_Bind_Param(Graph_params, "SCATTER_GRAPH", (void *) &gr->scatter);
Zoltan_Bind_Param(Graph_params, "FINAL_OUTPUT", (void *) &gr->final_output);
Zoltan_Bind_Param(Graph_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight);
Zoltan_Assign_Param_Vals(zz->Params, Graph_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
/* If reorder is true, we already have the id lists. Ignore weights. */
if ((*global_ids == NULL) || (!gr->id_known)){
int * input_part;
ierr = Zoltan_Get_Obj_List(zz, &gr->num_obj, global_ids, local_ids,
gr->obj_wgt_dim, &float_vwgt, &input_part);
if (prt) {
prt->input_part = input_part;
}
else { /* Ordering, dont need part */
ZOLTAN_FREE(&input_part);
}
if (ierr){
/* Return error */
ZOLTAN_PARMETIS_ERROR(ierr, "Get_Obj_List returned error.");
}
}
/* Build Graph for third party library data structures, or just get vtxdist. */
ierr = Zoltan_Build_Graph(zz, gr->graph_type, gr->check_graph, gr->num_obj,
*global_ids, *local_ids, gr->obj_wgt_dim, gr->edge_wgt_dim,
&gr->vtxdist, &gr->xadj, &gr->adjncy, &float_ewgts, &gr->adjproc);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
ZOLTAN_PARMETIS_ERROR(ierr, "Zoltan_Build_Graph returned error.");
}
if (prt) {
prt->part_sizes = prt->input_part_sizes;
if (gr->num_obj >0) {
prt->part = (indextype *)ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
if (!prt->part){
/* Not enough memory */
ZOLTAN_PARMETIS_ERROR(ZOLTAN_MEMERR, "Out of memory.");
}
memcpy (prt->part, prt->input_part, (gr->num_obj) * sizeof(indextype));
}
else {
prt->input_part = prt->part = NULL;
}
}
/* Convert from float. */
/* Get vertex weights if needed */
if (gr->obj_wgt_dim){
ierr = Zoltan_Preprocess_Scale_Weights (gr, float_vwgt, &gr->vwgt,
gr->num_obj, gr->obj_wgt_dim, 1, zz,
"vertex", gr->vtxdist[zz->Proc]);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
/* Return error code */
ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
}
ZOLTAN_FREE(&float_vwgt);
}
if (strcasecmp(add_obj_weight, "NONE")){
if (Zoltan_Preprocess_Add_Weight(zz, gr, prt, add_obj_weight) != ZOLTAN_OK)
ZOLTAN_PARMETIS_ERROR(ierr, "Error in adding vertex weights.");
}
/* Get edge weights if needed */
if (gr->get_data)
gr->num_edges = gr->xadj[gr->num_obj];
else {
gr->num_edges = 0;
gr->edge_wgt_dim = 0;
}
if (gr->edge_wgt_dim){
ierr = Zoltan_Preprocess_Scale_Weights (gr, float_ewgts, &gr->ewgts,
gr->num_edges, gr->edge_wgt_dim, 1, zz,
"edge", 0);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN){
/* Return error code */
ZOLTAN_PARMETIS_ERROR(ierr, "Error in scaling of weights.");
}
if (!gr->final_output)
ZOLTAN_FREE(&float_ewgts);
else
gr->float_ewgts = float_ewgts;
}
else
ZOLTAN_FREE(&float_ewgts);
if (geo){
ierr = Zoltan_Preprocess_Extract_Geom (zz, global_ids, local_ids, gr, geo);
if (ierr) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Extract_Geom");
}
}
if (vsp) {
ierr = Zoltan_Preprocess_Extract_Vsize (zz, global_ids, local_ids, gr, vsp);
if (ierr) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Extract_Vsize");
}
}
/* Scatter graph?
* If data distribution is highly imbalanced, it is better to
* redistribute the graph data structure before calling ParMetis.
* After partitioning, the results must be mapped back.
*/
if (gr->scatter < gr->scatter_min) gr->scatter = gr->scatter_min;
if (gr->scatter>0) {
ierr = Zoltan_Preprocess_Scatter_Graph (zz, gr, prt, geo, vsp);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PARMETIS_ERROR(ZOLTAN_FATAL,
"Error returned from Zoltan_Preprocess_Scatter_Graph");
}
}
/* Verify that graph is correct */
if (gr->get_data){
int flag;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
flag = 2; /* Maximum output level */
else
flag = 1; /* Medium output level */
ierr = Zoltan_Verify_Graph(zz->Communicator, gr->vtxdist, gr->xadj, gr->adjncy, gr->vwgt,
gr->ewgts, gr->obj_wgt_dim, gr->edge_wgt_dim, gr->graph_type, gr->check_graph, flag);
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
static int scale_round_weights(float *fwgts, indextype *iwgts, int n, int dim,
int mode, int max_wgt_sum, int debug_level, MPI_Comm comm)
{
int i, j, tmp, ierr, proc;
int *nonint, *nonint_local;
float *scale, *sum_wgt_local, *sum_wgt, *max_wgt_local, *max_wgt;
char msg[256];
static char *yo = "scale_round_weights";
ierr = ZOLTAN_OK;
MPI_Comm_rank(comm, &proc);
if (mode == 0) {
/* No scaling; just convert to int */
for (i=0; i<n*dim; i++){
iwgts[i] = (int) ceil((double) fwgts[i]);
}
}
else{
/* Allocate local arrays */
nonint = (int *)ZOLTAN_MALLOC(dim*sizeof(int));
nonint_local = (int *)ZOLTAN_MALLOC(dim*sizeof(int));
scale = (float *)ZOLTAN_MALLOC(dim*sizeof(float));
sum_wgt = (float *)ZOLTAN_MALLOC(dim*sizeof(float));
sum_wgt_local = (float *)ZOLTAN_MALLOC(dim*sizeof(float));
max_wgt = (float *)ZOLTAN_MALLOC(dim*sizeof(float));
max_wgt_local = (float *)ZOLTAN_MALLOC(dim*sizeof(float));
if (!(nonint && nonint_local && scale && sum_wgt && sum_wgt_local
&& max_wgt && max_wgt_local)){
ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
ZOLTAN_FREE(&nonint);
ZOLTAN_FREE(&nonint_local);
ZOLTAN_FREE(&scale);
ZOLTAN_FREE(&sum_wgt);
ZOLTAN_FREE(&sum_wgt_local);
ZOLTAN_FREE(&max_wgt);
ZOLTAN_FREE(&max_wgt_local);
return ZOLTAN_MEMERR;
}
/* Initialize */
for (j=0; j<dim; j++){
nonint_local[j] = 0;
sum_wgt_local[j] = 0;
max_wgt_local[j] = 0;
}
/* Compute local sums of the weights */
/* Check if all weights are integers */
for (i=0; i<n; i++){
for (j=0; j<dim; j++){
if (!nonint_local[j]){
/* tmp = (int) roundf(fwgts[i]); EB: Valid C99, but not C89 */
tmp = (int) floor((double) fwgts[i] + .5); /* Nearest int */
if (fabs((double)tmp-fwgts[i*dim+j]) > INT_EPSILON){
nonint_local[j] = 1;
}
}
sum_wgt_local[j] += fwgts[i*dim+j];
if (fwgts[i*dim+j] > max_wgt_local[j])
max_wgt_local[j] = fwgts[i*dim+j];
}
}
/* Compute global sum of the weights */
MPI_Allreduce(nonint_local, nonint, dim,
MPI_INT, MPI_LOR, comm);
MPI_Allreduce(sum_wgt_local, sum_wgt, dim,
MPI_FLOAT, MPI_SUM, comm);
MPI_Allreduce(max_wgt_local, max_wgt, dim,
MPI_FLOAT, MPI_MAX, comm);
/* Calculate scale factor */
for (j=0; j<dim; j++){
scale[j] = 1.;
/* Scale unless all weights are integers (not all zero) */
if (nonint[j] || (max_wgt[j] <= INT_EPSILON) || (sum_wgt[j] > max_wgt_sum)){
if (sum_wgt[j] == 0){
ierr = ZOLTAN_WARN;
if (proc == 0){
sprintf(msg, "All weights are zero in component %1d", j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
else /* sum_wgt[j] != 0 */
scale[j] = max_wgt_sum/sum_wgt[j];
}
}
/* If mode==2, let the scale factor be the same for all weights */
if (mode==2){
for (j=1; j<dim; j++){
if (scale[j]<scale[0])
scale[0] = scale[j];
}
for (j=1; j<dim; j++){
scale[j] = scale[0];
}
}
if ((debug_level >= ZOLTAN_DEBUG_ALL) && (proc==0)){
printf("ZOLTAN DEBUG in %s: scaling weights with scale factors = ", yo);
for (j=0; j<dim; j++)
printf("%f ", scale[j]);
printf("\n");
}
/* Convert weights to positive integers using the computed scale factor */
for (i=0; i<n; i++){
for (j=0; j<dim; j++){
iwgts[i*dim+j] = (int) ceil((double) fwgts[i*dim+j]*scale[j]);
}
}
ZOLTAN_FREE(&nonint);
ZOLTAN_FREE(&nonint_local);
ZOLTAN_FREE(&scale);
ZOLTAN_FREE(&sum_wgt);
ZOLTAN_FREE(&sum_wgt_local);
ZOLTAN_FREE(&max_wgt);
ZOLTAN_FREE(&max_wgt_local);
}
return ierr;
}
static int
Zoltan_Preprocess_Scatter_Graph (ZZ *zz,
ZOLTAN_Third_Graph *gr,
ZOLTAN_Third_Part *prt,
ZOLTAN_Third_Geom *geo,
ZOLTAN_Third_Vsize *vsp)
{
int ierr = ZOLTAN_OK;
int tmp;
if ((gr->scatter>0) && (gr->scatter<3)){
int j;
/* Decide if the data imbalance is so bad that we should scatter the graph. */
/* scatter==1: Scatter if all the objects are on a single processor. */
/* scatter==2: Scatter if any processor has no objects. */
gr->num_obj_orig = gr->num_obj; /* Save value for later. */
if (gr->num_obj==0)
j = 1;
else
j = 0;
MPI_Allreduce(&j, &tmp, 1, MPI_INT, MPI_SUM, zz->Communicator);
if (gr->scatter == 1){
if (tmp < zz->Num_Proc-1)
gr->scatter = 0;
}
else if (gr->scatter==2){
if (tmp == 0)
gr->scatter = 0;
}
}
/* We need to make sure we don't scatter the graph
* if graph_type = LOCAL_GRAPH, i.e. METIS is used.
*/
if (gr->scatter && (gr->graph_type == LOCAL_GRAPH)){
gr->scatter = 0;
ZOLTAN_PRINT_WARN(zz->Proc, __func__, "Setting scatter_graph=0 since the graph"
" is local on each proc");
ierr = ZOLTAN_WARN;
}
if (gr->scatter){
if (geo)
ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize,
&gr->ewgts, &geo->xyz, geo->ndims, gr->obj_wgt_dim, zz, &gr->comm_plan);
else {
float* xyz = NULL;
ierr = Zoltan_Scatter_Graph(&gr->vtxdist, &gr->xadj, &gr->adjncy, &gr->vwgt, &vsp->vsize,
&gr->ewgts, &xyz, 0, gr->obj_wgt_dim, zz, &gr->comm_plan);
}
if ((ierr == ZOLTAN_FATAL) || (ierr == ZOLTAN_MEMERR)){
ZOLTAN_THIRD_ERROR(ierr, "Zoltan_Scatter_Graph returned error.");
}
gr->num_obj = gr->vtxdist[zz->Proc+1]-gr->vtxdist[zz->Proc];
if (prt) {
prt->part_orig = prt->part;
prt->part = (indextype *) ZOLTAN_MALLOC((gr->num_obj+1) * sizeof(indextype));
Zoltan_Comm_Do(gr->comm_plan, TAG1, (char *) prt->part_orig, sizeof(indextype),
(char *) prt->part);
}
}
return ierr;
}
int Zoltan_Verify_Graph(MPI_Comm comm, indextype *vtxdist, indextype *xadj,
indextype *adjncy, weighttype *vwgt, weighttype *adjwgt,
int vwgt_dim, int ewgt_dim,
int graph_type, int check_graph, int output_level)
{
int ierr, flag, cross_edges = 0, mesg_size, sum;
int nprocs, proc, *proclist, errors, global_errors;
int *perm=NULL;
int free_adjncy_sort=0;
ZOLTAN_COMM_OBJ *comm_plan;
static char *yo = "Zoltan_Verify_Graph";
char msg[256];
ZOLTAN_GNO_TYPE num_obj;
int nrecv;
indextype *ptr, *ptr1, *ptr2;
indextype global_i, global_j;
indextype *sendgno=NULL, *recvgno=NULL, *adjncy_sort=NULL;
weighttype *sendwgt, *recvwgt;
ZOLTAN_GNO_TYPE num_duplicates, num_singletons;
ZOLTAN_GNO_TYPE num_selfs, nedges, global_sum, num_zeros;
ZOLTAN_GNO_TYPE i, j, ii, k;
MPI_Datatype zoltan_gno_mpi_type;
ierr = ZOLTAN_OK;
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
/* Make sure all procs have same value of check_graph. */
MPI_Allreduce(&check_graph, &i, 1, MPI_INT, MPI_MAX, comm);
check_graph = i;
if (check_graph == 0) /* perform no error checking at all */
return ierr;
/* Get number of procs and my rank */
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &proc);
/* Check number of vertices (objects) */
num_obj = (ZOLTAN_GNO_TYPE)(vtxdist[proc+1] - vtxdist[proc]);
MPI_Reduce(&num_obj, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum==0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0)
ZOLTAN_PRINT_WARN(proc, yo, "No vertices in graph.");
}
/* Verify that vertex weights are non-negative */
num_zeros = 0;
if (vwgt_dim){
for (i=0; i<num_obj; i++){
sum = 0;
for (k=0; k<vwgt_dim; k++){
if (vwgt[i*vwgt_dim+k] < 0) {
sprintf(msg, "Negative object weight of " TPL_WGT_SPEC " for object " ZOLTAN_GNO_SPEC ".",
vwgt[i*vwgt_dim+k], i);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
sum += vwgt[i*vwgt_dim+k];
}
if (sum == 0){
num_zeros++;
if (output_level>1) {
sprintf(msg, "Zero vertex (object) weights for object " ZOLTAN_GNO_SPEC ".", i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " objects have zero weights.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
/* Check number of edges */
nedges = (ZOLTAN_GNO_TYPE)xadj[num_obj];
MPI_Reduce(&nedges, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum==0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0)
ZOLTAN_PRINT_WARN(proc, yo, "No edges in graph.");
}
/* Verify that edge weights are non-negative */
num_zeros = 0;
if (ewgt_dim){
for (j=0; j<nedges; j++){
sum = 0;
for (k=0; k<ewgt_dim; k++){
if (adjwgt[j*ewgt_dim+k] < 0) {
sprintf(msg, "Negative edge weight of " TPL_WGT_SPEC " in edge " ZOLTAN_GNO_SPEC ".",
adjwgt[j*ewgt_dim+k], j);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
sum += adjwgt[j*ewgt_dim+k];
}
if (sum == 0){
num_zeros++;
if (output_level>1) {
sprintf(msg, "Zero edge (communication) weights for edge " ZOLTAN_GNO_SPEC ".", j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
MPI_Reduce(&num_zeros, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " edges have zero weights.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
/* Verify that the graph is symmetric (edge weights, too) */
/* Also check for self-edges and multi-edges */
/* Pre-processing: Check if edge lists are sorted. If not, */
/* make a copy and sort so we can save time in the lookups. */
flag = 0; /* Assume sorted. */
for (i=0; (i<num_obj) && (flag==0); i++){
for (ii=xadj[i]; ii<xadj[i+1]-1; ii++){
if (adjncy[ii] > adjncy[ii+1]){
flag = 1; /* Not sorted. */
break;
}
}
}
if (flag){ /* Need to sort. */
adjncy_sort = (indextype *) ZOLTAN_MALLOC(nedges*sizeof(indextype));
perm = (int *) ZOLTAN_MALLOC(nedges*sizeof(int));
free_adjncy_sort = 1;
if (nedges && (!adjncy_sort || !perm)){
/* Out of memory. */
ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
ierr = ZOLTAN_MEMERR;
}
for (k=0; k<nedges; k++){
adjncy_sort[k] = adjncy[k];
perm[k] = k;
}
if (sizeof(indextype) == sizeof(short)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_short((short *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(int)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_int((int *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(long)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_long((long *)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else if (sizeof(indextype) == sizeof(int64_t)){
for (i=0; i<num_obj; i++)
Zoltan_quicksort_list_inc_long_long((int64_t*)adjncy_sort, perm, (int)xadj[i], (int)xadj[i+1]-1);
}
else{
ZOLTAN_PRINT_ERROR(proc, yo, "Error in third party library data type support.");
ierr = ZOLTAN_MEMERR;
}
}
else { /* Already sorted. */
adjncy_sort = adjncy;
}
/* First pass: Check on-processor edges and count # off-proc edges */
cross_edges = 0;
num_selfs = 0;
num_duplicates = 0;
num_singletons = 0;
for (i=0; i<num_obj; i++){
if (IS_GLOBAL_GRAPH(graph_type)){
global_i = vtxdist[proc]+i;
}
else{ /* graph_type == LOCAL_GRAPH */
global_i = i; /* A bit confusingly, global_i = i for local graphs */
}
/* Singleton? */
if (xadj[i] == xadj[i+1]){
num_singletons++;
if (output_level>1){
sprintf(msg, "Vertex " TPL_IDX_SPEC " has no edges.", global_i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
for (ii=xadj[i]; ii<xadj[i+1]; ii++){
global_j = adjncy_sort[ii];
/* Valid vertex number? */
if ((IS_GLOBAL_GRAPH(graph_type) &&
((global_j < vtxdist[0]) || (global_j >= vtxdist[nprocs])))
|| (IS_LOCAL_GRAPH(graph_type) &&
((global_j < 0) || (global_j >= num_obj)))){
sprintf(msg, "Edge to invalid vertex " TPL_IDX_SPEC " detected.", global_j);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
/* Self edge? */
if (global_j == global_i){
num_selfs++;
if (output_level>1){
sprintf(msg, "Self edge for vertex " TPL_IDX_SPEC " detected.", global_i);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Duplicate edge? */
if ((ii+1<xadj[i+1]) && (adjncy_sort[ii]==adjncy_sort[ii+1])){
num_duplicates++;
if (output_level>1){
sprintf(msg, "Duplicate edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") detected.", global_i, global_j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Is global_j a local vertex? */
if (IS_LOCAL_GRAPH(graph_type) || (IS_GLOBAL_GRAPH(graph_type) &&
(global_j >= vtxdist[proc]) && (global_j < vtxdist[proc+1]))){
/* Check if (global_j, global_i) is an edge */
if (IS_GLOBAL_GRAPH(graph_type))
j = global_j - vtxdist[proc];
else /* graph_type == LOCAL_GRAPH */
j = global_j;
/* Binary search for edge (global_j, global_i) */
ptr = (indextype *)bsearch(&global_i, &adjncy_sort[xadj[j]], (int)(xadj[j+1]-xadj[j]),
sizeof(indextype), Zoltan_Compare_Indextypes);
if (ptr){
/* OK, found edge (global_j, global_i) */
if ((adjncy_sort==adjncy) && ewgt_dim){
/* Compare weights */
/* EBEB For now, don't compare weights if we sorted edge lists. */
flag = 0;
for (k=0; k<ewgt_dim; k++){
if (adjwgt[(ptr-adjncy_sort)*ewgt_dim+k] !=
adjwgt[ii*ewgt_dim+k]){
/* Numerically nonsymmetric */
flag = -1;
if (ierr == ZOLTAN_OK) ierr = ZOLTAN_WARN;
}
}
if (flag<0 && output_level>0){
sprintf(msg, "Graph is numerically nonsymmetric "
"in edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ")", global_i, global_j);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
}
else { /* bsearch failed */
sprintf(msg, "Graph is not symmetric. "
"Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but no edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").",
global_i, global_j, global_j, global_i);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
}
else {
cross_edges++;
}
}
}
/* Sum up warnings so far. */
MPI_Reduce(&num_selfs, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " self-edges in graph.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
MPI_Reduce(&num_duplicates, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " duplicate edges in graph.", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
MPI_Reduce(&num_singletons, &global_sum, 1, zoltan_gno_mpi_type, MPI_SUM, 0, comm);
if ((proc==0) && (global_sum>0)){
ierr = ZOLTAN_WARN;
if (output_level>0){
sprintf(msg, ZOLTAN_GNO_SPEC " vertices in the graph are singletons (have no edges).", global_sum);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
/* Check if any processor has encountered an error so far */
errors = 0;
if (ierr == ZOLTAN_WARN)
errors |= 1;
if (ierr == ZOLTAN_MEMERR)
errors |= 2;
if (ierr == ZOLTAN_FATAL)
errors |= 4;
MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);
if (global_errors & 4){
/* Fatal error: return now */
if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
if (free_adjncy_sort) ZOLTAN_FREE(&perm);
return ZOLTAN_FATAL;
}
if ((IS_GLOBAL_GRAPH(graph_type)) && (check_graph >= 2)) {
/* Test for consistency across processors. */
/* Allocate space for off-proc data */
mesg_size = (2*sizeof(indextype)) + (ewgt_dim * sizeof(weighttype));
sendgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size);
recvgno = (indextype *) ZOLTAN_MALLOC(cross_edges*mesg_size);
proclist = (int *) ZOLTAN_MALLOC(cross_edges*sizeof(int));
if (cross_edges && !(sendgno && recvgno && proclist)){
ZOLTAN_PRINT_ERROR(proc, yo, "Out of memory.");
ierr = ZOLTAN_MEMERR;
}
/* Second pass: Copy data to send buffer */
nedges = 0;
ptr1 = (indextype *) sendgno;
for (i=0; i<num_obj; i++){
global_i = vtxdist[proc]+i;
for (ii=xadj[i]; ii<xadj[i+1]; ii++){
global_j = adjncy[ii];
/* Is global_j off-proc? */
if ((global_j < vtxdist[proc]) || (global_j >= vtxdist[proc+1])){
/* Add to list */
k=0;
while (global_j >= vtxdist[k+1]) k++;
proclist[nedges++] = k;
/* Copy (global_i, global_j) and corresponding weights to sendgno */
*ptr1++ = global_i;
*ptr1++ = global_j;
for (k=0; k<ewgt_dim; k++){
*ptr1++ = adjwgt[ii*ewgt_dim+k];
}
}
}
}
/* Do the irregular communication */
ierr = Zoltan_Comm_Create(&comm_plan, cross_edges, proclist, comm, TAG1, &nrecv);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Create.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(proc, yo, msg);
}
else {
if (nrecv != cross_edges){
sprintf(msg, "Incorrect number of edges to/from proc %d.", proc);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
ierr = Zoltan_Comm_Do(comm_plan, TAG2, (char *)sendgno, mesg_size, (char *)recvgno);
Zoltan_Comm_Destroy(&comm_plan);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
sprintf(msg, "Error %s returned from Zoltan_Comm_Do.",
(ierr == ZOLTAN_MEMERR ? "ZOLTAN_MEMERR" : "ZOLTAN_FATAL"));
ZOLTAN_PRINT_ERROR(proc, yo, msg);
}
else {
/* Third pass: Compare on-proc data to the off-proc data we received */
/* sendgno and recvgno should contain the same data except (i,j) is */
/* (j,i) */
for (i=0, ptr1=sendgno; i<cross_edges; i++){
flag = 0;
sendwgt = (weighttype *)(ptr1 + 2);
for (j=0, ptr2=recvgno; j<cross_edges; j++){
recvwgt = (weighttype *)(ptr2 + 2);
if ((ptr2[0] == ptr1[1]) && (ptr2[1] == ptr1[0])){
/* Found matching edge */
flag = 1;
/* Check weights */
for (k=0; k<ewgt_dim; k++){
if (sendwgt[k] != recvwgt[k]){
flag = -1;
ierr = ZOLTAN_WARN;
}
}
if (flag<0 && output_level>0){
sprintf(msg, "Edge weight (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") is not symmetric",
ptr1[0], ptr1[1]);
ZOLTAN_PRINT_WARN(proc, yo, msg);
}
}
ptr2 = (indextype *)(recvwgt + ewgt_dim);
}
if (!flag){
sprintf(msg, "Graph is not symmetric. "
"Edge (" TPL_IDX_SPEC "," TPL_IDX_SPEC ") exists, but not (" TPL_IDX_SPEC "," TPL_IDX_SPEC ").",
ptr1[0], ptr1[1], ptr1[1], ptr1[0]);
ZOLTAN_PRINT_ERROR(proc, yo, msg);
ierr = ZOLTAN_FATAL;
}
ptr1 = (indextype *)(sendwgt + ewgt_dim);
}
}
}
/* Free memory */
ZOLTAN_FREE(&sendgno);
ZOLTAN_FREE(&recvgno);
ZOLTAN_FREE(&proclist);
}
if (free_adjncy_sort) ZOLTAN_FREE(&adjncy_sort);
if (free_adjncy_sort) ZOLTAN_FREE(&perm);
/* Compute global error code */
errors = 0;
if (ierr == ZOLTAN_WARN)
errors |= 1;
if (ierr == ZOLTAN_MEMERR)
errors |= 2;
if (ierr == ZOLTAN_FATAL)
errors |= 4;
MPI_Allreduce(&errors, &global_errors, 1, MPI_INT, MPI_BOR, comm);
if (global_errors & 4){
return ZOLTAN_FATAL;
}
else if (global_errors & 2){
return ZOLTAN_MEMERR;
}
else if (global_errors & 1){
return ZOLTAN_WARN;
}
else {
if (proc==0 && output_level>0){
printf("ZOLTAN %s: The graph is valid with check_graph = %1d\n",
yo, check_graph);
}
return ZOLTAN_OK;
}
}
int Zoltan_LB_Get_Part_Sizes(ZZ *zz,
int num_global_parts, int part_dim, float *part_sizes)
{
/*
* Function to get the scaled partition sizes.
*
* Input:
* zz -- The Zoltan structure to which this method
* applies.
* num_global_parts -- Number of global partitions.
* (This usually equals lb->Num_Global_Parts)
* part_dim -- The number of object weights per partition.
* (This usually equals lb->Obj_Wgt_Dim.)
*
* Output:
* part_sizes -- Array of floats that gives the set partition
* sizes, scaled such that they sum to one.
*/
int i, j, nparts, fpart;
float *temp_part_sizes=NULL, *sum=NULL;
int error = ZOLTAN_OK;
char msg[128];
static char *yo = "Zoltan_LB_Get_Part_Sizes";
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)
printf("[%1d] Debug: num_global_parts = %d\n", zz->Proc, num_global_parts);
/* Barrier to make sure all procs have finished Zoltan_LB_Set_Part_Sizes */
MPI_Barrier(zz->Communicator);
/* For convenience, if no weights are used, set part_dim to 1 */
if (part_dim==0) part_dim = 1;
if (part_sizes == NULL){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument part_sizes is NULL.");
error = ZOLTAN_FATAL;
goto End;
}
/* Find max Part_Info_Len over all procs to see if they are all zero. */
MPI_Allreduce((void*) &(zz->LB.Part_Info_Len), (void*) &j,
1, MPI_INT, MPI_MAX, zz->Communicator);
if (j == 0){
/* Uniform partition sizes. */
zz->LB.Uniform_Parts = 1;
for (i = 0; i < num_global_parts*part_dim; i++)
part_sizes[i] = 1.0 / (float)num_global_parts;
}
else {
/* Get the partition sizes set by the user (application).
* Each processor puts its data in a part_dim * num_global_parts
* array. Then we gather all the data across processors.
* Out-of-range partition size data is ignored.
*/
zz->LB.Uniform_Parts = 0;
/* Pack LB.Part_Info into temp array */
temp_part_sizes = (float *)ZOLTAN_MALLOC(num_global_parts*part_dim
*sizeof(float));
sum = (float *)ZOLTAN_MALLOC(part_dim*sizeof(float));
if ((!temp_part_sizes) || (!sum)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
for (i = 0; i < num_global_parts*part_dim; i++){
temp_part_sizes[i] = -1.0;
}
for (i = 0; i < zz->LB.Part_Info_Len; i++){
/* Only assemble partition sizes for partitions and weights
in the requested range. */
if (zz->LB.Part_Info[i].Idx < part_dim){
j = zz->LB.Part_Info[i].Part_id;
if (zz->LB.Part_Info[i].Global_num == 0) {
Zoltan_LB_Proc_To_Part(zz, zz->Proc, &nparts, &fpart);
j += fpart;
}
if (j >= num_global_parts){
sprintf(msg, "Partition number %d is >= num_global_parts %d.",
j, num_global_parts);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
error = ZOLTAN_WARN;
}
else
temp_part_sizes[j*part_dim + zz->LB.Part_Info[i].Idx]
= zz->LB.Part_Info[i].Size;
}
}
/* Reduce over all procs */
MPI_Allreduce((void*) temp_part_sizes, (void*) part_sizes,
num_global_parts*part_dim, MPI_FLOAT, MPI_MAX, zz->Communicator);
/* Check for errors. Scale the sizes so they sum to one for each weight. */
for (j = 0; j < part_dim; j++)
sum[j] = 0.0;
for (i = 0; i < num_global_parts; i++){
for (j = 0; j < part_dim; j++){
if (part_sizes[i*part_dim+j]<0)
part_sizes[i*part_dim+j] = 1.0; /* default value if not set */
sum[j] += part_sizes[i*part_dim+j];
}
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL){
printf("[%1d] In %s: Partition size %1d (before scaling) = ",
zz->Proc, yo, i);
for (j = 0; j < part_dim; j++)
printf("%f, ", part_sizes[i*part_dim+j]);
printf("\n");
}
}
/* Check for sum[j] == 0 (error). */
for (j = 0; j < part_dim; j++) {
if (sum[j] == 0.0) {
sprintf(msg, "Sum of weights (component %1d) is zero.", j);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
error = ZOLTAN_FATAL;
goto End;
}
}
/* Normalize partition sizes */
for (i = 0; i < num_global_parts; i++)
for (j = 0; j < part_dim; j++)
part_sizes[i*part_dim+j] /= sum[j];
}
End:
if (temp_part_sizes) ZOLTAN_FREE(&temp_part_sizes);
if (sum) ZOLTAN_FREE(&sum);
ZOLTAN_TRACE_EXIT(zz, yo);
return error;
}
static int Zoltan_LB(
ZZ *zz,
int include_parts, /* Flag indicating whether to generate
part informtion;
0 if called by Zoltan_LB_Balance,
1 if called by Zoltan_LB_Partition. */
int *changes, /* Set to zero or one depending on if
Zoltan determines a new
decomposition or not:
zero - No changes to the decomposition
were made by the load-balancing
algorithm; migration is not needed.
one - A new decomposition is suggested
by the load-balancer; migration is
needed to establish the new
decomposition. */
int *num_gid_entries, /* The number of array entries in a global ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Global_ID_Entries. */
int *num_lid_entries, /* The number of array entries in a local ID;
set to be the max over all processors in
zz->Communicator of the parameter
Num_Local_ID_Entries. */
int *num_import_objs, /* The number of non-local objects in the
processor's new decomposition. */
ZOLTAN_ID_PTR *import_global_ids,/* Array of global IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
ZOLTAN_ID_PTR *import_local_ids, /* Array of local IDs for non-local objects
(i.e., objs to be imported) in
the processor's new decomposition. */
int **import_procs, /* Array of processor IDs for processors
currently owning non-local objects (i.e.,
objs to be imported) in this processor's
new decomposition. */
int **import_to_part, /* Partition to which the objects should be
imported. */
int *num_export_objs, /* The number of local objects that need to
be exported from the processor to establish
the new decomposition. */
ZOLTAN_ID_PTR *export_global_ids,/* Array of global IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
ZOLTAN_ID_PTR *export_local_ids, /* Array of local IDs for objects that need
to be exported (assigned and sent to other
processors) to establish the new
decomposition. */
int **export_procs, /* Array of destination processor IDs for
objects that need to be exported
to establish the new decomposition. */
int **export_to_part /* Partition to which objects should be
exported. */
)
{
/*
* Main load-balancing routine.
* Input: a Zoltan structure with appropriate function pointers set.
* Output:
* changes
* num_import_objs
* import_global_ids
* import_local_ids
* import_procs
* import_to_part
* num_export_objs
* export_global_ids
* export_local_ids
* export_procs
* export_to_part
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_LB";
int gmax; /* Maximum number of imported/exported objects
over all processors. */
int error = ZOLTAN_OK; /* Error code */
double start_time, end_time;
double lb_time[2] = {0.0,0.0};
char msg[256];
int comm[3],gcomm[3];
float *part_sizes = NULL, *fdummy = NULL;
int wgt_dim, part_dim;
int all_num_obj, i, ts, idIdx;
struct Hash_Node **ht;
int *export_all_procs, *export_all_to_part, *parts=NULL;
ZOLTAN_ID_PTR all_global_ids=NULL, all_local_ids=NULL;
ZOLTAN_ID_PTR gid;
#ifdef ZOLTAN_OVIS
struct OVIS_parameters ovisParameters;
#endif
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS){
printf("Build configuration:\n");
Zoltan_Print_Configuration(" ");
printf("\n");
Zoltan_Print_Key_Params(zz);
}
start_time = Zoltan_Time(zz->Timer);
#ifdef ZOLTAN_OVIS
Zoltan_OVIS_Setup(zz, &ovisParameters);
if (zz->Proc == 0)
printf("OVIS PARAMETERS %s %s %d %f\n",
ovisParameters.hello,
ovisParameters.dll,
ovisParameters.outputLevel,
ovisParameters.minVersion);
ovis_enabled(zz->Proc, ovisParameters.dll);
#endif
/*
* Compute Max number of array entries per ID over all processors.
* Compute Max number of return arguments for Zoltan_LB_Balance.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
comm[2] = zz->LB.Return_Lists;
MPI_Allreduce(comm, gcomm, 3, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
zz->LB.Return_Lists = gcomm[2];
/* assume no changes */
*changes = 0;
*num_import_objs = *num_export_objs = 0;
*import_global_ids = NULL;
*import_local_ids = NULL;
*import_procs = NULL;
*import_to_part = NULL;
*export_global_ids = NULL;
*export_local_ids = NULL;
*export_procs = NULL;
*export_to_part = NULL;
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
goto End;
if (zz->LB.Method == NONE) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s Balancing method selected == NONE; no balancing performed\n",
yo);
error = ZOLTAN_WARN;
goto End;
}
/*
* Sync the random number generator across processors.
*/
Zoltan_Srand_Sync(Zoltan_Rand(NULL), NULL, zz->Communicator);
/* Since generating a new partition, need to free old mapping vector */
zz->LB.OldRemap = zz->LB.Remap;
zz->LB.Remap = NULL;
error = Zoltan_LB_Build_PartDist(zz);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN)
goto End;
if (zz->Debug_Level >= ZOLTAN_DEBUG_ALL) {
int i, np, fp;
for (i = 0; i < zz->Num_Proc; i++) {
Zoltan_LB_Proc_To_Part(zz, i, &np, &fp);
printf("%d Proc_To_Part Proc %d NParts %d FPart %d\n",
zz->Proc, i, np, fp);
}
}
/*
* Generate parts sizes.
*/
#ifdef ZOLTAN_OVIS
/* set part sizes computed by OVIS, if requested. Processes set only their own value */
{
float part_sizes[1];
int part_ids[1], wgt_idx[1];
wgt_idx[0] = 0;
part_ids[0] = 0;
ovis_getPartsize(&(part_sizes[0]));
printf("Rank %d ps %f\n",zz->Proc, part_sizes[0]);
/* clear out old part size info first */
Zoltan_LB_Set_Part_Sizes(zz, 0, -1, NULL, NULL, NULL);
Zoltan_LB_Set_Part_Sizes(zz, 0, 1, part_ids, wgt_idx, part_sizes);
}
#endif
wgt_dim = zz->Obj_Weight_Dim;
part_dim = ((wgt_dim > 0) ? wgt_dim : 1);
part_sizes = (float *) ZOLTAN_MALLOC(sizeof(float) * part_dim
* zz->LB.Num_Global_Parts);
if (part_sizes == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error.");
error = ZOLTAN_MEMERR;
goto End;
}
/* Get part sizes. */
Zoltan_LB_Get_Part_Sizes(zz, zz->LB.Num_Global_Parts, part_dim,
part_sizes);
#ifdef ZOLTAN_OVIS
/* if (ovisParameters.outputlevel > 3) */
{
int myRank = zz->Proc;
if (myRank == 0){
int i, j;
for (i = 0; i < zz->LB.Num_Global_Parts; i++){
for (j = 0; j < part_dim; j++){
printf("Rank %d AG: part_sizes[%d] = %f (Num_Global_Parts = %d, part_dim = %d)\n",zz->Proc,
(i*part_dim+j), part_sizes[i*part_dim+j],zz->LB.Num_Global_Parts, part_dim);
}
}
}
}
#endif
/*
* Call the actual load-balancing function.
*/
error = zz->LB.LB_Fn(zz, part_sizes,
num_import_objs, import_global_ids, import_local_ids,
import_procs, import_to_part,
num_export_objs, export_global_ids, export_local_ids,
export_procs, export_to_part);
ZOLTAN_FREE(&part_sizes);
if (error == ZOLTAN_FATAL || error == ZOLTAN_MEMERR){
sprintf(msg, "Partitioning routine returned code %d.", error);
#ifdef HOST_LINUX
if ((error == ZOLTAN_MEMERR) && (Zoltan_Memory_Get_Debug() > 0)){
Zoltan_write_linux_meminfo(0, "State of /proc/meminfo after malloc failure\n", 0);
}
#endif
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
else if (error){
if (zz->Debug_Level >ZOLTAN_DEBUG_NONE) {
sprintf(msg, "Partitioning routine returned code %d.", error);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done partitioning");
if (*num_import_objs >= 0)
MPI_Allreduce(num_import_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
else /* use export data */
MPI_Allreduce(num_export_objs, &gmax, 1, MPI_INT, MPI_MAX,
zz->Communicator);
if (gmax == 0) {
/*
* Decomposition was not changed by the load balancing; no migration
* is needed.
*/
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
printf("%s No changes to the decomposition due to partitioning; "
"no migration is needed.\n", yo);
/*
* Reset num_import_objs and num_export_objs; don't want to return
* -1 for the arrays that weren't returned by ZOLTAN_LB_FN.
*/
*num_import_objs = *num_export_objs = 0;
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS){
/*
* This parameter setting requires that all local objects
* and their assignments appear in the export list.
*/
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
goto End;
}
/*
* Check whether we know the import data, export data, or both.
*
* If we were given the import data,
* we know what the new decomposition should look like on the
* processor, but we don't know which of our local objects we have
* to export to other processors to establish the new decomposition.
* Reverse the argument if we were given the export data.
*
* Unless we were given both maps, compute the inverse map.
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
if (*num_import_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
if (*num_export_objs >= 0)
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
*num_import_objs = *num_export_objs = -1;
}
if (*num_import_objs >= 0){
if (*num_export_objs >= 0) {
/* Both maps already available; nothing to do. */;
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Export lists are requested; compute export map */
error = Zoltan_Invert_Lists(zz, *num_import_objs, *import_global_ids,
*import_local_ids, *import_procs,
*import_to_part,
num_export_objs, export_global_ids,
export_local_ids, export_procs,
export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_EXPORT_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/* Method returned import lists, but only export lists were desired. */
/* Import lists not needed; free them. */
*num_import_objs = -1;
Zoltan_LB_Special_Free_Part(zz, import_global_ids, import_local_ids,
import_procs, import_to_part);
}
}
}
else { /* (*num_import_objs < 0) */
if (*num_export_objs >= 0) {
/* Only export lists have been returned. */
if (zz->LB.Return_Lists == ZOLTAN_LB_ALL_LISTS ||
zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Compute import map */
error = Zoltan_Invert_Lists(zz, *num_export_objs, *export_global_ids,
*export_local_ids, *export_procs,
*export_to_part,
num_import_objs, import_global_ids,
import_local_ids, import_procs,
import_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error building return arguments; "
"%d returned by Zoltan_Compute_Destinations\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
if (zz->LB.Return_Lists == ZOLTAN_LB_IMPORT_LISTS) {
/* Method returned export lists, but only import lists are desired. */
/* Export lists not needed; free them. */
*num_export_objs = -1;
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
}
}
}
else { /* *num_export_objs < 0 && *num_import_objs < 0) */
if (zz->LB.Return_Lists) {
/* No map at all available */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Load-balancing function returned "
"neither import nor export data.");
error = ZOLTAN_WARN;
goto End;
}
}
}
if (zz->LB.Return_Lists == ZOLTAN_LB_COMPLETE_EXPORT_LISTS) {
/*
* Normally, Zoltan_LB returns in the export lists all local
* objects that are moving off processor, or that are assigned
* to a part on the local processor that is not the
* default part. This setting of Return_Lists requests
* that all local objects be included in the export list.
*/
if (*num_export_objs == 0){
/* all local objects are remaining on processor */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, num_export_objs,
export_global_ids, export_local_ids,
wgt_dim, &fdummy, export_to_part);
if (error == ZOLTAN_OK){
ZOLTAN_FREE(&fdummy);
if (*num_export_objs) {
if (Zoltan_Special_Malloc(zz, (void **)export_procs, *num_export_objs,
ZOLTAN_SPECIAL_MALLOC_INT)){
for (i=0; i<*num_export_objs; i++)
(*export_procs)[i] = zz->Proc;
}
else{
error = ZOLTAN_MEMERR;
}
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)) goto End;
}
else{
all_num_obj = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &error);
if (*num_export_objs < all_num_obj){
/* Create a lookup table for exported IDs */
ts = Zoltan_Recommended_Hash_Size(*num_export_objs);
ht = create_hash_table(zz, *export_global_ids, *num_export_objs, ts);
/* Create a list of all gids, lids and parts */
error= Zoltan_Get_Obj_List_Special_Malloc(zz, &all_num_obj,
&all_global_ids, &all_local_ids,
wgt_dim, &fdummy, &parts);
if ((error == ZOLTAN_OK) || (error == ZOLTAN_WARN)){
ZOLTAN_FREE(&fdummy);
if ((Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_procs,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0) ||
(Zoltan_Special_Malloc(zz, (void **)(void*)&export_all_to_part,
all_num_obj, ZOLTAN_SPECIAL_MALLOC_INT)==0)){
error = ZOLTAN_MEMERR;
}
}
if ((error != ZOLTAN_OK) && (error != ZOLTAN_WARN)){
sprintf(msg, "Error building complete export list; "
"%d returned by Zoltan_Get_Obj_List\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
gid = all_global_ids;
for (i=0; i < all_num_obj; i++, gid += zz->Num_GID){
idIdx = search_hash_table(zz, gid, ht, ts);
if (idIdx >= 0){
export_all_procs[i] = (*export_procs)[idIdx];
export_all_to_part[i] = (*export_to_part)[idIdx];
}
else{
export_all_procs[i] = zz->Proc;
export_all_to_part[i] = parts[i];
}
}
free_hash_table(ht, ts);
Zoltan_LB_Special_Free_Part(zz, export_global_ids, export_local_ids,
export_procs, export_to_part);
Zoltan_Special_Free(zz, (void **)(void*)&parts,
ZOLTAN_SPECIAL_MALLOC_INT);
*export_global_ids = all_global_ids;
*export_local_ids = all_local_ids;
*export_procs = export_all_procs;
*export_to_part = export_all_to_part;
*num_export_objs = all_num_obj;
}
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done building return arguments");
end_time = Zoltan_Time(zz->Timer);
lb_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i;
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: Objects to be imported to Proc %d\n", zz->Proc);
for (i = 0; i < *num_import_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*import_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*import_to_part != NULL ? (*import_to_part)[i]
: zz->Proc));
printf(" From processor: %4d\n", (*import_procs)[i]);
}
printf("\n");
printf("ZOLTAN: Objects to be exported from Proc %d\n", zz->Proc);
for (i = 0; i < *num_export_objs; i++) {
printf(" Obj: ");
ZOLTAN_PRINT_GID(zz, &((*export_global_ids)[i*zz->Num_GID]));
printf(" To part: %4d",
(*export_to_part != NULL ? (*export_to_part)[i]
: (*export_procs)[i]));
printf(" To processor: %4d\n", (*export_procs)[i]);
}
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/*
* If the Help_Migrate flag is set, perform migration for the application.
*/
if (zz->Migrate.Auto_Migrate) {
ZOLTAN_TRACE_DETAIL(zz, yo, "Begin auto-migration");
start_time = Zoltan_Time(zz->Timer);
error = Zoltan_Migrate(zz,
*num_import_objs, *import_global_ids,
*import_local_ids, *import_procs, *import_to_part,
*num_export_objs, *export_global_ids,
*export_local_ids, *export_procs, *export_to_part);
if (error != ZOLTAN_OK && error != ZOLTAN_WARN) {
sprintf(msg, "Error in auto-migration; %d returned from "
"Zoltan_Help_Migrate\n", error);
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
goto End;
}
end_time = Zoltan_Time(zz->Timer);
lb_time[1] = end_time - start_time;
ZOLTAN_TRACE_DETAIL(zz, yo, "Done auto-migration");
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[0],
"ZOLTAN Partition: ");
if (zz->Migrate.Auto_Migrate)
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, lb_time[1],
"ZOLTAN Migrate: ");
}
*changes = 1;
End:
ZOLTAN_TRACE_EXIT(zz, yo);
return (error);
}
int Zoltan_DD_Find (
Zoltan_DD_Directory *dd, /* contains directory state information */
ZOLTAN_ID_PTR gid, /* Incoming list of GIDs to get owners proc */
ZOLTAN_ID_PTR lid, /* Outgoing corresponding list of LIDs */
ZOLTAN_ID_PTR data, /* Outgoing optional corresponding user data */
int *partition, /* Outgoing optional partition information */
int count, /* Count of GIDs in above list (in) */
int *owner) /* Outgoing corresponding list of data locations */
{
ZOLTAN_COMM_OBJ *plan = NULL ; /* efficient MPI communication */
char *rbuff = NULL ; /* receive buffer */
char *sbuff = NULL ; /* send buffer */
int *procs = NULL ; /* list of processors to contact */
DD_Find_Msg *ptr = NULL ;
int i ;
int nrec ; /* number of messages to receive */
int err ; /* return error condition */
int errcount ; /* count of GIDs not found */
char *yo = "Zoltan_DD_Find" ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_IN(dd->my_proc, yo, NULL);
/* input sanity check */
if (dd == NULL || count < 0 || ((owner == NULL || gid == NULL) && count > 0))
{
ZOLTAN_PRINT_ERROR ((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, "Invalid input argument.") ;
if (dd != NULL && dd->debug_level > 1)
ZOLTAN_TRACE_OUT((dd == NULL ? ZOLTAN_DD_NO_PROC : dd->my_proc),
yo, NULL);
return ZOLTAN_DD_INPUT_ERROR ;
}
/* allocate memory for processors to contact for directory info */
if (count > 0)
{
procs = (int *) ZOLTAN_MALLOC (sizeof (int) * count) ;
if (procs == NULL)
{
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc proc list.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
/* allocate memory for DD_Find_Msg send buffer */
if (count > 0)
{
sbuff = (char *) ZOLTAN_MALLOC (dd->find_msg_size * count) ;
if (sbuff == NULL)
{
ZOLTAN_FREE (&procs) ;
ZOLTAN_PRINT_ERROR (dd->my_proc, yo, "Unable to malloc send buffer.") ;
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return ZOLTAN_DD_MEMORY_ERROR ;
}
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After mallocs.");
/* for each GID, fill DD_Find_Msg buffer and contact list */
for (i = 0 ; i < count ; i++)
{
procs[i] = dd->hash (gid + i*dd->gid_length, dd->gid_length, dd->nproc) ;
ptr = (DD_Find_Msg *) (sbuff + i * dd->find_msg_size) ;
ptr->index = i ;
ptr->proc = procs[i] ;
ZOLTAN_SET_ID (dd->gid_length, ptr->id, gid + i*dd->gid_length) ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill.");
/* create efficient communication plan */
err = Zoltan_Comm_Create (&plan, count, procs, dd->comm,
ZOLTAN_DD_FIND_MSG_TAG, &nrec) ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Create.");
if (err != ZOLTAN_OK)
goto fini ;
/* allocate receive buffer */
if (nrec > 0)
{
rbuff = (char *) ZOLTAN_MALLOC (nrec * dd->find_msg_size) ;
if (rbuff == NULL)
{
err = ZOLTAN_DD_MEMORY_ERROR ;
goto fini ;
}
}
/* send out find messages across entire system */
err = Zoltan_Comm_Do (plan, ZOLTAN_DD_FIND_MSG_TAG+1, sbuff,
dd->find_msg_size, rbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Do.");
/* get find messages directed to me, fill in return information */
errcount = 0 ;
for (i = 0 ; i < nrec ; i++)
{
ptr = (DD_Find_Msg *) (rbuff + i*dd->find_msg_size) ;
err = DD_Find_Local (dd, ptr->id, ptr->id, ptr->id + dd->max_id_length,
(partition) ? (&ptr->partition) : NULL, &ptr->proc) ;
if (err == ZOLTAN_DD_GID_NOT_FOUND_ERROR)
errcount++ ;
}
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill in return info.");
/* send return information back to requester */
err = Zoltan_Comm_Do_Reverse(plan, ZOLTAN_DD_FIND_MSG_TAG+2, rbuff,
dd->find_msg_size, NULL, sbuff) ;
if (err != ZOLTAN_OK)
goto fini ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After Comm_Reverse.");
/* fill in user supplied lists with returned information */
for (i = 0 ; i < count ; i++)
{
ptr = (DD_Find_Msg *) (sbuff + i*dd->find_msg_size) ;
owner[ptr->index] = ptr->proc ;
if (partition != NULL)
partition[ptr->index] = ptr->partition ;
if (lid != NULL)
ZOLTAN_SET_ID (dd->lid_length, lid + ptr->index * dd->lid_length,
ptr->id) ;
if (data != NULL)
ZOLTAN_SET_ID (dd->user_data_length, data + ptr->index
* dd->user_data_length, ptr->id + dd->max_id_length) ;
}
/* if at least one GID was not found, notify caller of error */
err = (errcount == 0) ? ZOLTAN_DD_NORMAL_RETURN
: ZOLTAN_DD_GID_NOT_FOUND_ERROR ;
if (dd->debug_level > 2)
ZOLTAN_PRINT_INFO(dd->my_proc, yo, "After fill return lists.");
fini:
ZOLTAN_FREE (&sbuff) ;
ZOLTAN_FREE (&rbuff) ;
ZOLTAN_FREE (&procs) ;
Zoltan_Comm_Destroy (&plan) ;
if (err != ZOLTAN_DD_NORMAL_RETURN)
ZOLTAN_PRINT_WARN (dd->my_proc, yo, "Return is not normal.") ;
if (dd->debug_level > 0)
{
char str[100] ; /* diagnostic message string */
sprintf (str, "Processed %d GIDs, GIDs not found: %d", count, errcount) ;
ZOLTAN_PRINT_INFO (dd->my_proc, yo, str) ;
}
if (dd->debug_level > 1)
ZOLTAN_TRACE_OUT(dd->my_proc, yo, NULL);
return err ;
}
int Zoltan_Order(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Number of objects to order */
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
/* The application must allocate enough space */
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
/* The application must allocate enough space */
int *rank, /* rank[i] is the rank of gids[i] */
int *iperm, /* inverse permutation of rank */
ZOS *order_info /* Method-specific ordering info. Currently not used. */
)
{
/*
* Main user-call for ordering.
* Input:
* zz, a Zoltan structure with appropriate function pointers set.
* gids, a list of global ids or enough space to store such a list
* lids, a list of local ids or enough space to store such a list
* Output:
* num_gid_entries
* num_lid_entries
* gids, a list of global ids (filled in if empty on entry)
* lids, a list of local ids (filled in if empty on entry)
* rank, rank[i] is the global rank of gids[i]
* iperm, inverse permutation of rank
* order_info, a Zoltan Ordering Struct with additional info.
* Return values:
* Zoltan error code.
*/
char *yo = "Zoltan_Order";
int ierr;
int *vtxdist;
double start_time, end_time;
double order_time[2] = {0.0,0.0};
char msg[256];
int comm[2],gcomm[2];
ZOLTAN_ORDER_FN *Order_fn;
struct Zoltan_Order_Options opt;
ZOLTAN_TRACE_ENTER(zz, yo);
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
Zoltan_Print_Key_Params(zz);
start_time = Zoltan_Time(zz->Timer);
/*
* Compute Max number of array entries per ID over all processors.
* This is a sanity-maintaining step; we don't want different
* processors to have different values for these numbers.
*/
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/*
* Return if this processor is not in the Zoltan structure's
* communicator.
*/
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_OK);
}
/*
* Get ordering options from parameter list.
*/
/* Set default parameter values */
strncpy(opt.method, "PARMETIS", MAX_PARAM_STRING_LEN);
strncpy(opt.order_type, "GLOBAL", MAX_PARAM_STRING_LEN);
opt.use_order_info = 0;
opt.start_index = 0;
opt.reorder = 0;
Zoltan_Bind_Param(Order_params, "ORDER_METHOD", (void *) opt.method);
Zoltan_Bind_Param(Order_params, "ORDER_TYPE", (void *) opt.order_type);
Zoltan_Bind_Param(Order_params, "ORDER_START_INDEX", (void *) &opt.start_index);
Zoltan_Bind_Param(Order_params, "REORDER", (void *) &opt.reorder);
Zoltan_Bind_Param(Order_params, "USE_ORDER_INFO", (void *) &opt.use_order_info);
Zoltan_Assign_Param_Vals(zz->Params, Order_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (opt.use_order_info == 0) order_info = NULL;
/*
* Check that the user has allocated space for the return args.
*/
if (!(gids && lids && rank && iperm)){
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input argument is NULL. Please allocate all required arrays before calling this routine.");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Find the selected method.
*/
if (!strcmp(opt.method, "NONE")) {
if (zz->Proc == zz->Debug_Proc && zz->Debug_Level >= ZOLTAN_DEBUG_PARAMS)
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Ordering method selected == NONE; no ordering performed\n");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_WARN);
}
else if (!strcmp(opt.method, "NODEND")) {
Order_fn = Zoltan_ParMetis_Order;
}
else if (!strcmp(opt.method, "METIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "LOCAL");
}
else if (!strcmp(opt.method, "PARMETIS")) {
Order_fn = Zoltan_ParMetis_Order;
/* Set ORDER_METHOD to NODEND and ORDER_TYPE to LOCAL */
strcpy(opt.method, "NODEND");
strcpy(opt.order_type, "GLOBAL");
}
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Unknown ordering method");
ZOLTAN_TRACE_EXIT(zz, yo);
return (ZOLTAN_FATAL);
}
/*
* Construct the heterogenous machine description.
*/
ierr = Zoltan_Build_Machine_Desc(zz);
if (ierr == ZOLTAN_FATAL){
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done machine description");
/*
* Call the actual ordering function.
*/
ierr = (*Order_fn)(zz, num_obj, gids, lids, rank, iperm, &opt, order_info);
if (ierr) {
sprintf(msg, "Ordering routine returned error code %d.", ierr);
if (ierr == ZOLTAN_WARN){
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
} else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}
}
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
/* Compute inverse permutation if necessary */
ierr = Zoltan_Get_Distribution(zz, &vtxdist);
if (ierr){
/* Error */
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error returned from Zoltan_Get_Distribution.\n");
return (ierr);
}
if (!(opt.return_args & RETURN_RANK)){
/* Compute rank from iperm */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, iperm, rank, vtxdist, opt.order_type, opt.start_index);
}
else if (!(opt.return_args & RETURN_IPERM)){
/* Compute iperm from rank */
ZOLTAN_TRACE_DETAIL(zz, yo, "Inverting permutation");
Zoltan_Inverse_Perm(zz, rank, iperm, vtxdist, opt.order_type, opt.start_index);
}
ZOLTAN_FREE(&vtxdist);
ZOLTAN_TRACE_DETAIL(zz, yo, "Done ordering");
end_time = Zoltan_Time(zz->Timer);
order_time[0] = end_time - start_time;
if (zz->Debug_Level >= ZOLTAN_DEBUG_LIST) {
int i, nobjs;
nobjs = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &i);
Zoltan_Print_Sync_Start(zz->Communicator, TRUE);
printf("ZOLTAN: rank for ordering on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("GID = ");
ZOLTAN_PRINT_GID(zz, &(gids[i*(*num_gid_entries)]));
printf(", rank = %3d\n", rank[i]);
}
printf("\n");
printf("ZOLTAN: inverse permutation on Proc %d\n", zz->Proc);
for (i = 0; i < nobjs; i++) {
printf("iperm[%3d] = %3d\n", i, iperm[i]);
}
printf("\n");
Zoltan_Print_Sync_End(zz->Communicator, TRUE);
}
/* Print timing info */
if (zz->Debug_Level >= ZOLTAN_DEBUG_ZTIME) {
if (zz->Proc == zz->Debug_Proc) {
printf("ZOLTAN Times: \n");
}
Zoltan_Print_Stats (zz->Communicator, zz->Debug_Proc, order_time[0],
"ZOLTAN Balance: ");
}
ZOLTAN_TRACE_EXIT(zz, yo);
if (ierr)
return (ierr);
else
return (ZOLTAN_OK);
}
int Zoltan_Color_Test(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */
/* The application must allocate enough space */
ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */
/* The application must allocate enough space */
int *color_exp /* Input: Colors assigned to local vertices */
)
{
static char *yo = "color_test_fn";
int nvtx = num_obj; /* number of vertices */
int i, j;
int ierr = ZOLTAN_OK;
int ferr = ZOLTAN_OK; /* final error signal */
char coloring_problem; /* Input: which coloring to perform;
currently only supports D1, D2 coloring and variants */
char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */
int ss=100;
char comm_pattern='S', coloring_order='I', coloring_method='F';
int comm[2],gcomm[2];
int *color=NULL;
int *adjproc=NULL, *xadj=NULL;
ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */
ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL;
ZG graph;
ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of
the requested GIDs. */
int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */
int *partialD2 = NULL; /* global binary array showing which vertices to be colored */
struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */
ZOLTAN_GNO_TYPE *local_GNOs = NULL;
ZOLTAN_GNO_TYPE *global_GNOs = NULL;
memset (&graph, 0, sizeof(ZG));
/* PARAMETER SETTINGS */
Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr);
Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss);
Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern);
Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order);
Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method);
strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN);
Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
/* Check validity of parameters - they should be consistent with Zoltan_Color */
if (!strcasecmp(coloring_problemStr, "distance-1"))
coloring_problem = '1';
else if (!strcasecmp(coloring_problemStr, "distance-2"))
coloring_problem = '2';
else if (!strcasecmp(coloring_problemStr, "partial-distance-2")
|| !strcasecmp(coloring_problemStr, "bipartite"))
coloring_problem = 'P';
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring.");
coloring_problem = '1';
}
if (ss == 0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100.");
ss = 100;
}
if (comm_pattern != 'S' && comm_pattern != 'A') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_method !='F') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F).");
coloring_method = 'F';
}
/* Compute Max number of array entries per ID over all processors.
This is a sanity-maintaining step; we don't want different
processors to have different values for these numbers. */
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/* Return if this processor is not in the Zoltan structure's
communicator. */
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
return ZOLTAN_OK;
/* BUILD THE GRAPH */
/* Check that the user has allocated space for the return args. */
if (!color_exp)
ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices.");
requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj *
sizeof(ZOLTAN_GNO_TYPE));
Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj,
global_ids, requested_GNOs);
Zoltan_ZG_Export (zz, &graph,
&gvtx, &nvtx, NULL, NULL,
&vtxdist, &xadj, &adjncy, &adjproc,
NULL, NULL);
if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){
if (zz->Proc == 0){
fprintf(stderr,
"Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx);
}
ierr = ZOLTAN_FATAL;
goto End;
}
KDDKDDKDD(zz->Proc, "COLORTEST DD");
/* Exchange global color information */
if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<nvtx; ++i)
local_GNOs[i] = i+vtxdist[zz->Proc];
if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
global_GNOs[i] = i;
ierr = Zoltan_DD_Create (&dd_color, zz->Communicator,
sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory.");
/* Put req obs with 1 but first inialize the rest with 0 */
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL,
NULL, color, nvtx);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL,
NULL, color_exp, num_obj);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
/* Get requested colors from the DDirectory. */
ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL,
color, vtxdist[zz->Num_Proc], NULL);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory.");
/* Free DDirectory */
Zoltan_DD_Destroy(&dd_color);
ZOLTAN_FREE(&local_GNOs);
ZOLTAN_FREE(&global_GNOs);
KDDKDDKDD(zz->Proc, "COLORTEST CHECK");
if (coloring_problem == 'P' || coloring_problem == '2') {
if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (coloring_problem == 'P') {
for (i=0; i<num_obj; ++i) {
int gno=requested_GNOs[i];
loc_partialD2[gno] = 1;
}
MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator);
} else {
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
partialD2[i] = 1;
}
}
/* Check if there is an error in coloring */
if (coloring_problem == '1') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j];
if (color[gno] == color[v]) { /* neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]);
break;
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else if (coloring_problem == '2' || coloring_problem == 'P') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j], k;
if (partialD2[v] && color[v] <= 0) {
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
for (k = j+1; k < xadj[i+1]; ++k) {
int w = (int)adjncy[k];
if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]);
break;
}
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else
ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification.");
End:
KDDKDDKDD(zz->Proc, "COLORTEST DONE");
if (ierr==ZOLTAN_FATAL)
ierr = 2;
MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (ferr == 2)
ierr = ZOLTAN_FATAL;
else
ierr = ZOLTAN_OK;
Zoltan_ZG_Free (&graph);
ZOLTAN_FREE(&adjproc);
ZOLTAN_FREE(&color);
ZOLTAN_FREE(&requested_GNOs);
ZOLTAN_FREE(&partialD2);
ZOLTAN_FREE(&loc_partialD2);
return ierr;
}
int Zoltan_Get_Coordinates(
ZZ *zz,
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global IDs of objects */
ZOLTAN_ID_PTR local_ids, /* Input: local IDs of objects; may be NULL. */
int *num_dim, /* Output: dimension of coordinates */
double **coords /* Output: array of coordinates; malloc'ed by
fn if NULL upon input. */
)
{
char *yo = "Zoltan_Get_Coordinates";
int i,j,rc;
int num_gid_entries = zz->Num_GID;
int num_lid_entries = zz->Num_LID;
int alloced_coords = 0;
ZOLTAN_ID_PTR lid; /* Temporary pointers to local IDs; used to pass
NULL to query functions when NUM_LID_ENTRIES == 0. */
double dist[3];
double im[3][3];
double deg_ratio;
double x;
int order[3];
int reduce_dimensions, d, axis_aligned;
int target_dim;
int ierr = ZOLTAN_OK;
char msg[256];
ZZ_Transform *tran;
ZOLTAN_TRACE_ENTER(zz, yo);
/* Error check -- make sure needed callback functions are registered. */
if (zz->Get_Num_Geom == NULL ||
(zz->Get_Geom == NULL && zz->Get_Geom_Multi == NULL)) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Must register ZOLTAN_NUM_GEOM_FN and "
"either ZOLTAN_GEOM_MULTI_FN or ZOLTAN_GEOM_FN");
goto End;
}
/* Get problem dimension. */
*num_dim = zz->Get_Num_Geom(zz->Get_Num_Geom_Data, &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_NUM_GEOM_FN");
goto End;
}
if (*num_dim < 0 || *num_dim > 3) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Invalid dimension returned from ZOLTAN_NUM_GEOM_FN");
goto End;
}
/* Get coordinates for object; allocate memory if not already provided. */
if (*num_dim > 0 && num_obj > 0) {
if (*coords == NULL) {
alloced_coords = 1;
*coords = (double *) ZOLTAN_MALLOC(num_obj * (*num_dim) * sizeof(double));
if (*coords == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Memory error");
goto End;
}
}
if (zz->Get_Geom_Multi != NULL) {
zz->Get_Geom_Multi(zz->Get_Geom_Multi_Data, zz->Num_GID, zz->Num_LID,
num_obj, global_ids, local_ids, *num_dim, *coords,
&ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_MULTI_FN");
goto End;
}
}
else {
for (i = 0; i < num_obj; i++) {
lid = (num_lid_entries ? &(local_ids[i*num_lid_entries]) : NULL);
zz->Get_Geom(zz->Get_Geom_Data, num_gid_entries, num_lid_entries,
global_ids + i*num_gid_entries, lid,
(*coords) + i*(*num_dim), &ierr);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Error returned from ZOLTAN_GET_GEOM_FN");
goto End;
}
}
}
}
/*
* For RCB, RIB, and HSFC: if REDUCE_DIMENSIONS was selected, compute the
* center of mass and inertial matrix of the coordinates.
*
* For 3D problems: If the geometry is "flat", transform the points so the
* two primary directions lie along the X and Y coordinate axes and project
* to the Z=0 plane. If in addition the geometry is "skinny", project to
* the X axis. (This creates a 2D or 1D problem respectively.)
*
* For 2D problems: If the geometry is essentially a line, transform it's
* primary direction to the X axis and project to the X axis, yielding a
* 1D problem.
*
* Return these points to the partitioning algorithm, in effect partitioning
* in only the 2 (or 1) significant dimensions.
*/
if (((*num_dim == 3) || (*num_dim == 2)) &&
((zz->LB.Method==RCB) || (zz->LB.Method==RIB) || (zz->LB.Method==HSFC))){
Zoltan_Bind_Param(Reduce_Dim_Params, "KEEP_CUTS", (void *)&i);
Zoltan_Bind_Param(Reduce_Dim_Params, "REDUCE_DIMENSIONS",
(void *)&reduce_dimensions);
Zoltan_Bind_Param(Reduce_Dim_Params, "DEGENERATE_RATIO", (void *)°_ratio);
i = 0;
reduce_dimensions = 0;
deg_ratio = 10.0;
Zoltan_Assign_Param_Vals(zz->Params, Reduce_Dim_Params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
if (reduce_dimensions == 0){
goto End;
}
if (deg_ratio <= 1){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "DEGENERATE_RATIO <= 1, setting it to 10.0");
}
deg_ratio = 10.0;
}
if (zz->LB.Method == RCB){
tran = &(((RCB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else if (zz->LB.Method == RIB){
tran = &(((RIB_STRUCT *)(zz->LB.Data_Structure))->Tran);
}
else{
tran = &(((HSFC_Data*)(zz->LB.Data_Structure))->tran);
}
d = *num_dim;
if (tran->Target_Dim >= 0){
/*
* On a previous load balancing call, we determined whether
* or not the geometry was degenerate. If the geometry was
* determined to be not degenerate, then we assume it is still
* not degenerate, and we skip the degeneracy calculation.
*/
if (tran->Target_Dim > 0){
/*
* The geometry *was* degenerate. We test the extent
* of the geometry along the principal directions determined
* last time to determine if it is still degenerate with that
* orientation. If so, we transform the coordinates using the
* same transformation we used last time. If not, we do the
* entire degeneracy calculation again.
*/
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
else{
axis_aligned = 0;
}
projected_distances(zz, *coords, num_obj, tran->CM,
tran->Evecs, dist, d, axis_aligned, tran->Axis_Order);
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
transform_coordinates(*coords, num_obj, d, tran);
}
else{
/* Set's Target_Dim to -1, flag to recompute degeneracy */
Zoltan_Initialize_Transformation(tran);
}
}
}
if (tran->Target_Dim < 0){
tran->Target_Dim = 0;
/*
* Get the center of mass and inertial matrix of coordinates. Ignore
* vertex weights, we are only interested in geometry. Global operation.
*/
if (d == 2){
inertial_matrix2D(zz, *coords, num_obj, tran->CM, im);
}
else{
inertial_matrix3D(zz, *coords, num_obj, tran->CM, im);
}
/*
* The inertial matrix is a 3x3 or 2x2 real symmetric matrix. Get its
* three or two orthonormal eigenvectors. These usually indicate the
* orientation of the geometry.
*/
rc = eigenvectors(im, tran->Evecs, d);
if (rc){
if (zz->Proc == 0){
ZOLTAN_PRINT_WARN(0, yo, "REDUCE_DIMENSIONS calculation failed");
}
goto End;
}
/*
* Here we check to see if the eigenvectors are very close
* to the coordinate axes. If so, we can more quickly
* determine whether the geometry is degenerate, and also more
* quickly transform the geometry to the lower dimensional
* space.
*/
axis_aligned = 0;
for (i=0; i<d; i++){
tran->Axis_Order[i] = -1;
}
for (j=0; j<d; j++){
for (i=0; i<d; i++){
x = fabs(tran->Evecs[i][j]);
if (NEAR_ONE(x)){
tran->Axis_Order[j] = i; /* e'vector j is very close to i axis */
break;
}
}
if (tran->Axis_Order[j] < 0){
break;
}
}
if ((tran->Axis_Order[0] >= 0) &&
(tran->Axis_Order[1] >= 0) && (tran->Axis_Order[2] >= 0)){
axis_aligned = 1;
}
/*
* Calculate the extent of the geometry along the three lines defined
* by the direction of the eigenvectors through the center of mass.
*/
projected_distances(zz, *coords, num_obj, tran->CM, tran->Evecs, dist,
d, axis_aligned, tran->Axis_Order);
/*
* Decide whether these distances indicate the geometry is
* very flat in one or two directions.
*/
target_dim = get_target_dimension(dist, order, deg_ratio, d);
if (target_dim > 0){
/*
* Yes, geometry is degenerate
*/
if ((zz->Debug_Level > 0) && (zz->Proc == 0)){
if (d == 2){
sprintf(msg,
"Geometry (~%lf x %lf), exceeds %lf to 1.0 ratio",
dist[order[0]], dist[order[1]], deg_ratio);
}
else{
sprintf(msg,
"Geometry (~%lf x %lf x %lf), exceeds %lf to 1.0 ratio",
dist[order[0]], dist[order[1]], dist[order[2]], deg_ratio);
}
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
sprintf(msg, "We'll treat it as %d dimensional",target_dim);
ZOLTAN_PRINT_INFO(zz->Proc, yo, msg);
}
if (axis_aligned){
/*
** Create new geometry, transforming the primary direction
** to the X-axis, and the secondary to the Y-axis.
*/
tran->Permutation[0] = tran->Axis_Order[order[0]];
if (target_dim == 2){
tran->Permutation[1] = tran->Axis_Order[order[1]];
}
}
else{
/*
* Reorder the eigenvectors (they're the columns of evecs) from
* longest projected distance to shorted projected distance. Compute
* the transpose (the inverse) of the matrix. This will transform
* the geometry to align along the X-Y plane, or along the X axis.
*/
for (i=0; i< target_dim; i++){
tran->Transformation[i][2] = 0.0;
for (j=0; j<d; j++){
tran->Transformation[i][j] = tran->Evecs[j][order[i]];
}
}
for (i=target_dim; i< 3; i++){
for (j=0; j<3; j++){
tran->Transformation[i][j] = 0.0;
}
}
}
tran->Target_Dim = target_dim;
transform_coordinates(*coords, num_obj, d, tran);
} /* If geometry is very flat */
} /* If REDUCE_DIMENSIONS is true */
} /* If 2-D or 3-D rcb, rib or hsfc */
End:
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Error found; no coordinates returned.");
if (alloced_coords) ZOLTAN_FREE(coords);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return ierr;
}
static int scale_round_weights(
float *fwgts,
int *iwgts,
int n,
int dim,
int mode
)
{
/* Convert floating point weights to integer weights.
* This routine is stolen from scale_round_weights in parmetis_jostle.c.
* Because it needs to run only serially, and because it uses only
* integers (not idxtype), it has been largely duplicated here.
*/
int i, j, tmp, ierr;
int max_wgt_sum = INT_MAX/8;
int *nonint;
float *scale, *sum_wgt, *max_wgt;
char msg[256];
static char *yo = "scale_round_weights";
ierr = ZOLTAN_OK;
if (mode == 0) {
/* No scaling; just convert to int */
for (i=0; i<n*dim; i++){
iwgts[i] = (int) ceil((double) fwgts[i]);
}
}
else{
/* Allocate local arrays */
nonint = (int *)ZOLTAN_MALLOC(dim*sizeof(int));
scale = (float *)ZOLTAN_MALLOC(3*dim*sizeof(float));
sum_wgt = scale + dim;
max_wgt = sum_wgt + dim;
if (!(nonint && scale)){
ZOLTAN_PRINT_ERROR(0, yo, "Out of memory.");
ZOLTAN_FREE(&nonint);
ZOLTAN_FREE(&scale);
return ZOLTAN_MEMERR;
}
/* Initialize */
for (j=0; j<dim; j++){
nonint[j] = 0;
sum_wgt[j] = 0;
max_wgt[j] = 0;
}
/* Compute local sums of the weights */
/* Check if all weights are integers */
for (i=0; i<n; i++){
for (j=0; j<dim; j++){
if (!nonint[j]){
/* tmp = (int) roundf(fwgts[i]); EB: Valid C99, but not C89 */
tmp = (int) floor((double) fwgts[i] + .5); /* Nearest int */
if (fabs((double)tmp-fwgts[i*dim+j]) > INT_EPSILON){
nonint[j] = 1;
}
}
sum_wgt[j] += fwgts[i*dim+j];
if (fwgts[i*dim+j] > max_wgt[j])
max_wgt[j] = fwgts[i*dim+j];
}
}
/* Calculate scale factor */
for (j=0; j<dim; j++){
scale[j] = 1.;
/* Scale unless all weights are integers (not all zero) */
if (nonint[j] || (max_wgt[j] <= INT_EPSILON)
|| (sum_wgt[j] > max_wgt_sum)){
if (sum_wgt[j] == 0){
ierr = ZOLTAN_WARN;
sprintf(msg, "All weights are zero in component %1d", j);
ZOLTAN_PRINT_WARN(0, yo, msg);
}
else /* sum_wgt[j] != 0) */
scale[j] = max_wgt_sum/sum_wgt[j];
}
}
/* If mode==2, let the scale factor be the same for all weights */
if (mode==2){
for (j=1; j<dim; j++){
if (scale[j]<scale[0])
scale[0] = scale[j];
}
for (j=1; j<dim; j++){
scale[j] = scale[0];
}
}
/* Convert weights to positive integers using the computed scale factor */
for (i=0; i<n; i++){
for (j=0; j<dim; j++){
iwgts[i*dim+j] = (int) ceil((double) fwgts[i*dim+j]*scale[j]);
}
}
ZOLTAN_FREE(&nonint);
ZOLTAN_FREE(&scale);
}
return ierr;
}
static int Zoltan_Reftree_Partition(ZZ *zz, float *part_sizes, int *num_export,
ZOLTAN_ID_PTR *export_global_ids, ZOLTAN_ID_PTR *export_local_ids,
int **export_to_partition, int **export_procs)
{
/*
* Function to partition the grid and fill the export arrays.
*/
char *yo = "Zoltan_Reftree_Partition";
char msg[256];
int num_exp; /* count the number of export objects */
ZOLTAN_REFTREE *root; /* root of the tree */
float *cutoff; /* the relative sizes of the partitions */
int part; /* partition under construction */
float current_size; /* amount of weight consumed so far */
int num_part; /* number of partitions */
int ierr; /* error flag */
int wdim; /* Max(zz->Obj_Weight_Dim, 1) */
ZOLTAN_TRACE_ENTER(zz, yo);
root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root;
/*
* determine the size of the partitions and tolerance interval
*/
num_part = zz->LB.Num_Global_Parts;
/* Set the cutoff points of summed weights for the end of each partition */
/* TEMP HA
Determine a partition of unity for the sizes of the partitions
relative to the processing power of the processors, to support
heterogeneous architectures.
cutoff(0) = is the percent of work that should be assigned to
partition 0
cutoff(i) - cutoff(i-1) is the percent of work that should be
assigned to partition i.
cutoff(num_part-1) = 1.0
When support for heterogeneous architectures is added to Zoltan,
there should be a "vector of partition sizes" passed into this
routine, which would be used to define cutoff. For now, it is
just set to be equal sizes.
*/
cutoff = (float *)ZOLTAN_MALLOC((num_part)*sizeof(float));
if (cutoff == NULL) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_MEMERR);
}
cutoff[0] = part_sizes[0]; /* TEMP SINGLE WEIGHT */
wdim = ((zz->Obj_Weight_Dim == 0) ? 1 : zz->Obj_Weight_Dim);
for (part = 1; part < num_part; part++)
cutoff[part] = cutoff[part-1] + part_sizes[part*wdim]; /* TEMP SINGLE WEIGHT */
cutoff[num_part-1] = 1.0; /* just to make sure roundoff doesn't bite us */
/* multiply cutoff by the total weight so that cutoff gives the desired
actual weight of each partition instead of relative sizes */
/* TEMP HA
It is quite possible that once we know what is passed into this
routine to determine relative partition sizes, then this can be combined
with the above partition of unity to simplify the code. I just did it
this way to make it clear what is happening.
*/
for (part=0; part<num_part; part++) {
cutoff[part] = cutoff[part]*root->summed_weight[0]; /* TEMP SINGLE WEIGHT */
}
/*
* traverse the tree to define the partition and count the number of exports
*/
num_exp = 0;
part = 0;
current_size = 0.0;
ierr = Zoltan_Reftree_Part_Recursive(zz,root,&part,¤t_size,&num_exp,
cutoff,num_part);
ZOLTAN_FREE(&cutoff);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
/*
* if no exports, we're done
*/
if (zz->LB.Return_Lists == ZOLTAN_LB_NO_LISTS) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
else if (zz->LB.Return_Lists == ZOLTAN_LB_CANDIDATE_LISTS) {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Candidate Lists not supported in REFTREE;"
"change RETURN_LISTS parameter.");
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_FATAL);
}
else if (num_exp == 0) {
*num_export = 0;
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
/*
* allocate space for the export lists
*/
if (!Zoltan_Special_Malloc(zz,(void **)export_global_ids,num_exp,
ZOLTAN_SPECIAL_MALLOC_GID)) {
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_local_ids,num_exp,
ZOLTAN_SPECIAL_MALLOC_LID)) {
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_to_partition,num_exp,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)export_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
if (!Zoltan_Special_Malloc(zz,(void **)export_procs,num_exp,
ZOLTAN_SPECIAL_MALLOC_INT)) {
Zoltan_Special_Free(zz,(void **)export_to_partition,ZOLTAN_SPECIAL_MALLOC_INT);
Zoltan_Special_Free(zz,(void **)export_global_ids,ZOLTAN_SPECIAL_MALLOC_GID);
Zoltan_Special_Free(zz,(void **)export_local_ids,ZOLTAN_SPECIAL_MALLOC_LID);
ZOLTAN_TRACE_EXIT(zz, yo);
return ZOLTAN_MEMERR;
}
/*
* traverse the tree to make the export lists
*/
*num_export = 0;
ierr = Zoltan_Reftree_Export_Lists(zz,root,num_export,export_global_ids,
export_local_ids,export_to_partition,export_procs);
if (ierr != ZOLTAN_OK && ierr != ZOLTAN_WARN) {
ZOLTAN_TRACE_EXIT(zz, yo);
return(ierr);
}
if (num_exp != *num_export) {
sprintf(msg, "num_exp = %d not equal to num_export = %d.",
num_exp,*num_export);
ZOLTAN_PRINT_WARN(zz->Proc, yo, msg);
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_WARN);
}
ZOLTAN_TRACE_EXIT(zz, yo);
return(ZOLTAN_OK);
}
int Zoltan_PHG_Initialize_Params(
ZZ *zz, /* the Zoltan data structure */
float *part_sizes,
PHGPartParams *hgp
)
{
int err = ZOLTAN_OK;
char *yo = "Zoltan_PHG_Initialize_Params";
int nProc;
int usePrimeComm;
MPI_Comm communicator;
char add_obj_weight[MAX_PARAM_STRING_LEN];
char edge_weight_op[MAX_PARAM_STRING_LEN];
char cut_objective[MAX_PARAM_STRING_LEN];
char *package = hgp->hgraph_pkg;
char *method = hgp->hgraph_method;
char buf[1024];
memset(hgp, 0, sizeof(*hgp)); /* in the future if we forget to initialize
another param at least it will be 0 */
Zoltan_Bind_Param(PHG_params, "HYPERGRAPH_PACKAGE", &hgp->hgraph_pkg);
Zoltan_Bind_Param(PHG_params, "PHG_MULTILEVEL", &hgp->useMultilevel);
Zoltan_Bind_Param(PHG_params, "PHG_FROM_GRAPH_METHOD", hgp->convert_str);
Zoltan_Bind_Param(PHG_params, "PHG_OUTPUT_LEVEL", &hgp->output_level);
Zoltan_Bind_Param(PHG_params, "FINAL_OUTPUT", &hgp->final_output);
Zoltan_Bind_Param(PHG_params, "CHECK_GRAPH", &hgp->check_graph);
Zoltan_Bind_Param(PHG_params, "CHECK_HYPERGRAPH", &hgp->check_graph);
Zoltan_Bind_Param(PHG_params, "PHG_NPROC_VERTEX", &hgp->nProc_x_req);
Zoltan_Bind_Param(PHG_params, "PHG_NPROC_EDGE", &hgp->nProc_y_req);
Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_LIMIT", &hgp->redl);
Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_NCANDIDATE", &hgp->nCand);
Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_METHOD", hgp->redm_str);
Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_METHOD_FAST", hgp->redm_fast);
Zoltan_Bind_Param(PHG_params, "PHG_VERTEX_VISIT_ORDER", &hgp->visit_order);
Zoltan_Bind_Param(PHG_params, "PHG_EDGE_SCALING", &hgp->edge_scaling);
Zoltan_Bind_Param(PHG_params, "PHG_VERTEX_SCALING", &hgp->vtx_scaling);
Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_METHOD", hgp->refinement_str);
Zoltan_Bind_Param(PHG_params, "PHG_DIRECT_KWAY", &hgp->kway);
Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_LOOP_LIMIT",
&hgp->fm_loop_limit);
Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_MAX_NEG_MOVE",
&hgp->fm_max_neg_move);
Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_QUALITY",
&hgp->refinement_quality);
Zoltan_Bind_Param(PHG_params, "PHG_COARSEPARTITION_METHOD",
hgp->coarsepartition_str);
Zoltan_Bind_Param(PHG_params, "PHG_USE_TIMERS",
(void*) &hgp->use_timers);
Zoltan_Bind_Param(PHG_params, "USE_TIMERS",
(void*) &hgp->use_timers);
Zoltan_Bind_Param(PHG_params, "PHG_EDGE_SIZE_THRESHOLD",
(void*) &hgp->EdgeSizeThreshold);
Zoltan_Bind_Param(PHG_params, "PHG_MATCH_EDGE_SIZE_THRESHOLD",
(void*) &hgp->MatchEdgeSizeThreshold);
Zoltan_Bind_Param(PHG_params, "PHG_BAL_TOL_ADJUSTMENT",
(void*) &hgp->bal_tol_adjustment);
Zoltan_Bind_Param(PHG_params, "PARKWAY_SERPART",
(void *) hgp->parkway_serpart);
Zoltan_Bind_Param(PHG_params, "PHG_CUT_OBJECTIVE",
(void *) &cut_objective);
Zoltan_Bind_Param(PHG_params, "ADD_OBJ_WEIGHT",
(void *) add_obj_weight);
Zoltan_Bind_Param(PHG_params, "PHG_EDGE_WEIGHT_OPERATION",
(void *) edge_weight_op);
Zoltan_Bind_Param(PHG_params, "PHG_RANDOMIZE_INPUT",
(void*) &hgp->RandomizeInitDist);
Zoltan_Bind_Param(PHG_params, "PHG_PROCESSOR_REDUCTION_LIMIT",
(void*) &hgp->ProRedL);
Zoltan_Bind_Param(PHG_params, "PHG_REPART_MULTIPLIER",
(void*) &hgp->RepartMultiplier);
Zoltan_Bind_Param(PHG_params, "PATOH_ALLOC_POOL0",
(void*) &hgp->patoh_alloc_pool0);
Zoltan_Bind_Param(PHG_params, "PATOH_ALLOC_POOL1",
(void*) &hgp->patoh_alloc_pool1);
/* Set default values */
strncpy(hgp->hgraph_pkg, "phg", MAX_PARAM_STRING_LEN);
strncpy(hgp->convert_str, "neighbors", MAX_PARAM_STRING_LEN);
strncpy(hgp->redm_str, "agg", MAX_PARAM_STRING_LEN);
hgp->match_array_type = 0;
strncpy(hgp->redm_fast, "l-ipm", MAX_PARAM_STRING_LEN);
strncpy(hgp->coarsepartition_str, "auto", MAX_PARAM_STRING_LEN);
strncpy(hgp->refinement_str, "fm2", MAX_PARAM_STRING_LEN);
strncpy(hgp->parkway_serpart, "patoh", MAX_PARAM_STRING_LEN);
strncpy(cut_objective, "connectivity", MAX_PARAM_STRING_LEN);
strncpy(add_obj_weight, "none", MAX_PARAM_STRING_LEN);
strncpy(edge_weight_op, "max", MAX_PARAM_STRING_LEN);
/* LB.Approach is initialized to "REPARTITION", and set in Set_Key_Params */
strncpy(hgp->hgraph_method, zz->LB.Approach, MAX_PARAM_STRING_LEN);
if (!strcasecmp(zz->LB.Approach,"REFINE"))
hgp->useMultilevel = 0;
else
hgp->useMultilevel = 1;
hgp->use_timers = 0;
hgp->LocalCoarsePartition = 0;
hgp->edge_scaling = 0;
hgp->vtx_scaling = 0;
hgp->vtx_scal_size = 0;
hgp->vtx_scal = NULL; /* Array for storing vertex degree scale vector.
Should perhaps go in hg structure, not the
param struct? */
hgp->connectivity_cut = 1;
hgp->visit_order = 0; /* Random */
hgp->check_graph = 0;
hgp->bal_tol = zz->LB.Imbalance_Tol[0]; /* Make vector for multiconstraint */
hgp->bal_tol_adjustment = 0.7;
hgp->nCand = 100;
hgp->redl = MAX(2*zz->LB.Num_Global_Parts, 100);
hgp->output_level = PHG_DEBUG_NONE;
hgp->final_output = 0;
hgp->nProc_x_req = -1;
hgp->nProc_y_req = -1;
hgp->kway = 0;
hgp->fm_loop_limit = 10;
hgp->fm_max_neg_move = 250;
hgp->refinement_quality = 1;
hgp->RandomizeInitDist = 0;
hgp->EdgeSizeThreshold = 0.25;
hgp->MatchEdgeSizeThreshold = 500;
hgp->hybrid_keep_factor = 0.;
hgp->ProRedL = 0.0; /* UVCUVC: CHECK default set to 0 until we run more experiments */
hgp->RepartMultiplier = 100.;
hgp->patoh_alloc_pool0 = 0;
hgp->patoh_alloc_pool1 = 0;
hgp->UseFixedVtx = 0;
hgp->UsePrefPart = 0;
/* Get application values of parameters. */
err = Zoltan_Assign_Param_Vals(zz->Params, PHG_params, zz->Debug_Level,
zz->Proc, zz->Debug_Proc);
nProc = zz->Num_Proc;
usePrimeComm = 0;
/* Parse add_obj_weight parameter */
if (!strcasecmp(add_obj_weight, "none")) {
hgp->add_obj_weight = PHG_ADD_NO_WEIGHT;
hgp->part_sizes = part_sizes;
}
else if (zz->Obj_Weight_Dim > 0) {
/* Do not add_obj_weight until multiconstraint PHG is implemented */
ZOLTAN_PRINT_WARN(zz->Proc, yo,
"Both application supplied *and* ADD_OBJ_WEIGHT "
"calculated vertex weights were provided.");
ZOLTAN_PRINT_WARN(zz->Proc, yo,
"Only the first application supplied weight per vertex will be used.");
hgp->add_obj_weight = PHG_ADD_NO_WEIGHT;
hgp->part_sizes = part_sizes;
}
else {
if (!strcasecmp(add_obj_weight, "vertices")){
hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT;
} else if (!strcasecmp(add_obj_weight, "unit")){
hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT;
} else if (!strcasecmp(add_obj_weight, "vertex degree")){
hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT;
} else if (!strcasecmp(add_obj_weight, "nonzeros")){
hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT;
} else if (!strcasecmp(add_obj_weight, "pins")){
hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT;
} else{
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid ADD_OBJ_WEIGHT parameter.\n");
err = ZOLTAN_WARN;
}
/* Set hgp->part_sizes to new array of part_sizes with added obj weight. */
if (part_sizes)
err = Zoltan_LB_Add_Part_Sizes_Weight(zz,
(zz->Obj_Weight_Dim ? zz->Obj_Weight_Dim : 1),
zz->Obj_Weight_Dim+1,
part_sizes, &(hgp->part_sizes));
}
if ((zz->Obj_Weight_Dim==0) && /* no application supplied weights */
(hgp->add_obj_weight==PHG_ADD_NO_WEIGHT)){ /* no calculated weight */
hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT; /* default object weight */
}
if (!strcasecmp(cut_objective, "default")
|| !strcasecmp(cut_objective, "connectivity"))
hgp->connectivity_cut = 1;
else if (!strcasecmp(cut_objective, "hyperedges"))
hgp->connectivity_cut = 0;
else {
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid PHG_CUT_OBJECTIVE parameter.\n");
goto End;
}
if (!strcasecmp(edge_weight_op, "max")){
hgp->edge_weight_op = PHG_MAX_EDGE_WEIGHTS;
} else if (!strcasecmp(edge_weight_op, "add")){
hgp->edge_weight_op = PHG_ADD_EDGE_WEIGHTS;
} else if (!strcasecmp(edge_weight_op, "error")){
hgp->edge_weight_op = PHG_FLAG_ERROR_EDGE_WEIGHTS;
} else{
ZOLTAN_PRINT_ERROR(zz->Proc, yo,
"Invalid PHG_EDGE_WEIGHT_OPERATION parameter.\n");
err = ZOLTAN_WARN;
}
if ((strcasecmp(method, "PARTITION")) &&
(strcasecmp(method, "REPARTITION")) &&
(strcasecmp(method, "REFINE"))) {
sprintf(buf,"%s is not a valid hypergraph method\n",method);
ZOLTAN_PRINT_ERROR (zz->Proc, yo, buf);
err = ZOLTAN_FATAL;
goto End;
}
/* Adjust refinement parameters using hgp->refinement_quality */
if (hgp->refinement_quality < 0.5/hgp->fm_loop_limit)
/* No refinement */
strncpy(hgp->refinement_str, "no", MAX_PARAM_STRING_LEN);
else {
/* Scale FM parameters */
hgp->fm_loop_limit *= hgp->refinement_quality;
hgp->fm_max_neg_move *= hgp->refinement_quality;
}
if (!strcasecmp(package, "PHG")){
/* Test to determine whether we should change the number of processors
used for partitioning to make more efficient 2D decomposition */
if (hgp->nProc_x_req != 1 && hgp->nProc_y_req != 1) /* Want 2D decomp */
if (zz->Num_Proc > SMALL_PRIME && Zoltan_PHG_isPrime(zz->Num_Proc))
/* 2D data decomposition is requested but we have a prime
* number of processors. */
usePrimeComm = 1;
if ((!strcasecmp(method, "REPARTITION"))){
zz->LB.Remap_Flag = 0;
}
if ((!strcasecmp(method, "REPARTITION")) ||
(!strcasecmp(method, "REFINE"))) {
hgp->fm_loop_limit = 4; /* experimental evaluation showed that for
repartitioning/refinement small number of passes
is "good enough". These are all heuristics hence
it is possible to create a pathological cases;
but in general this seems to be sufficient */
}
if (!hgp->useMultilevel) {
/* don't do coarsening */
strncpy(hgp->redm_str, "no", MAX_PARAM_STRING_LEN);
/* we have modified all coarse partitioners to handle preferred part
if user wants to choose one she can choose; otherwise default
partitioner
(greedy growing) does work better than previous default partitioning
for phg_refine ("no"). */
hgp->UsePrefPart = 1;
}
if (!strcasecmp(method, "REFINE") && hgp->useMultilevel){
/* UVCUVC: as a heuristic we prefer local matching;
in our experiments for IPDPS'07 and WileyChapter multilevel_refine
didn't prove itself useful; it is too costly even with local matching
hence it will not be be released yet (i.e. not in v3). */
strncpy(hgp->redm_str, "l-ipm", MAX_PARAM_STRING_LEN);
hgp->UsePrefPart = 1;
}
}
else if (!strcasecmp(package, "PARKWAY")){
if (hgp->nProc_x_req>1) {
err = ZOLTAN_FATAL;
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "ParKway requires nProc_x=1 or -1.");
goto End;
}
hgp->nProc_x_req = 1;
}
else if (!strcasecmp(package, "PATOH")){
if (zz->Num_Proc>1) {
err = ZOLTAN_FATAL;
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "PaToH only works with Num_Proc=1.");
goto End;
}
}
if (!usePrimeComm)
MPI_Comm_dup(zz->Communicator, &communicator);
else {
MPI_Group newgrp, zzgrp;
nProc--;
MPI_Comm_group(zz->Communicator, &zzgrp);
MPI_Group_excl(zzgrp, 1, &nProc, &newgrp);
MPI_Comm_create(zz->Communicator, newgrp, &communicator);
MPI_Group_free(&newgrp);
MPI_Group_free(&zzgrp);
}
err = Zoltan_PHG_Set_2D_Proc_Distrib(zz, communicator, zz->Proc,
nProc, hgp->nProc_x_req,
hgp->nProc_y_req,
&hgp->globalcomm);
if (err != ZOLTAN_OK)
goto End;
/* Convert strings to function pointers. */
err = Zoltan_PHG_Set_Part_Options (zz, hgp);
End:
return err;
}