int
Zoltan_Matrix2d_adjproc (ZZ* zz, const Zoltan_matrix_2d * const mat, int **adjproc)
{
static char *yo = "Zoltan_Matrix2d_adjproc";
int ierr = ZOLTAN_OK;
int i;
ZOLTAN_TRACE_ENTER(zz, yo);
*adjproc = (int*) ZOLTAN_MALLOC(mat->mtx.nPins*sizeof(int));
if (mat->mtx.nPins && (*adjproc == NULL))
MEMORY_ERROR;
for (i = 0 ; i < mat->mtx.nPins ; ++i ) {
(*adjproc)[i] = EDGE_TO_PROC_Y(mat, mat->mtx.pinGNO[i]);
}
End:
ZOLTAN_TRACE_EXIT(zz, yo);
KDDKDDKDD(zz->Proc, "Zoltan_Matrix2d_adjproc done");
return (ierr);
}
int Zoltan_Color_Test(
ZZ *zz, /* Zoltan structure */
int *num_gid_entries, /* # of entries for a global id */
int *num_lid_entries, /* # of entries for a local id */
int num_obj, /* Input: number of objects */
ZOLTAN_ID_PTR global_ids, /* Input: global ids of the vertices */
/* The application must allocate enough space */
ZOLTAN_ID_PTR local_ids, /* Input: local ids of the vertices */
/* The application must allocate enough space */
int *color_exp /* Input: Colors assigned to local vertices */
)
{
static char *yo = "color_test_fn";
int nvtx = num_obj; /* number of vertices */
int i, j;
int ierr = ZOLTAN_OK;
int ferr = ZOLTAN_OK; /* final error signal */
char coloring_problem; /* Input: which coloring to perform;
currently only supports D1, D2 coloring and variants */
char coloring_problemStr[MAX_PARAM_STRING_LEN]; /* string version coloring problem name */
int ss=100;
char comm_pattern='S', coloring_order='I', coloring_method='F';
int comm[2],gcomm[2];
int *color=NULL;
int *adjproc=NULL, *xadj=NULL;
ZOLTAN_GNO_TYPE gvtx; /* number of global vertices */
ZOLTAN_GNO_TYPE *vtxdist=NULL, *adjncy=NULL;
ZG graph;
ZOLTAN_GNO_TYPE *requested_GNOs = NULL; /* Return GNOs of
the requested GIDs. */
int *loc_partialD2 = NULL; /* local binary array showing which vertices to be colored */
int *partialD2 = NULL; /* global binary array showing which vertices to be colored */
struct Zoltan_DD_Struct *dd_color; /* DDirectory for colors */
ZOLTAN_GNO_TYPE *local_GNOs = NULL;
ZOLTAN_GNO_TYPE *global_GNOs = NULL;
memset (&graph, 0, sizeof(ZG));
/* PARAMETER SETTINGS */
Zoltan_Bind_Param(Color_params, "COLORING_PROBLEM", (void *) &coloring_problemStr);
Zoltan_Bind_Param(Color_params, "SUPERSTEP_SIZE", (void *) &ss);
Zoltan_Bind_Param(Color_params, "COMM_PATTERN", (void *) &comm_pattern);
Zoltan_Bind_Param(Color_params, "VERTEX_VISIT_ORDER", (void *) &coloring_order);
Zoltan_Bind_Param(Color_params, "COLORING_METHOD", (void *) &coloring_method);
strncpy(coloring_problemStr, "distance-1", MAX_PARAM_STRING_LEN);
Zoltan_Assign_Param_Vals(zz->Params, Color_params, zz->Debug_Level, zz->Proc,
zz->Debug_Proc);
/* Check validity of parameters - they should be consistent with Zoltan_Color */
if (!strcasecmp(coloring_problemStr, "distance-1"))
coloring_problem = '1';
else if (!strcasecmp(coloring_problemStr, "distance-2"))
coloring_problem = '2';
else if (!strcasecmp(coloring_problemStr, "partial-distance-2")
|| !strcasecmp(coloring_problemStr, "bipartite"))
coloring_problem = 'P';
else {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Unknown coloring requested. Using Distance-1 coloring.");
coloring_problem = '1';
}
if (ss == 0) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid superstep size. Using default value 100.");
ss = 100;
}
if (comm_pattern != 'S' && comm_pattern != 'A') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid communication pattern. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (comm_pattern == 'A' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Asynchronous communication pattern is not implemented for distance-2 coloring and its variants. Using synchronous communication (S).");
comm_pattern = 'S';
}
if (coloring_order != 'I' && coloring_order != 'B' && coloring_order != 'U' && coloring_order != 'L' && coloring_order != 'N' && coloring_order != 'S') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring order. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_order == 'U' && (coloring_problem == '2' || coloring_problem == 'P')) {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Interleaved coloring order is not implemented for distance-2 coloring and its variants. Using internal first coloring order (I).");
coloring_order = 'I';
}
if (coloring_method !='F') {
ZOLTAN_PRINT_WARN(zz->Proc, yo, "Invalid coloring method. Using first fit method (F).");
coloring_method = 'F';
}
/* Compute Max number of array entries per ID over all processors.
This is a sanity-maintaining step; we don't want different
processors to have different values for these numbers. */
comm[0] = zz->Num_GID;
comm[1] = zz->Num_LID;
MPI_Allreduce(comm, gcomm, 2, MPI_INT, MPI_MAX, zz->Communicator);
zz->Num_GID = *num_gid_entries = gcomm[0];
zz->Num_LID = *num_lid_entries = gcomm[1];
/* Return if this processor is not in the Zoltan structure's
communicator. */
if (ZOLTAN_PROC_NOT_IN_COMMUNICATOR(zz))
return ZOLTAN_OK;
/* BUILD THE GRAPH */
/* Check that the user has allocated space for the return args. */
if (!color_exp)
ZOLTAN_COLOR_ERROR(ZOLTAN_FATAL, "Color argument is NULL. Please give colors of local vertices.");
requested_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(num_obj *
sizeof(ZOLTAN_GNO_TYPE));
Zoltan_ZG_Build (zz, &graph, 0, 1, num_obj,
global_ids, requested_GNOs);
Zoltan_ZG_Export (zz, &graph,
&gvtx, &nvtx, NULL, NULL,
&vtxdist, &xadj, &adjncy, &adjproc,
NULL, NULL);
if (gvtx > (ZOLTAN_GNO_TYPE)INT_MAX){
if (zz->Proc == 0){
fprintf(stderr,
"Zoltan_Color_Test assumes number of vertices (%ld) is less than INT_MAX\n",gvtx);
}
ierr = ZOLTAN_FATAL;
goto End;
}
KDDKDDKDD(zz->Proc, "COLORTEST DD");
/* Exchange global color information */
if (vtxdist[zz->Num_Proc] && !(color = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (nvtx && !(local_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(nvtx * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<nvtx; ++i)
local_GNOs[i] = i+vtxdist[zz->Proc];
if (vtxdist[zz->Num_Proc] && !(global_GNOs = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(vtxdist[zz->Num_Proc] * sizeof(ZOLTAN_GNO_TYPE))))
MEMORY_ERROR;
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
global_GNOs[i] = i;
ierr = Zoltan_DD_Create (&dd_color, zz->Communicator,
sizeof(ZOLTAN_GNO_TYPE)/sizeof(ZOLTAN_ID_TYPE), 0, 0, num_obj, 0);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot construct DDirectory.");
/* Put req obs with 1 but first inialize the rest with 0 */
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)local_GNOs, NULL,
NULL, color, nvtx);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
ierr = Zoltan_DD_Update (dd_color, (ZOLTAN_ID_PTR)requested_GNOs, NULL,
NULL, color_exp, num_obj);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot update DDirectory.");
/* Get requested colors from the DDirectory. */
ierr = Zoltan_DD_Find (dd_color, (ZOLTAN_ID_PTR)global_GNOs, NULL, NULL,
color, vtxdist[zz->Num_Proc], NULL);
if (ierr != ZOLTAN_OK)
ZOLTAN_COLOR_ERROR(ierr, "Cannot find object in DDirectory.");
/* Free DDirectory */
Zoltan_DD_Destroy(&dd_color);
ZOLTAN_FREE(&local_GNOs);
ZOLTAN_FREE(&global_GNOs);
KDDKDDKDD(zz->Proc, "COLORTEST CHECK");
if (coloring_problem == 'P' || coloring_problem == '2') {
if (vtxdist[zz->Num_Proc] && !(partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (vtxdist[zz->Num_Proc] && !(loc_partialD2 = (int *) ZOLTAN_CALLOC(vtxdist[zz->Num_Proc], sizeof(int))))
MEMORY_ERROR;
if (coloring_problem == 'P') {
for (i=0; i<num_obj; ++i) {
int gno=requested_GNOs[i];
loc_partialD2[gno] = 1;
}
MPI_Allreduce(loc_partialD2, partialD2, vtxdist[zz->Num_Proc], MPI_INT, MPI_LOR, zz->Communicator);
} else {
for (i=0; i<vtxdist[zz->Num_Proc]; ++i)
partialD2[i] = 1;
}
}
/* Check if there is an error in coloring */
if (coloring_problem == '1') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j];
if (color[gno] == color[v]) { /* neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, v:%d, cu:%d, cv:%d\n", gno, v, color[gno], color[v]);
break;
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else if (coloring_problem == '2' || coloring_problem == 'P') {
for (i=0; i<nvtx; i++) {
int gno = i + (int)vtxdist[zz->Proc];
if (partialD2[gno] && color[gno] <= 0) { /* object i is not colored properly */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! u:%d, cu:%d\n", gno, color[gno]);
break;
}
for (j = xadj[i]; j < xadj[i+1]; ++j) {
int v = (int)adjncy[j], k;
if (partialD2[v] && color[v] <= 0) {
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
if (partialD2[gno] && partialD2[v] && color[gno] == color[v]) { /* d-1 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d1-neigh: u:%d, v:%d, cu:%d, cv:%d pu:%d pv:%d\n", gno, v, color[gno], color[v], partialD2[gno], partialD2[v]);
break;
}
for (k = j+1; k < xadj[i+1]; ++k) {
int w = (int)adjncy[k];
if (partialD2[v] && partialD2[w] && color[v] == color[w]) { /* d-2 neighbors have the same color */
ierr = ZOLTAN_FATAL;
printf("Error in coloring! d2-neigh: v:%d, w:%d, cv:%d, cw:%d pv:%d pw:%d\n", v, w, color[v], color[w], partialD2[v], partialD2[w]);
break;
}
}
}
if (ierr == ZOLTAN_FATAL)
break;
}
} else
ZOLTAN_COLOR_ERROR(ZOLTAN_WARN, "Zoltan_Color_Test is only implemented for distance-1 and distance-2 coloring. Unknown coloring, skipping verification.");
End:
KDDKDDKDD(zz->Proc, "COLORTEST DONE");
if (ierr==ZOLTAN_FATAL)
ierr = 2;
MPI_Allreduce(&ierr, &ferr, 1, MPI_INT, MPI_MAX, zz->Communicator);
if (ferr == 2)
ierr = ZOLTAN_FATAL;
else
ierr = ZOLTAN_OK;
Zoltan_ZG_Free (&graph);
ZOLTAN_FREE(&adjproc);
ZOLTAN_FREE(&color);
ZOLTAN_FREE(&requested_GNOs);
ZOLTAN_FREE(&partialD2);
ZOLTAN_FREE(&loc_partialD2);
return ierr;
}
/* if !copy, inmat is not usable after this call */
int
Zoltan_Matrix2d_Distribute (ZZ* zz, Zoltan_matrix inmat, /* Cannot be const as we can share it inside outmat */
Zoltan_matrix_2d *outmat, int copy)
{
static char *yo = "Zoltan_Matrix2d_Distribute";
int ierr = ZOLTAN_OK;
int nProc_x, nProc_y;
int myProc_x, myProc_y;
int i, j, cnt;
int *proclist = NULL;
Zoltan_Arc *nonzeros= NULL, *sendbuf= NULL;
ZOLTAN_GNO_TYPE *perm_y = NULL;
float *wgtarray = NULL;
float *tmpwgtarray = NULL;
int msg_tag = 1021982;
ZOLTAN_COMM_OBJ *plan;
MPI_Comm communicator = MPI_COMM_NULL;
int nProc;
ZOLTAN_GNO_TYPE *yGNO = NULL;
ZOLTAN_GNO_TYPE *pinGNO = NULL;
ZOLTAN_GNO_TYPE tmp_gno;
void *partdata = NULL;
MPI_Datatype zoltan_gno_mpi_type;
ZOLTAN_TRACE_ENTER(zz, yo);
zoltan_gno_mpi_type = Zoltan_mpi_gno_type();
memcpy(&outmat->mtx, &inmat, sizeof(Zoltan_matrix));
if(copy) {
/* TODO: We need to copy the arrays also */
Zoltan_Matrix_Reset (&outmat->mtx);
/* Copy also directories */
outmat->mtx.ddX = Zoltan_DD_Copy (inmat.ddX);
if (inmat.ddY == inmat.ddX)
outmat->mtx.ddY = outmat->mtx.ddX;
else
outmat->mtx.ddY = Zoltan_DD_Copy (inmat.ddY);
}
communicator = outmat->comm->Communicator;
nProc = outmat->comm->nProc;
nProc_x = outmat->comm->nProc_x;
nProc_y = outmat->comm->nProc_y;
myProc_x = outmat->comm->myProc_x;
myProc_y = outmat->comm->myProc_y;
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_Duplicates");
ierr = Zoltan_Matrix_Remove_Duplicates(zz, outmat->mtx, &outmat->mtx);
/* KDDKDDKDD FIX INDENTATION OF THIS BLOCK */
if (inmat.opts.speed != MATRIX_NO_REDIST) {
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin) {
/* I need to know the original distribution */
if (outmat->mtx.ddX != outmat->mtx.ddY) { /* No initial distribution */
outmat->hashDistFct = (distFnct *)&Zoltan_Distribute_Linear;
}
else {
int *cmember = NULL;
cmember = (int*)ZOLTAN_MALLOC(outmat->mtx.nY*sizeof(int));
if (outmat->mtx.nY > 0 && cmember == NULL) MEMORY_ERROR;
Zoltan_DD_Find (outmat->mtx.ddY, (ZOLTAN_ID_PTR)outmat->mtx.yGNO, NULL, (char *)cmember, NULL,
outmat->mtx.nY, NULL);
KDDKDDKDD(zz->Proc, " Zoltan_Distribute_Partition_Register");
partdata = Zoltan_Distribute_Partition_Register(zz, outmat->mtx.nY, outmat->mtx.yGNO,
cmember, zz->Num_Proc, zz->Num_Proc);
ZOLTAN_FREE(&cmember);
Zoltan_Distribute_Set(outmat, (distFnct *)&Zoltan_Distribute_Origin, partdata);
}
}
/*
* Build comm plan for sending non-zeros to their target processors in
* 2D data distribution.
*/
/* TRICK: create fake arc (edgeno, -1) for empty Y. Upper bound for size might be nPins + nY */
proclist = (int *)ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY) *sizeof(int));
sendbuf = (Zoltan_Arc*) ZOLTAN_MALLOC((outmat->mtx.nPins +outmat->mtx.nY)* sizeof(Zoltan_Arc));
if ((outmat->mtx.nPins + outmat->mtx.nY >0) && (proclist == NULL || sendbuf == NULL)) MEMORY_ERROR;
wgtarray = (float*) ZOLTAN_MALLOC((outmat->mtx.nPins+outmat->mtx.nY)*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins*outmat->mtx.pinwgtdim && !wgtarray) MEMORY_ERROR;
yGNO = outmat->mtx.yGNO;
pinGNO = outmat->mtx.pinGNO;
KDDKDDKDD(zz->Proc, " CommPlan Hash");
cnt = 0;
for (i = 0; i < outmat->mtx.nY; i++) {
ZOLTAN_GNO_TYPE edge_gno=-1;
/* processor row for the edge */
edge_gno = yGNO[i];
for (j = outmat->mtx.ystart[i]; j < outmat->mtx.yend[i]; j++) {
ZOLTAN_GNO_TYPE vtx_gno=-1;
/* processor column for the vertex */
vtx_gno = pinGNO[j];
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, vtx_gno, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = vtx_gno;
memcpy(wgtarray+cnt*outmat->mtx.pinwgtdim, outmat->mtx.pinwgt+j*outmat->mtx.pinwgtdim,
outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
if(outmat->mtx.ystart[i] == outmat->mtx.yend[i]) {
proclist[cnt] = (*outmat->hashDistFct)(edge_gno, -1, outmat->hashDistData,
&sendbuf[cnt].part_y);
if (proclist[cnt] < 0) /* Discard this nnz */
continue;
sendbuf[cnt].GNO[0] = edge_gno;
sendbuf[cnt].GNO[1] = -1;
memset(wgtarray+cnt*outmat->mtx.pinwgtdim, 0,outmat->mtx.pinwgtdim*sizeof(float));
cnt++;
}
}
if (outmat->hashDistFct == (distFnct *)&Zoltan_Distribute_Origin)
Zoltan_Distribute_Partition_Free(&outmat->hashDistData);
if (outmat->mtx.yend != outmat->mtx.ystart + 1)
ZOLTAN_FREE(&outmat->mtx.yend);
outmat->mtx.yend = NULL;
ZOLTAN_FREE(&outmat->mtx.ystart);
ZOLTAN_FREE(&outmat->mtx.yGNO);
ZOLTAN_FREE(&outmat->mtx.pinGNO);
ZOLTAN_FREE(&outmat->mtx.pinwgt);
ZOLTAN_FREE(&outmat->mtx.yGID);
/*
* Send pins to their target processors.
* They become non-zeros in the 2D data distribution.
*/
KDDKDDKDD(zz->Proc, " CommPlan Create");
msg_tag--;
ierr = Zoltan_Comm_Create(&plan, cnt, proclist, communicator, msg_tag, &outmat->mtx.nPins);
ZOLTAN_FREE(&proclist);
nonzeros = (Zoltan_Arc *) ZOLTAN_MALLOC((outmat->mtx.nPins) * sizeof(Zoltan_Arc));
if (outmat->mtx.nPins && nonzeros == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) sendbuf, sizeof(Zoltan_Arc), (char *) nonzeros);
ZOLTAN_FREE(&sendbuf);
if (outmat->mtx.pinwgtdim) { /* We have to take care about weights */
tmpwgtarray = (float*) ZOLTAN_MALLOC(outmat->mtx.nPins*outmat->mtx.pinwgtdim*sizeof(float));
if (outmat->mtx.nPins && tmpwgtarray == NULL) MEMORY_ERROR;
msg_tag--;
Zoltan_Comm_Do(plan, msg_tag, (char *) wgtarray, outmat->mtx.pinwgtdim*sizeof(float),
(char *) tmpwgtarray);
ZOLTAN_FREE(&wgtarray);
}
Zoltan_Comm_Destroy(&plan);
/* Unpack the non-zeros received. */
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Remove_DupArcs");
/* TODO: do take care about singletons */
Zoltan_Matrix_Remove_DupArcs(zz, outmat->mtx.nPins, (Zoltan_Arc*)nonzeros, tmpwgtarray,
&outmat->mtx);
}
/* Now we just have to change numbering */
outmat->dist_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_y+1), sizeof(ZOLTAN_GNO_TYPE));
outmat->dist_x = (ZOLTAN_GNO_TYPE *) ZOLTAN_CALLOC((nProc_x+1), sizeof(ZOLTAN_GNO_TYPE));
if (outmat->dist_y == NULL || outmat->dist_x == NULL) MEMORY_ERROR;
/* FIXME: Work only in 1D */
tmp_gno = (ZOLTAN_GNO_TYPE)outmat->mtx.nY;
MPI_Allgather(&tmp_gno, 1, zoltan_gno_mpi_type, outmat->dist_y+1, 1, zoltan_gno_mpi_type, communicator);
for (i = 1 ; i <= nProc_y ; i ++) {
outmat->dist_y[i] += outmat->dist_y[i-1];
}
outmat->dist_x[1] = outmat->mtx.globalX;
perm_y = (ZOLTAN_GNO_TYPE *) ZOLTAN_MALLOC(outmat->mtx.nY * sizeof(ZOLTAN_GNO_TYPE));
if (outmat->mtx.nY > 0 && perm_y == NULL) MEMORY_ERROR;
for (i = 0 ; i < outmat->mtx.nY ; ++i){
perm_y[i] = i + outmat->dist_y[myProc_y];
}
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute");
Zoltan_Matrix_Permute(zz, &outmat->mtx, perm_y);
KDDKDDKDD(zz->Proc, " Zoltan_Matrix_Permute done");
End:
ZOLTAN_FREE(&perm_y);
ZOLTAN_FREE(&proclist);
ZOLTAN_FREE(&sendbuf);
ZOLTAN_FREE(&nonzeros);
ZOLTAN_FREE(&tmpwgtarray);
ZOLTAN_TRACE_EXIT(zz, yo);
return (ierr);
}